Turn off color for downstairs dump Creating region directory at: /var/tmp/test_repair Creating dsc directory at: /tmp/dsc Downstairs region 0 created at /var/tmp/test_repair/8810 in 0.26462752 Downstairs region 1 created at /var/tmp/test_repair/8820 in 0.020643055 Downstairs region 2 created at /var/tmp/test_repair/8830 in 0.020671172 Created 3 regions /var/tmp/bins/crutest with -t 127.0.0.1:8810 -t 127.0.0.1:8820 -t 127.0.0.1:8830 1184 1185 1186 Dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:10:58.044957985Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1188,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:10:58.117217919Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1188,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:10:58.118331349Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1188} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"abd5430f1508af0bc008143ea2566277ef714ec2\",\n git_commit_timestamp: \"2025-01-17T20:55:03.000000000Z\",\n git_branch: \"alan/tokio-to-140\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:10:58.118455962Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1188} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:10:58.118474193Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1188} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:10:58.118488421Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1188} {"msg":"Crucible ba66abf1-9183-416f-a691-0a85f0c29de8 has session id: 4ddd29d1-7240-4b99-a5aa-a2e16695a462","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:10:58.119415933Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1188,"session_id":"4ddd29d1-7240-4b99-a5aa-a2e16695a462"} {"msg":"Upstairs opts: Upstairs UUID: ba66abf1-9183-416f-a691-0a85f0c29de8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:10:58.11954677Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1188,"session_id":"4ddd29d1-7240-4b99-a5aa-a2e16695a462"} {"msg":"Crucible stats registered with UUID: ba66abf1-9183-416f-a691-0a85f0c29de8","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:10:58.119582401Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1188,"session_id":"4ddd29d1-7240-4b99-a5aa-a2e16695a462"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:10:58.119600451Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1188,"session_id":"4ddd29d1-7240-4b99-a5aa-a2e16695a462"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:10:58.120588148Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1188,"task":"crutest"} {"msg":"The guest has requested activation with gen:1","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:10:58.120612583Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1188} {"msg":"ba66abf1-9183-416f-a691-0a85f0c29de8 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:10:58.120681803Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1188,"session_id":"4ddd29d1-7240-4b99-a5aa-a2e16695a462"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:10:58.120698963Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1188,"client":"0","":"downstairs","session_id":"4ddd29d1-7240-4b99-a5aa-a2e16695a462"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:10:58.120748022Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1188,"client":"0","":"downstairs","session_id":"4ddd29d1-7240-4b99-a5aa-a2e16695a462"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:10:58.120762951Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1188,"client":"1","":"downstairs","session_id":"4ddd29d1-7240-4b99-a5aa-a2e16695a462"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:10:58.120776828Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1188,"client":"1","":"downstairs","session_id":"4ddd29d1-7240-4b99-a5aa-a2e16695a462"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:10:58.120857275Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1188,"client":"2","":"downstairs","session_id":"4ddd29d1-7240-4b99-a5aa-a2e16695a462"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:10:58.120883451Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1188,"client":"2","":"downstairs","session_id":"4ddd29d1-7240-4b99-a5aa-a2e16695a462"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:10:58.120904113Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1188,"":"io task","client":"2","":"downstairs","session_id":"4ddd29d1-7240-4b99-a5aa-a2e16695a462"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:10:58.120924875Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1188,"":"io task","client":"0","":"downstairs","session_id":"4ddd29d1-7240-4b99-a5aa-a2e16695a462"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:10:58.120946067Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1188,"":"io task","client":"1","":"downstairs","session_id":"4ddd29d1-7240-4b99-a5aa-a2e16695a462"} {"msg":"ds_connection connected from Ok(127.0.0.1:44426)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:10:58.12096759Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1188,"":"io task","client":"1","":"downstairs","session_id":"4ddd29d1-7240-4b99-a5aa-a2e16695a462"} {"msg":"ds_connection connected from Ok(127.0.0.1:40879)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:10:58.120988972Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1188,"":"io task","client":"0","":"downstairs","session_id":"4ddd29d1-7240-4b99-a5aa-a2e16695a462"} {"msg":"ds_connection connected from Ok(127.0.0.1:58102)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:10:58.121060834Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1188,"":"io task","client":"2","":"downstairs","session_id":"4ddd29d1-7240-4b99-a5aa-a2e16695a462"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:10:58.12254163Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1188,"client":"2","":"downstairs","session_id":"4ddd29d1-7240-4b99-a5aa-a2e16695a462"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:10:58.12257644Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1188,"client":"0","":"downstairs","session_id":"4ddd29d1-7240-4b99-a5aa-a2e16695a462"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:10:58.122602586Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1188,"client":"1","":"downstairs","session_id":"4ddd29d1-7240-4b99-a5aa-a2e16695a462"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:10:58.123605041Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1188,"client":"2","":"downstairs","session_id":"4ddd29d1-7240-4b99-a5aa-a2e16695a462"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:10:58.123640512Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1188,"client":"0","":"downstairs","session_id":"4ddd29d1-7240-4b99-a5aa-a2e16695a462"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:10:58.123663195Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1188,"client":"1","":"downstairs","session_id":"4ddd29d1-7240-4b99-a5aa-a2e16695a462"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:10:58.124667903Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1188,"session_id":"4ddd29d1-7240-4b99-a5aa-a2e16695a462"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:10:58.124750671Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1188,"session_id":"4ddd29d1-7240-4b99-a5aa-a2e16695a462"} {"msg":"[0]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:10:58.124955541Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1188,"":"downstairs","":"downstairs","session_id":"4ddd29d1-7240-4b99-a5aa-a2e16695a462"} {"msg":"[0]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:10:58.124974232Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1188,"":"downstairs","":"downstairs","session_id":"4ddd29d1-7240-4b99-a5aa-a2e16695a462"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:10:58.1249888Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1188,"":"downstairs","":"downstairs","session_id":"4ddd29d1-7240-4b99-a5aa-a2e16695a462"} {"msg":"[1]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:10:58.125016606Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1188,"":"downstairs","":"downstairs","session_id":"4ddd29d1-7240-4b99-a5aa-a2e16695a462"} {"msg":"[1]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:10:58.125030675Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1188,"":"downstairs","":"downstairs","session_id":"4ddd29d1-7240-4b99-a5aa-a2e16695a462"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:10:58.125044573Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1188,"":"downstairs","":"downstairs","session_id":"4ddd29d1-7240-4b99-a5aa-a2e16695a462"} {"msg":"[2]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:10:58.125057841Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1188,"":"downstairs","":"downstairs","session_id":"4ddd29d1-7240-4b99-a5aa-a2e16695a462"} {"msg":"[2]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:10:58.125070798Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1188,"":"downstairs","":"downstairs","session_id":"4ddd29d1-7240-4b99-a5aa-a2e16695a462"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:10:58.125087918Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1188,"":"downstairs","":"downstairs","session_id":"4ddd29d1-7240-4b99-a5aa-a2e16695a462"} {"msg":"Max found gen is 1","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:10:58.125100866Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1188,"":"downstairs","":"downstairs","session_id":"4ddd29d1-7240-4b99-a5aa-a2e16695a462"} {"msg":"Generation requested: 1 >= found:1","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:10:58.125113613Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1188,"":"downstairs","":"downstairs","session_id":"4ddd29d1-7240-4b99-a5aa-a2e16695a462"} {"msg":"Next flush: 1","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:10:58.125125961Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1188,"":"downstairs","":"downstairs","session_id":"4ddd29d1-7240-4b99-a5aa-a2e16695a462"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:10:58.125229681Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1188,"":"downstairs","":"downstairs","session_id":"4ddd29d1-7240-4b99-a5aa-a2e16695a462"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:10:58.125247632Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1188,"session_id":"4ddd29d1-7240-4b99-a5aa-a2e16695a462"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:10:58.125259789Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1188,"client":"0","":"downstairs","session_id":"4ddd29d1-7240-4b99-a5aa-a2e16695a462"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:10:58.125272836Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1188,"client":"1","":"downstairs","session_id":"4ddd29d1-7240-4b99-a5aa-a2e16695a462"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:10:58.125295089Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1188,"client":"2","":"downstairs","session_id":"4ddd29d1-7240-4b99-a5aa-a2e16695a462"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:10:58.125320524Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1188,"session_id":"4ddd29d1-7240-4b99-a5aa-a2e16695a462"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:10:58.125339265Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1188,"session_id":"4ddd29d1-7240-4b99-a5aa-a2e16695a462"} {"msg":"ba66abf1-9183-416f-a691-0a85f0c29de8 is now active with session: 4ddd29d1-7240-4b99-a5aa-a2e16695a462","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:10:58.125352523Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1188,"session_id":"4ddd29d1-7240-4b99-a5aa-a2e16695a462"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:10:58.12536444Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1188,"session_id":"4ddd29d1-7240-4b99-a5aa-a2e16695a462"} {"msg":"The guest has finished waiting for activation with:1","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:10:58.12538146Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1188} Wait for a query_work_queue command to finish before sending IO Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Fill test Read and Verify all blocks (0..600 range:false) Wrote out file "/tmp/test_repair_verify.data" CLIENT: Tests done. All submitted work has been ACK'd ---------------------------------------------------------------- Crucible gen:1 GIO:true work queues: Upstairs:0 downstairs:3 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1004 Acked Read 88 Done Done Done false 1005 Acked Read 256 Done Done Done false 1006 Acked Read 256 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1003 1003 1003 Downstairs last five completed: 1003 1002 1001 1000 Upstairs last five completed: 1006 1005 1004 1003 1002 CLIENT: Up:0 ds:3 act:3 ---------------------------------------------------------------- Crucible gen:1 GIO:true work queues: Upstairs:0 downstairs:0 Downstairs last five completed: 1007 1006 1005 1004 1003 Upstairs last five completed: 1007 1006 1005 1004 1003 CLIENT: Up:0 ds:0 act:3 CLIENT: All crucible jobs finished, exiting program {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:11:02.220835974Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1188} Begin loop 1 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:02.253229688Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1190,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:02.298942756Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1190,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:02.299596027Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1190} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"abd5430f1508af0bc008143ea2566277ef714ec2\",\n git_commit_timestamp: \"2025-01-17T20:55:03.000000000Z\",\n git_branch: \"alan/tokio-to-140\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:02.299678476Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1190} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:02.29970208Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1190} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:02.299717278Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1190} {"msg":"Crucible dc92aece-8268-469d-b7f5-0e0e48c236ab has session id: 8dd3d685-672d-446a-8786-d8e11fa3459f","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:02.300215449Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1190,"session_id":"8dd3d685-672d-446a-8786-d8e11fa3459f"} {"msg":"Upstairs opts: Upstairs UUID: dc92aece-8268-469d-b7f5-0e0e48c236ab, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:02.300263758Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1190,"session_id":"8dd3d685-672d-446a-8786-d8e11fa3459f"} {"msg":"Crucible stats registered with UUID: dc92aece-8268-469d-b7f5-0e0e48c236ab","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:02.300288002Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1190,"session_id":"8dd3d685-672d-446a-8786-d8e11fa3459f"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:11:02.300307943Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1190,"session_id":"8dd3d685-672d-446a-8786-d8e11fa3459f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:02.300841546Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1190,"task":"crutest"} {"msg":"The guest has requested activation with gen:2","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:02.300868031Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1190} {"msg":"dc92aece-8268-469d-b7f5-0e0e48c236ab active request set","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:02.300944246Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1190,"session_id":"8dd3d685-672d-446a-8786-d8e11fa3459f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:02.300962046Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1190,"client":"0","":"downstairs","session_id":"8dd3d685-672d-446a-8786-d8e11fa3459f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:02.300985579Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1190,"client":"0","":"downstairs","session_id":"8dd3d685-672d-446a-8786-d8e11fa3459f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:02.300999378Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1190,"client":"1","":"downstairs","session_id":"8dd3d685-672d-446a-8786-d8e11fa3459f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:02.301013266Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1190,"client":"1","":"downstairs","session_id":"8dd3d685-672d-446a-8786-d8e11fa3459f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:02.301025983Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1190,"client":"2","":"downstairs","session_id":"8dd3d685-672d-446a-8786-d8e11fa3459f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:02.301048796Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1190,"client":"2","":"downstairs","session_id":"8dd3d685-672d-446a-8786-d8e11fa3459f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:02.301070059Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1190,"":"io task","client":"2","":"downstairs","session_id":"8dd3d685-672d-446a-8786-d8e11fa3459f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:02.301095063Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1190,"":"io task","client":"0","":"downstairs","session_id":"8dd3d685-672d-446a-8786-d8e11fa3459f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:02.301120018Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1190,"":"io task","client":"1","":"downstairs","session_id":"8dd3d685-672d-446a-8786-d8e11fa3459f"} {"msg":"ds_connection connected from Ok(127.0.0.1:55659)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:02.301222658Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1190,"":"io task","client":"2","":"downstairs","session_id":"8dd3d685-672d-446a-8786-d8e11fa3459f"} {"msg":"ds_connection connected from Ok(127.0.0.1:34685)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:02.301263782Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1190,"":"io task","client":"0","":"downstairs","session_id":"8dd3d685-672d-446a-8786-d8e11fa3459f"} {"msg":"ds_connection connected from Ok(127.0.0.1:53495)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:02.30129494Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1190,"":"io task","client":"1","":"downstairs","session_id":"8dd3d685-672d-446a-8786-d8e11fa3459f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:02.302022816Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1190,"client":"0","":"downstairs","session_id":"8dd3d685-672d-446a-8786-d8e11fa3459f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:02.302061548Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1190,"client":"1","":"downstairs","session_id":"8dd3d685-672d-446a-8786-d8e11fa3459f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:02.302670624Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1190,"client":"0","":"downstairs","session_id":"8dd3d685-672d-446a-8786-d8e11fa3459f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:02.302959704Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1190,"client":"1","":"downstairs","session_id":"8dd3d685-672d-446a-8786-d8e11fa3459f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:02.303420253Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1190,"session_id":"8dd3d685-672d-446a-8786-d8e11fa3459f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:02.303453062Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1190,"session_id":"8dd3d685-672d-446a-8786-d8e11fa3459f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:02.303713874Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1190,"client":"2","":"downstairs","session_id":"8dd3d685-672d-446a-8786-d8e11fa3459f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:02.304206502Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1190,"client":"2","":"downstairs","session_id":"8dd3d685-672d-446a-8786-d8e11fa3459f"} {"msg":"[0]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:02.304691045Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1190,"":"downstairs","":"downstairs","session_id":"8dd3d685-672d-446a-8786-d8e11fa3459f"} {"msg":"[0]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:02.304714099Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1190,"":"downstairs","":"downstairs","session_id":"8dd3d685-672d-446a-8786-d8e11fa3459f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:02.304729158Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1190,"":"downstairs","":"downstairs","session_id":"8dd3d685-672d-446a-8786-d8e11fa3459f"} {"msg":"[1]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:02.304742856Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1190,"":"downstairs","":"downstairs","session_id":"8dd3d685-672d-446a-8786-d8e11fa3459f"} {"msg":"[1]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:02.304756083Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1190,"":"downstairs","":"downstairs","session_id":"8dd3d685-672d-446a-8786-d8e11fa3459f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:02.304769531Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1190,"":"downstairs","":"downstairs","session_id":"8dd3d685-672d-446a-8786-d8e11fa3459f"} {"msg":"[2]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:02.304782569Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1190,"":"downstairs","":"downstairs","session_id":"8dd3d685-672d-446a-8786-d8e11fa3459f"} {"msg":"[2]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:02.304798658Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1190,"":"downstairs","":"downstairs","session_id":"8dd3d685-672d-446a-8786-d8e11fa3459f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:02.304812376Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1190,"":"downstairs","":"downstairs","session_id":"8dd3d685-672d-446a-8786-d8e11fa3459f"} {"msg":"Max found gen is 2","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:02.304825074Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1190,"":"downstairs","":"downstairs","session_id":"8dd3d685-672d-446a-8786-d8e11fa3459f"} {"msg":"Generation requested: 2 >= found:2","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:02.304837791Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1190,"":"downstairs","":"downstairs","session_id":"8dd3d685-672d-446a-8786-d8e11fa3459f"} {"msg":"Next flush: 2","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:02.304850338Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1190,"":"downstairs","":"downstairs","session_id":"8dd3d685-672d-446a-8786-d8e11fa3459f"} {"Wait for a query_work_queue command to finish before sending IO msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:02.304871441Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1190,"":"downstairs","":"downstairs","session_id":"8dd3d685-672d-446a-8786-d8e11fa3459f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:02.304894935Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1190,"session_id":"8dd3d685-672d-446a-8786-d8e11fa3459f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:02.304918008Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1190,"client":"0","":"downstairs","session_id":"8dd3d685-672d-446a-8786-d8e11fa3459f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:02.304943893Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1190,"client":"1","":"downstairs","session_id":"8dd3d685-672d-446a-8786-d8e11fa3459f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:02.304969648Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1190,"client":"2","":"downstairs","session_id":"8dd3d685-672d-446a-8786-d8e11fa3459f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:02.305012263Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1190,"session_id":"8dd3d685-672d-446a-8786-d8e11fa3459f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:02.30504087Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1190,"session_id":"8dd3d685-672d-446a-8786-d8e11fa3459f"} {"msg":"dc92aece-8268-469d-b7f5-0e0e48c236ab is now active with session: 8dd3d685-672d-446a-8786-d8e11fa3459f","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:02.305067215Z","hostname":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1190,"session_id":"8dd3d685-672d-446a-8786-d8e11fa3459f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:02.305096052Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1190,"session_id":"8dd3d685-672d-446a-8786-d8e11fa3459f"} {"msg":"The guest has finished waiting for activation with:2","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:02.305119226Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1190} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 456 len 40960 data: 2 2 2 2 2 2 2 2 2 2 02/30 Read block 317 len 28672 03/30 Write block 21 len 4096 data: 2 04/30 Read block 54 len 32768 05/30 Write block 377 len 28672 data: 2 2 2 2 2 2 2 06/30 Write block 188 len 36864 data: 2 2 2 2 2 2 2 2 2 07/30 Read block 555 len 32768 08/30 Write block 521 len 4096 data: 2 09/30 Read block 494 len 36864 10/30 Read block 560 len 36864 11/30 Write block 487 len 8192 data: 2 2 12/30 Read block 162 len 12288 13/30 Read block 51 len 20480 14/30 Write block 74 len 24576 data: 2 2 2 2 2 2 15/30 Write block 377 len 12288 data: 3 3 3 16/30 Read block 118 len 20480 17/30 Read block 121 len 36864 18/30 Write block 0 len 24576 data: 2 2 2 2 2 2 19/30 Read block 21 len 24576 20/30 Read block 161 len 4096 21/30 Write block 361 len 24576 data: 2 2 2 2 2 2 22/30 Read block 389 len 28672 23/30 Write block 266 len 28672 data: 2 2 2 2 2 2 2 24/30 Flush 25/30 Write block 503 len 32768 data: 2 2 2 2 2 2 2 2 26/30 Read block 258 len 12288 27/30 Write block 480 len 40960 data: 2 2 2 2 2 2 2 3 3 2 28/30 Read block 569 len 20480 29/30 Write block 566 len 4096 data: 2 30/30 Read block 238 len 20480 ---------------------------------------------------------------- Crucible gen:2 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 8 Done Done Done false 1025 Acked Read 3 Done Done Done false 1026 Acked Write 10 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Write 1 Done Done Done false 1029 Acked Read 5 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 6 5 5 16 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:11:02.326219938Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1190} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 2 2 2 2 2 2 F F F 1 020-039 2 2 2 2 2 2 F F F 2 040-059 1 1 1 1 1 1 F F F 3 060-079 2 2 2 2 2 2 F F F 4 080-099 1 1 1 1 1 1 F F F 5 100-119 1 1 1 1 1 1 F F F 6 120-139 1 1 1 1 1 1 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 1 1 1 1 1 1 F F F 9 180-199 2 2 2 2 2 2 F F F 10 200-219 1 1 1 1 1 1 F F F 11 220-239 1 1 1 1 1 1 F F F 12 240-259 1 1 1 1 1 1 F F F 13 260-279 2 2 2 2 2 2 F F F 14 280-299 1 1 1 1 1 1 F F F 15 300-319 1 1 1 1 1 1 F F F 16 320-339 1 1 1 1 1 1 F F F 17 340-359 1 1 1 1 1 1 F F F 18 360-379 2 2 2 2 2 2 F F F 19 380-399 2 2 2 2 2 2 F F F 20 400-419 1 1 1 1 1 1 F F F 21 420-439 1 1 1 1 1 1 F F F 22 440-459 2 2 2 2 2 2 F F F 23 460-479 2 2 2 2 2 2 F F F 24 480-499 2 2 2 2 2 2 T T T 25 500-519 1 1 1 1 1 1 T T T 26 520-539 2 2 2 2 2 2 F F F 27 540-559 1 1 1 1 1 1 F F F 28 560-579 1 1 1 1 1 1 T T T 29 580-599 1 1 1 1 1 1 F F F Max gen: 2, Max flush: 2 A Difference in extent metadata is expected here On loop 1 Verifying data now No change after verify Loop: 1 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 2 2 2 2 2 2 F F F 1 020-039 2 2 2 2 2 2 F F F 2 040-059 1 1 1 1 1 1 F F F 3 060-079 2 2 2 2 2 2 F F F 4 080-099 1 1 1 1 1 1 F F F 5 100-119 1 1 1 1 1 1 F F F 6 120-139 1 1 1 1 1 1 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 1 1 1 1 1 1 F F F 9 180-199 2 2 2 2 2 2 F F F 10 200-219 1 1 1 1 1 1 F F F 11 220-239 1 1 1 1 1 1 F F F 12 240-259 1 1 1 1 1 1 F F F 13 260-279 2 2 2 2 2 2 F F F 14 280-299 1 1 1 1 1 1 F F F 15 300-319 1 1 1 1 1 1 F F F 16 320-339 1 1 1 1 1 1 F F F 17 340-359 1 1 1 1 1 1 F F F 18 360-379 2 2 2 2 2 2 F F F 19 380-399 2 2 2 2 2 2 F F F 20 400-419 1 1 1 1 1 1 F F F 21 420-439 1 1 1 1 1 1 F F F 22 440-459 2 2 2 2 2 2 F F F 23 460-479 2 2 2 2 2 2 F F F 24 480-499 3 3 3 3 3 3 F F F 25 500-519 3 3 3 3 3 3 F F F 26 520-539 2 2 2 2 2 2 F F F 27 540-559 1 1 1 1 1 1 F F F 28 560-579 3 3 3 3 3 3 F F F 29 580-599 1 1 1 1 1 1 F F F Max gen: 3, Max flush: 3 Begin loop 2 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:06.56571171Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1198,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:11:06.609158623Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1198,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:06.609570454Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1198,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:06.641845968Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1198,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:06.642445819Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1198} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"abd5430f1508af0bc008143ea2566277ef714ec2\",\n git_commit_timestamp: \"2025-01-17T20:55:03.000000000Z\",\n git_branch: \"alan/tokio-to-140\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:06.642479809Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1198} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:06.642493647Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1198} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:06.642508795Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1198} {"msg":"Crucible 55089a67-6472-46dd-af74-4cf4e2f93578 has session id: 728d73ab-fe27-45cb-8f64-d0968c8e6787","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:06.642929452Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1198,"session_id":"728d73ab-fe27-45cb-8f64-d0968c8e6787"} {"msg":"Upstairs opts: Upstairs UUID: 55089a67-6472-46dd-af74-4cf4e2f93578, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:06.642968744Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1198,"session_id":"728d73ab-fe27-45cb-8f64-d0968c8e6787"} {"msg":"Crucible stats registered with UUID: 55089a67-6472-46dd-af74-4cf4e2f93578","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:06.642993229Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1198,"session_id":"728d73ab-fe27-45cb-8f64-d0968c8e6787"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:11:06.643013961Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1198,"session_id":"728d73ab-fe27-45cb-8f64-d0968c8e6787"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:06.643557198Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1198,"task":"crutest"} {"msg":"The guest has requested activation with gen:4","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:06.643587036Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1198} {"msg":"55089a67-6472-46dd-af74-4cf4e2f93578 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:06.643680491Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1198,"session_id":"728d73ab-fe27-45cb-8f64-d0968c8e6787"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:06.643712409Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1198,"client":"0","":"downstairs","session_id":"728d73ab-fe27-45cb-8f64-d0968c8e6787"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:06.643760387Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1198,"client":"0","":"downstairs","session_id":"728d73ab-fe27-45cb-8f64-d0968c8e6787"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:06.64379997Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1198,"client":"1","":"downstairs","session_id":"728d73ab-fe27-45cb-8f64-d0968c8e6787"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:06.643825905Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1198,"client":"1","":"downstairs","session_id":"728d73ab-fe27-45cb-8f64-d0968c8e6787"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:06.64385108Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1198,"client":"2","":"downstairs","session_id":"728d73ab-fe27-45cb-8f64-d0968c8e6787"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:06.643877816Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1198,"client":"2","":"downstairs","session_id":"728d73ab-fe27-45cb-8f64-d0968c8e6787"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:06.643899248Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1198,"":"io task","client":"2","":"downstairs","session_id":"728d73ab-fe27-45cb-8f64-d0968c8e6787"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:06.643916979Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1198,"":"io task","client":"0","":"downstairs","session_id":"728d73ab-fe27-45cb-8f64-d0968c8e6787"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:06.643930727Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1198,"":"io task","client":"1","":"downstairs","session_id":"728d73ab-fe27-45cb-8f64-d0968c8e6787"} {"msg":"ds_connection connected from Ok(127.0.0.1:49065)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:06.643955381Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1198,"":"io task","client":"2","":"downstairs","session_id":"728d73ab-fe27-45cb-8f64-d0968c8e6787"} {"msg":"ds_connection connected from Ok(127.0.0.1:48664)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:06.643969369Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1198,"":"io task","client":"0","":"downstairs","session_id":"728d73ab-fe27-45cb-8f64-d0968c8e6787"} {"msg":"ds_connection connected from Ok(127.0.0.1:38099)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:06.644157599Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1198,"":"io task","client":"1","":"downstairs","session_id":"728d73ab-fe27-45cb-8f64-d0968c8e6787"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:06.644687919Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1198,"client":"1","":"downstairs","session_id":"728d73ab-fe27-45cb-8f64-d0968c8e6787"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:06.644727843Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1198,"client":"2","":"downstairs","session_id":"728d73ab-fe27-45cb-8f64-d0968c8e6787"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:06.644873277Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1198,"client":"0","":"downstairs","session_id":"728d73ab-fe27-45cb-8f64-d0968c8e6787"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:06.645312684Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1198,"client":"1","":"downstairs","session_id":"728d73ab-fe27-45cb-8f64-d0968c8e6787"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:06.645345944Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1198,"client":"2","":"downstairs","session_id":"728d73ab-fe27-45cb-8f64-d0968c8e6787"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:06.645602934Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1198,"client":"0","":"downstairs","session_id":"728d73ab-fe27-45cb-8f64-d0968c8e6787"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:06.645994453Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1198,"session_id":"728d73ab-fe27-45cb-8f64-d0968c8e6787"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:06.646019217Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1198,"session_id":"728d73ab-fe27-45cb-8f64-d0968c8e6787"} {"msg":"[0]R flush_numbers[0..12]: [2, 2, 1, 2, 1, 1, 1, 1, 1, 2, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:06.646129042Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1198,"":"downstairs","":"downstairs","session_id":"728d73ab-fe27-45cb-8f64-d0968c8e6787"} {"msg":"[0]R generation[0..12]: [2, 2, 1, 2, 1, 1, 1, 1, 1, 2, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:06.646149053Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1198,"":"downstairs","":"downstairs","session_id":"728d73ab-fe27-45cb-8f64-d0968c8e6787"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:06.646166664Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1198,"":"downstairs","":"downstairs","session_id":"728d73ab-fe27-45cb-8f64-d0968c8e6787"} {"msg":"[1]R flush_numbers[0..12]: [2, 2, 1, 2, 1, 1, 1, 1, 1, 2, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:06.646180542Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1198,"":"downstairs","":"downstairs","session_id":"728d73ab-fe27-45cb-8f64-d0968c8e6787"} {"msg":"[1]R generation[0..12]: [2, 2, 1, 2, 1, 1, 1, 1, 1, 2, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:06.646193809Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1198,"":"downstairs","":"downstairs","session_id":"728d73ab-fe27-45cb-8f64-d0968c8e6787"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:06.646207848Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1198,"":"downstairs","":"downstairs","session_id":"728d73ab-fe27-45cb-8f64-d0968c8e6787"} {"msg":"[2]R flush_numbers[0..12]: [2, 2, 1, 2, 1, 1, 1, 1, 1, 2, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:06.646230421Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1198,"":"downstairs","":"downstairs","session_id":"728d73ab-fe27-45cb-8f64-d0968c8e6787"} {"msg":"[2]R generation[0..12]: [2, 2, 1, 2, 1, 1, 1, 1, 1, 2, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:06.646257016Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1198,"":"downstairs","":"downstairs","session_id":"728d73ab-fe27-45cb-8f64-d0968c8e6787"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:06.646283532Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1198,"":"downstairs","":"downstairs","session_id":"728d73ab-fe27-45cb-8f64-d0968c8e6787"Wait for a query_work_queue command to finish before sending IO } {"msg":"Max found gen is 4","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:06.646312619Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1198,"":"downstairs","":"downstairs","session_id":"728d73ab-fe27-45cb-8f64-d0968c8e6787"} {"msg":"Generation requested: 4 >= found:4","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:06.646338734Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1198,"":"downstairs","":"downstairs","session_id":"728d73ab-fe27-45cb-8f64-d0968c8e6787"} {"msg":"Next flush: 4","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:06.646354513Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1198,"":"downstairs","":"downstairs","session_id":"728d73ab-fe27-45cb-8f64-d0968c8e6787"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:06.646367191Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1198,"":"downstairs","":"downstairs","session_id":"728d73ab-fe27-45cb-8f64-d0968c8e6787"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:06.646385251Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1198,"session_id":"728d73ab-fe27-45cb-8f64-d0968c8e6787"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:06.646406013Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1198,"client":"0","":"downstairs","session_id":"728d73ab-fe27-45cb-8f64-d0968c8e6787"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:06.646432359Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1198,"client":"1","":"downstairs","session_id":"728d73ab-fe27-45cb-8f64-d0968c8e6787"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:06.646449949Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1198,"client":"2","":"downstairs","session_id":"728d73ab-fe27-45cb-8f64-d0968c8e6787"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:06.646472132Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1198,"session_id":"728d73ab-fe27-45cb-8f64-d0968c8e6787"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:06.646495136Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1198,"session_id":"728d73ab-fe27-45cb-8f64-d0968c8e6787"} {"msg":"55089a67-6472-46dd-af74-4cf4e2f93578 is now active with session: 728d73ab-fe27-45cb-8f64-d0968c8e6787","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:06.64651978Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1198,"session_id":"728d73ab-fe27-45cb-8f64-d0968c8e6787"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:06.646537821Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1198,"session_id":"728d73ab-fe27-45cb-8f64-d0968c8e6787"} {"msg":"The guest has finished waiting for activation with:4","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:06.646550378Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1198} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 114 len 28672 data: 2 2 2 2 2 2 2 02/30 Read block 264 len 36864 03/30 Write block 599 len 4096 data: 2 04/30 Write block 406 len 32768 data: 2 2 2 2 2 2 2 2 05/30 Read block 546 len 40960 06/30 Read block 474 len 40960 07/30 Write block 428 len 20480 data: 2 2 2 2 2 08/30 Write block 399 len 8192 data: 2 2 09/30 Write block 372 len 36864 data: 2 2 2 2 2 4 4 4 3 10/30 Read block 464 len 24576 11/30 Read block 374 len 20480 12/30 Read block 569 len 12288 13/30 Read block 451 len 12288 14/30 Read block 575 len 8192 15/30 Read block 312 len 28672 16/30 Write block 61 len 4096 data: 2 17/30 Read block 578 len 8192 18/30 Read block 7 len 40960 19/30 Read block 212 len 36864 20/30 Read block 467 len 20480 21/30 Write block 45 len 32768 data: 2 2 2 2 2 2 2 2 22/30 Flush 23/30 Write block 555 len 36864 data: 2 2 2 2 2 2 2 2 2 24/30 Read block 561 len 20480 25/30 Read block 558 len 28672 26/30 Flush 27/30 Write block 374 len 40960 data: 3 3 3 5 5 5 4 3 3 3 28/30 Write block 394 len 40960 data: 2 2 2 2 2 3 3 2 2 2 29/30 Read block 2 len 24576 30/30 Read block 408 len 36864 ---------------------------------------------------------------- Crucible gen:4 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 10 Done Done Done false 1027 Acked Write 10 Done Done Done false 1028 Acked Read 6 Done Done Done false 1029 Acked Read 9 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 0 1 Done 3 4 4 11 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:11:06.66718825Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1198} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 2 2 2 2 2 2 F F F 1 020-039 2 2 2 2 2 2 F F F 2 040-059 4 4 4 4 4 4 F F F 3 060-079 4 4 4 4 4 4 F F F 4 080-099 1 1 1 1 1 1 F F F 5 100-119 4 4 4 4 4 4 F F F 6 120-139 4 4 4 4 4 4 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 1 1 1 1 1 1 F F F 9 180-199 2 2 2 2 2 2 F F F 10 200-219 1 1 1 1 1 1 F F F 11 220-239 1 1 1 1 1 1 F F F 12 240-259 1 1 1 1 1 1 F F F 13 260-279 2 2 2 2 2 2 F F F 14 280-299 1 1 1 1 1 1 F F F 15 300-319 1 1 1 1 1 1 F F F 16 320-339 1 1 1 1 1 1 F F F 17 340-359 1 1 1 1 1 1 F F F 18 360-379 4 4 4 4 4 4 T T T 19 380-399 4 4 4 4 4 4 T T T 20 400-419 4 4 4 4 4 4 T T T 21 420-439 4 4 4 4 4 4 F F F 22 440-459 2 2 2 2 2 2 F F F 23 460-479 2 2 2 2 2 2 F F F 24 480-499 3 3 3 3 3 3 F F F 25 500-519 3 3 3 3 3 3 F F F 26 520-539 2 2 2 2 2 2 F F F 27 540-559 4 4 4 5 5 5 F F F 28 560-579 4 4 4 5 5 5 F F F 29 580-599 4 4 4 4 4 4 F F F Max gen: 4, Max flush: 5 A Difference in extent metadata is expected here On loop 2 Verifying data now No change after verify Loop: 2 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 2 2 2 2 2 2 F F F 1 020-039 2 2 2 2 2 2 F F F 2 040-059 4 4 4 4 4 4 F F F 3 060-079 4 4 4 4 4 4 F F F 4 080-099 1 1 1 1 1 1 F F F 5 100-119 4 4 4 4 4 4 F F F 6 120-139 4 4 4 4 4 4 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 1 1 1 1 1 1 F F F 9 180-199 2 2 2 2 2 2 F F F 10 200-219 1 1 1 1 1 1 F F F 11 220-239 1 1 1 1 1 1 F F F 12 240-259 1 1 1 1 1 1 F F F 13 260-279 2 2 2 2 2 2 F F F 14 280-299 1 1 1 1 1 1 F F F 15 300-319 1 1 1 1 1 1 F F F 16 320-339 1 1 1 1 1 1 F F F 17 340-359 1 1 1 1 1 1 F F F 18 360-379 5 5 5 6 6 6 F F F 19 380-399 5 5 5 6 6 6 F F F 20 400-419 5 5 5 6 6 6 F F F 21 420-439 4 4 4 4 4 4 F F F 22 440-459 2 2 2 2 2 2 F F F 23 460-479 2 2 2 2 2 2 F F F 24 480-499 3 3 3 3 3 3 F F F 25 500-519 3 3 3 3 3 3 F F F 26 520-539 2 2 2 2 2 2 F F F 27 540-559 4 4 4 5 5 5 F F F 28 560-579 4 4 4 5 5 5 F F F 29 580-599 4 4 4 4 4 4 F F F Max gen: 5, Max flush: 6 Begin loop 3 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:10.924661401Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1206,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:11:10.968800299Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1206,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:10.96924644Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1206,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:11.002486457Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1206,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:11.003436973Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1206} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"abd5430f1508af0bc008143ea2566277ef714ec2\",\n git_commit_timestamp: \"2025-01-17T20:55:03.000000000Z\",\n git_branch: \"alan/tokio-to-140\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:11.003471743Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1206} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:11.003485151Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1206} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:11.00349934Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1206} {"msg":"Crucible 7f905feb-e9ad-4668-879a-7a7f585d87c1 has session id: c18ac76c-4eb1-43f5-b7a5-25d5b6a3566e","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:11.004145927Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1206,"session_id":"c18ac76c-4eb1-43f5-b7a5-25d5b6a3566e"} {"msg":"Upstairs opts: Upstairs UUID: 7f905feb-e9ad-4668-879a-7a7f585d87c1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:11.004187111Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1206,"session_id":"c18ac76c-4eb1-43f5-b7a5-25d5b6a3566e"} {"msg":"Crucible stats registered with UUID: 7f905feb-e9ad-4668-879a-7a7f585d87c1","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:11.004211245Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1206,"session_id":"c18ac76c-4eb1-43f5-b7a5-25d5b6a3566e"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:11:11.004225244Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1206,"session_id":"c18ac76c-4eb1-43f5-b7a5-25d5b6a3566e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:11.004649381Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1206,"task":"crutest"} {"msg":"The guest has requested activation with gen:6","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:11.004672425Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1206} {"msg":"7f905feb-e9ad-4668-879a-7a7f585d87c1 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:11.004738994Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1206,"session_id":"c18ac76c-4eb1-43f5-b7a5-25d5b6a3566e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:11.004755724Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1206,"client":"0","":"downstairs","session_id":"c18ac76c-4eb1-43f5-b7a5-25d5b6a3566e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:11.004780068Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1206,"client":"0","":"downstairs","session_id":"c18ac76c-4eb1-43f5-b7a5-25d5b6a3566e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:11.004796758Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1206,"client":"1","":"downstairs","session_id":"c18ac76c-4eb1-43f5-b7a5-25d5b6a3566e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:11.00481719Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1206,"client":"1","":"downstairs","session_id":"c18ac76c-4eb1-43f5-b7a5-25d5b6a3566e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:11.004841744Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1206,"client":"2","":"downstairs","session_id":"c18ac76c-4eb1-43f5-b7a5-25d5b6a3566e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:11.004867039Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1206,"client":"2","":"downstairs","session_id":"c18ac76c-4eb1-43f5-b7a5-25d5b6a3566e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:11.004883889Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1206,"":"io task","client":"2","":"downstairs","session_id":"c18ac76c-4eb1-43f5-b7a5-25d5b6a3566e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:11.004904751Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1206,"":"io task","client":"0","":"downstairs","session_id":"c18ac76c-4eb1-43f5-b7a5-25d5b6a3566e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:11.004929705Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1206,"":"io task","client":"1","":"downstairs","session_id":"c18ac76c-4eb1-43f5-b7a5-25d5b6a3566e"} {"msg":"ds_connection connected from Ok(127.0.0.1:55011)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:11.004974402Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1206,"":"io task","client":"2","":"downstairs","session_id":"c18ac76c-4eb1-43f5-b7a5-25d5b6a3566e"} {"msg":"ds_connection connected from Ok(127.0.0.1:37161)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:11.005000997Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1206,"":"io task","client":"1","":"downstairs","session_id":"c18ac76c-4eb1-43f5-b7a5-25d5b6a3566e"} {"msg":"ds_connection connected from Ok(127.0.0.1:58873)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:11.005025391Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1206,"":"io task","client":"0","":"downstairs","session_id":"c18ac76c-4eb1-43f5-b7a5-25d5b6a3566e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:11.005696213Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1206,"client":"1","":"downstairs","session_id":"c18ac76c-4eb1-43f5-b7a5-25d5b6a3566e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:11.00572429Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1206,"client":"2","":"downstairs","session_id":"c18ac76c-4eb1-43f5-b7a5-25d5b6a3566e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:11.005903024Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1206,"client":"0","":"downstairs","session_id":"c18ac76c-4eb1-43f5-b7a5-25d5b6a3566e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:11.006403397Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1206,"client":"1","":"downstairs","session_id":"c18ac76c-4eb1-43f5-b7a5-25d5b6a3566e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:11.006430042Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1206,"client":"0","":"downstairs","session_id":"c18ac76c-4eb1-43f5-b7a5-25d5b6a3566e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:11.006448733Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1206,"client":"2","":"downstairs","session_id":"c18ac76c-4eb1-43f5-b7a5-25d5b6a3566e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:11.006998685Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1206,"session_id":"c18ac76c-4eb1-43f5-b7a5-25d5b6a3566e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:11.007033325Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1206,"session_id":"c18ac76c-4eb1-43f5-b7a5-25d5b6a3566e"} {"msg":"[0]R flush_numbers[0..12]: [2, 2, 4, 4, 1, 4, 4, 1, 1, 2, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:11.007068415Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1206,"":"downstairs","":"downstairs","session_id":"c18ac76c-4eb1-43f5-b7a5-25d5b6a3566e"} {"msg":"[0]R generation[0..12]: [2, 2, 4, 4, 1, 4, 4, 1, 1, 2, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:11.007084215Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1206,"":"downstairs","":"downstairs","session_id":"c18ac76c-4eb1-43f5-b7a5-25d5b6a3566e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:11.007098433Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1206,"":"downstairs","":"downstairs","session_id":"c18ac76c-4eb1-43f5-b7a5-25d5b6a3566e"} {"msg":"[1]R flush_numbers[0..12]: [2, 2, 4, 4, 1, 4, 4, 1, 1, 2, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:11.007111731Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1206,"":"downstairs","":"downstairs","session_id":"c18ac76c-4eb1-43f5-b7a5-25d5b6a3566e"} {"msg":"[1]R generation[0..12]: [2, 2, 4, 4, 1, 4, 4, 1, 1, 2, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:11.007124818Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1206,"":"downstairs","":"downstairs","session_id":"c18ac76c-4eb1-43f5-b7a5-25d5b6a3566e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:11.007138857Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1206,"":"downstairs","":"downstairs","session_id":"c18ac76c-4eb1-43f5-b7a5-25d5b6a3566e"} {"msg":"[2]R flush_numbers[0..12]: [2, 2, 4, 4, 1, 4, 4, 1, 1, 2, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:11.007152255Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1206,"":"downstairs","":"downstairs","session_id":"c18ac76c-4eb1-43f5-b7a5-25d5b6a3566e"} {"msg":"[2]R generation[0..12]: [2, 2, 4, 4, 1, 4, 4, 1, 1, 2, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:11.007165362Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1206,"":"downstairs","":"downstairs","session_id":"c18ac76c-4eb1-43f5-b7a5-25d5b6a3566e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:11.00717874Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1206,"":"downstairs","":"downstairs","session_id":"c18ac76c-4eb1-43f5-b7a5-25d5b6a3566e"} {"msg":"Max found gen is 6","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:11.007191457Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1206,"":"downstairs","":"downstairs","session_id":"c18ac76c-4eb1-43f5-b7a5-25d5b6a3566e"} {"msg":"Generation requested: 6 >= found:6","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:11.007204325Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1206,"":"downstairs","":"downstairs","session_id":"c18ac76c-4eb1-43f5-b7a5-25d5b6a3566e"} {"msg":"Next flush: 7","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:11.007220014Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1206,"":"downstairs","":"downstairs","session_id":"c18ac76c-4eb1-43f5-b7a5-25d5b6a3566e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":Wait for a query_work_queue command to finish before sending IO "2025-01-17T21:11:11.007232911Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1206,"":"downstairs","":"downstairs","session_id":"c18ac76c-4eb1-43f5-b7a5-25d5b6a3566e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:11.007262209Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1206,"session_id":"c18ac76c-4eb1-43f5-b7a5-25d5b6a3566e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:11.007284652Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1206,"client":"0","":"downstairs","session_id":"c18ac76c-4eb1-43f5-b7a5-25d5b6a3566e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:11.007301632Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1206,"client":"1","":"downstairs","session_id":"c18ac76c-4eb1-43f5-b7a5-25d5b6a3566e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:11.007323615Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1206,"client":"2","":"downstairs","session_id":"c18ac76c-4eb1-43f5-b7a5-25d5b6a3566e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:11.007348309Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1206,"session_id":"c18ac76c-4eb1-43f5-b7a5-25d5b6a3566e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:11.007364138Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1206,"session_id":"c18ac76c-4eb1-43f5-b7a5-25d5b6a3566e"} {"msg":"7f905feb-e9ad-4668-879a-7a7f585d87c1 is now active with session: c18ac76c-4eb1-43f5-b7a5-25d5b6a3566e","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:11.007385811Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1206,"session_id":"c18ac76c-4eb1-43f5-b7a5-25d5b6a3566e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2025-01-17T21:11:11.007408034Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1206,"session_id":"c18ac76c-4eb1-43f5-b7a5-25d5b6a3566e"} {"msg":"The guest has finished waiting for activation with:6","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:11.007427916Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1206} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 403 len 16384 data: 3 2 2 3 02/30 Read block 418 len 20480 03/30 Read block 111 len 12288 04/30 Write block 596 len 12288 data: 2 2 2 05/30 Read block 594 len 16384 06/30 Write block 118 len 24576 data: 3 3 3 2 2 2 07/30 Write block 50 len 40960 data: 3 3 3 2 2 2 2 2 2 2 08/30 Write block 449 len 16384 data: 2 2 2 2 09/30 Write block 38 len 8192 data: 2 2 10/30 Write block 356 len 20480 data: 2 2 2 2 2 11/30 Read block 168 len 40960 12/30 Read block 370 len 32768 13/30 Write block 221 len 12288 data: 2 2 2 14/30 Read block 525 len 16384 15/30 Write block 265 len 24576 data: 2 3 3 3 3 3 16/30 Read block 506 len 36864 17/30 Write block 198 len 32768 data: 2 2 2 2 2 2 2 2 18/30 Read block 36 len 4096 19/30 Read block 510 len 16384 20/30 Read block 283 len 40960 21/30 Read block 257 len 16384 22/30 Write block 312 len 40960 data: 2 2 2 2 2 2 2 2 2 2 23/30 Flush 24/30 Flush 25/30 Write block 4 len 4096 data: 3 26/30 Write block 384 len 40960 data: 2 2 2 2 2 2 2 2 2 2 27/30 Write block 90 len 8192 data: 2 2 28/30 Write block 421 len 4096 data: 2 29/30 Read block 8 len 40960 30/30 Write block 510 len 28672 data: 3 2 2 2 2 2 2 ---------------------------------------------------------------- Crucible gen:6 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 1 Done Done Done false 1025 Acked Write 10 Done Done Done false 1026 Acked Write 2 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Read 10 Done Done Done false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:11:11.026914975Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1206} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 2 2 2 2 2 2 T T T 1 020-039 6 6 6 7 7 7 F F F 2 040-059 6 6 6 7 7 7 F F F 3 060-079 4 4 4 4 4 4 F F F 4 080-099 1 1 1 1 1 1 T T T 5 100-119 6 6 6 7 7 7 F F F 6 120-139 6 6 6 7 7 7 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 1 1 1 1 1 1 F F F 9 180-199 6 6 6 7 7 7 F F F 10 200-219 6 6 6 7 7 7 F F F 11 220-239 6 6 6 7 7 7 F F F 12 240-259 1 1 1 1 1 1 F F F 13 260-279 6 6 6 7 7 7 F F F 14 280-299 1 1 1 1 1 1 F F F 15 300-319 6 6 6 7 7 7 F F F 16 320-339 6 6 6 7 7 7 F F F 17 340-359 6 6 6 7 7 7 F F F 18 360-379 6 6 6 7 7 7 F F F 19 380-399 5 5 5 6 6 6 T T T 20 400-419 6 6 6 7 7 7 F F F 21 420-439 4 4 4 4 4 4 T T T 22 440-459 6 6 6 7 7 7 F F F 23 460-479 2 2 2 2 2 2 F F F 24 480-499 3 3 3 3 3 3 F F F 25 500-519 3 3 3 3 3 3 T T T 26 520-539 2 2 2 2 2 2 F F F 27 540-559 4 4 4 5 5 5 F F F 28 560-579 4 4 4 5 5 5 F F F 29 580-599 6 6 6 7 7 7 F F F Max gen: 6, Max flush: 7 A Difference in extent metadata is expected here On loop 3 Verifying data now No change after verify Loop: 3 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 7 7 7 8 8 8 F F F 1 020-039 6 6 6 7 7 7 F F F 2 040-059 6 6 6 7 7 7 F F F 3 060-079 4 4 4 4 4 4 F F F 4 080-099 7 7 7 8 8 8 F F F 5 100-119 6 6 6 7 7 7 F F F 6 120-139 6 6 6 7 7 7 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 1 1 1 1 1 1 F F F 9 180-199 6 6 6 7 7 7 F F F 10 200-219 6 6 6 7 7 7 F F F 11 220-239 6 6 6 7 7 7 F F F 12 240-259 1 1 1 1 1 1 F F F 13 260-279 6 6 6 7 7 7 F F F 14 280-299 1 1 1 1 1 1 F F F 15 300-319 6 6 6 7 7 7 F F F 16 320-339 6 6 6 7 7 7 F F F 17 340-359 6 6 6 7 7 7 F F F 18 360-379 6 6 6 7 7 7 F F F 19 380-399 7 7 7 8 8 8 F F F 20 400-419 6 6 6 7 7 7 F F F 21 420-439 7 7 7 8 8 8 F F F 22 440-459 6 6 6 7 7 7 F F F 23 460-479 2 2 2 2 2 2 F F F 24 480-499 3 3 3 3 3 3 F F F 25 500-519 7 7 7 8 8 8 F F F 26 520-539 2 2 2 2 2 2 F F F 27 540-559 4 4 4 5 5 5 F F F 28 560-579 4 4 4 5 5 5 F F F 29 580-599 6 6 6 7 7 7 F F F Max gen: 7, Max flush: 8 Begin loop 4 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:15.298142465Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1214,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:11:15.34196846Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1214,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:15.342398341Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1214,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:15.374678448Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1214,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:15.375327507Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1214} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"abd5430f1508af0bc008143ea2566277ef714ec2\",\n git_commit_timestamp: \"2025-01-17T20:55:03.000000000Z\",\n git_branch: \"alan/tokio-to-140\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:15.375368711Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1214} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:15.37538455Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1214} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:15.375402111Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1214} {"msg":"Crucible c1502bca-8c1f-460c-9dfa-63d3e7159417 has session id: 878de2fe-4244-4af1-ade0-1d79c73b3f8c","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:15.376022723Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1214,"session_id":"878de2fe-4244-4af1-ade0-1d79c73b3f8c"} {"msg":"Upstairs opts: Upstairs UUID: c1502bca-8c1f-460c-9dfa-63d3e7159417, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:15.376061326Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1214,"session_id":"878de2fe-4244-4af1-ade0-1d79c73b3f8c"} {"msg":"Crucible stats registered with UUID: c1502bca-8c1f-460c-9dfa-63d3e7159417","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:15.376084359Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1214,"session_id":"878de2fe-4244-4af1-ade0-1d79c73b3f8c"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:11:15.376108994Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1214,"session_id":"878de2fe-4244-4af1-ade0-1d79c73b3f8c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:15.376583982Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1214,"task":"crutest"} {"msg":"The guest has requested activation with gen:8","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:15.376618262Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1214} {"msg":"c1502bca-8c1f-460c-9dfa-63d3e7159417 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:15.376692495Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1214,"session_id":"878de2fe-4244-4af1-ade0-1d79c73b3f8c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:15.37673551Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1214,"client":"0","":"downstairs","session_id":"878de2fe-4244-4af1-ade0-1d79c73b3f8c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:15.376777785Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1214,"client":"0","":"downstairs","session_id":"878de2fe-4244-4af1-ade0-1d79c73b3f8c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:15.376867988Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1214,"client":"1","":"downstairs","session_id":"878de2fe-4244-4af1-ade0-1d79c73b3f8c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:15.376895974Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1214,"client":"1","":"downstairs","session_id":"878de2fe-4244-4af1-ade0-1d79c73b3f8c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:15.376911003Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1214,"client":"2","":"downstairs","session_id":"878de2fe-4244-4af1-ade0-1d79c73b3f8c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:15.376925061Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1214,"client":"2","":"downstairs","session_id":"878de2fe-4244-4af1-ade0-1d79c73b3f8c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:15.376938178Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1214,"":"io task","client":"2","":"downstairs","session_id":"878de2fe-4244-4af1-ade0-1d79c73b3f8c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:15.376952037Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1214,"":"io task","client":"0","":"downstairs","session_id":"878de2fe-4244-4af1-ade0-1d79c73b3f8c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:15.376965324Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1214,"":"io task","client":"1","":"downstairs","session_id":"878de2fe-4244-4af1-ade0-1d79c73b3f8c"} {"msg":"ds_connection connected from Ok(127.0.0.1:40538)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:15.376978672Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1214,"":"io task","client":"2","":"downstairs","session_id":"878de2fe-4244-4af1-ade0-1d79c73b3f8c"} {"msg":"ds_connection connected from Ok(127.0.0.1:42657)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:15.377150222Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1214,"":"io task","client":"1","":"downstairs","session_id":"878de2fe-4244-4af1-ade0-1d79c73b3f8c"} {"msg":"ds_connection connected from Ok(127.0.0.1:44793)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:15.377184032Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1214,"":"io task","client":"0","":"downstairs","session_id":"878de2fe-4244-4af1-ade0-1d79c73b3f8c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:15.377774547Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1214,"client":"1","":"downstairs","session_id":"878de2fe-4244-4af1-ade0-1d79c73b3f8c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:15.377811069Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1214,"client":"2","":"downstairs","session_id":"878de2fe-4244-4af1-ade0-1d79c73b3f8c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:15.377827909Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1214,"client":"0","":"downstairs","session_id":"878de2fe-4244-4af1-ade0-1d79c73b3f8c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:15.378356197Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1214,"client":"1","":"downstairs","session_id":"878de2fe-4244-4af1-ade0-1d79c73b3f8c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:15.378494648Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1214,"client":"0","":"downstairs","session_id":"878de2fe-4244-4af1-ade0-1d79c73b3f8c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:15.378596227Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1214,"client":"2","":"downstairs","session_id":"878de2fe-4244-4af1-ade0-1d79c73b3f8c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:15.378838239Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1214,"session_id":"878de2fe-4244-4af1-ade0-1d79c73b3f8c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:15.379008979Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1214,"session_id":"878de2fe-4244-4af1-ade0-1d79c73b3f8c"} {"msg":"[0]R flush_numbers[0..12]: [8, 7, 7, 4, 8, 7, 7, 1, 1, 7, 7, 7]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:15.379151702Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1214,"":"downstairs","":"downstairs","session_id":"878de2fe-4244-4af1-ade0-1d79c73b3f8c"} {"msg":"[0]R generation[0..12]: [7, 6, 6, 4, 7, 6, 6, 1, 1, 6, 6, 6]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:15.379179599Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1214,"":"downstairs","":"downstairs","session_id":"878de2fe-4244-4af1-ade0-1d79c73b3f8c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:15.379195488Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1214,"":"downstairs","":"downstairs","session_id":"878de2fe-4244-4af1-ade0-1d79c73b3f8c"} {"msg":"[1]R flush_numbers[0..12]: [8, 7, 7, 4, 8, 7, 7, 1, 1, 7, 7, 7]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:15.379209616Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1214,"":"downstairs","":"downstairs","session_id":"878de2fe-4244-4af1-ade0-1d79c73b3f8c"} {"msg":"[1]R generation[0..12]: [7, 6, 6, 4, 7, 6, 6, 1, 1, 6, 6, 6]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:15.379222874Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1214,"":"downstairs","":"downstairs","session_id":"878de2fe-4244-4af1-ade0-1d79c73b3f8c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:15.379236552Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1214,"":"downstairs","":"downstairs","session_id":"878de2fe-4244-4af1-ade0-1d79c73b3f8c"} {"msg":"[2]R flush_numbers[0..12]: [8, 7, 7, 4, 8, 7, 7, 1, 1, 7, 7, 7]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:15.379253022Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1214,"":"downstairs","":"downstairs","session_id":"878de2fe-4244-4af1-ade0-1d79c73b3f8c"} {"msg":"[2]R generation[0..12]: [7, 6, 6, 4, 7, 6, 6, 1, 1, 6, 6, 6]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:15.379266439Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1214,"":"downstairs","":"downstairs","session_id":"878de2fe-4244-4af1-ade0-1d79c73b3f8c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:15.379279887Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1214,"":"downstairs","":"downstairs","session_id":"878de2fe-4244-4af1-ade0-1d79c73b3f8c"} {"msg":"Max found gen is 8","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:15.379292394Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1214,"":"downstairs","":"downstairs","session_id":"878de2fe-4244-4af1-ade0-1d79c73b3f8c"} {"msg":"Generation requested: 8 >= found:8","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:15.379305212Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1214,"":"downstairs","":"downstairs","session_id":"878de2fe-4244-4af1-ade0-1d79c73b3f8c"} {"msg":"Next flush: 9","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-17T21:11:15.379317539Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1214,"":"downstairs","":"downstairs","session_id":"878de2fe-4244-4af1-ade0-1d79c73b3f8c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:15.379333618Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1214,"":"downstairs","":"downstairs","session_id":"878de2fe-4244-4af1-ade0-1d79c73b3f8c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:15.379346456Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1214,"session_id":"878de2fe-4244-4af1-ade0-1d79c73b3f8c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:15.379358213Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1214,"client":"0","":"downstairs","session_id":"878de2fe-4244-4af1-ade0-1d79c73b3f8c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:15.379371111Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1214,"client":"1","":"downstairs","session_id":"878de2fe-4244-4af1-ade0-1d79c73b3f8c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:15.379383948Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1214,"client":"2","":"downstairs","session_id":"878de2fe-4244-4af1-ade0-1d79c73b3f8c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:15.379396846Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1214,"session_id":"878de2fe-4244-4af1-ade0-1d79c73b3f8c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:15.379408662Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1214,"session_id":"878de2fe-4244-4af1-ade0-1d79c73b3f8c"} {"msg":"c1502bca-8c1f-460c-9dfa-63d3e7159417 is now active with session: 878de2fe-4244-4af1-ade0-1d79c73b3f8c","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:15.379421Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1214,"session_id":"878de2fe-4244-4af1-ade0-1d79c73b3f8c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:15.379432707Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1214,"session_id":"878de2fe-4244-4af1-ade0-1d79c73b3f8c"} {"msg":"The guest has finished waiting for activation with:8","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:15.379446094Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1214} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 32 len 24576 data: 2 2 2 2 2 2 02/30 Write block 560 len 4096 data: 3 03/30 Read block 465 len 4096 04/30 Read block 480 len 16384 05/30 Read block 573 len 8192 06/30 Read block 176 len 36864 07/30 Read block 234 len 40960 08/30 Read block 480 len 8192 09/30 Write block 467 len 24576 data: 2 2 2 2 2 2 10/30 Flush 11/30 Write block 190 len 20480 data: 3 3 3 3 3 12/30 Flush 13/30 Write block 172 len 20480 data: 2 2 2 2 2 14/30 Read block 490 len 4096 15/30 Write block 207 len 32768 data: 2 2 2 2 2 2 2 2 16/30 Read block 330 len 4096 17/30 Flush 18/30 Write block 139 len 16384 data: 2 2 2 2 19/30 Read block 283 len 20480 20/30 Read block 223 len 24576 21/30 Read block 458 len 32768 22/30 Read block 466 len 28672 23/30 Read block 300 len 24576 24/30 Write block 504 len 4096 data: 3 25/30 Read block 275 len 4096 26/30 Read block 256 len 36864 27/30 Read block 555 len 8192 28/30 Read block 222 len 12288 29/30 Write block 303 len 36864 data: 2 2 2 2 2 2 2 2 2 30/30 Read block 10 len 36864 ---------------------------------------------------------------- Crucible gen:8 GIO:true work queues: Upstairs:0 downstairs:13 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1017 Acked Write 4 Done Done Done false 1018 Acked Read 5 Done Done Done false 1019 Acked Read 6 Done Done Done false 1020 Acked Read 8 Done Done Done false 1021 Acked Read 7 Done Done Done false 1022 Acked Read 6 Done Done Done false 1023 Acked Write 1 Done Done Done false 1024 Acked Read 1 Done Done Done false 1025 Acked Read 9 Done Done Done false 1026 Acked Read 2 Done Done Done false 1027 Acked Read 3 Done Done Done false 1028 Acked Write 9 Done Done Sent false 1029 Acked Read 9 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 2 3 Done 12 13 11 36 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1016 1016 1016 Downstairs last five completed: 1016 1015 1014 1013 1012 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:11:15.398899354Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1214} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 7 7 7 8 8 8 F F F 1 020-039 8 8 8 9 9 9 F F F 2 040-059 6 6 6 7 7 7 F F F 3 060-079 4 4 4 4 4 4 F F F 4 080-099 7 7 7 8 8 8 F F F 5 100-119 6 6 6 7 7 7 F F F 6 120-139 6 6 6 7 7 7 T T T 7 140-159 1 1 1 1 1 1 T T T 8 160-179 8 8 8 11 11 11 F F F 9 180-199 8 8 8 10 10 10 F F F 10 200-219 8 8 8 11 11 11 F F F 11 220-239 6 6 6 7 7 7 F F F 12 240-259 1 1 1 1 1 1 F F F 13 260-279 6 6 6 7 7 7 F F F 14 280-299 1 1 1 1 1 1 F F F 15 300-319 6 6 6 7 7 7 T T T 16 320-339 6 6 6 7 7 7 F F F 17 340-359 6 6 6 7 7 7 F F F 18 360-379 6 6 6 7 7 7 F F F 19 380-399 7 7 7 8 8 8 F F F 20 400-419 6 6 6 7 7 7 F F F 21 420-439 7 7 7 8 8 8 F F F 22 440-459 6 6 6 7 7 7 F F F 23 460-479 8 8 8 9 9 9 F F F 24 480-499 3 3 3 3 3 3 F F F 25 500-519 7 7 7 8 8 8 T T T 26 520-539 2 2 2 2 2 2 F F F 27 540-559 4 4 4 5 5 5 F F F 28 560-579 8 8 8 9 9 9 F F F 29 580-599 6 6 6 7 7 7 F F F Max gen: 8, Max flush: 11 A Difference in extent metadata is expected here On loop 4 Verifying data now No change after verify Loop: 4 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 7 7 7 8 8 8 F F F 1 020-039 8 8 8 9 9 9 F F F 2 040-059 6 6 6 7 7 7 F F F 3 060-079 4 4 4 4 4 4 F F F 4 080-099 7 7 7 8 8 8 F F F 5 100-119 6 6 6 7 7 7 F F F 6 120-139 9 9 9 12 12 12 F F F 7 140-159 9 9 9 12 12 12 F F F 8 160-179 8 8 8 11 11 11 F F F 9 180-199 8 8 8 10 10 10 F F F 10 200-219 8 8 8 11 11 11 F F F 11 220-239 6 6 6 7 7 7 F F F 12 240-259 1 1 1 1 1 1 F F F 13 260-279 6 6 6 7 7 7 F F F 14 280-299 1 1 1 1 1 1 F F F 15 300-319 9 9 9 12 12 12 F F F 16 320-339 6 6 6 7 7 7 F F F 17 340-359 6 6 6 7 7 7 F F F 18 360-379 6 6 6 7 7 7 F F F 19 380-399 7 7 7 8 8 8 F F F 20 400-419 6 6 6 7 7 7 F F F 21 420-439 7 7 7 8 8 8 F F F 22 440-459 6 6 6 7 7 7 F F F 23 460-479 8 8 8 9 9 9 F F F 24 480-499 3 3 3 3 3 3 F F F 25 500-519 9 9 9 12 12 12 F F F 26 520-539 2 2 2 2 2 2 F F F 27 540-559 4 4 4 5 5 5 F F F 28 560-579 8 8 8 9 9 9 F F F 29 580-599 6 6 6 7 7 7 F F F Max gen: 9, Max flush: 12 Begin loop 5 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:19.665872928Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1222,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:11:19.708501323Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1222,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:19.708913083Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1222,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:19.74061152Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1222,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:19.741224858Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1222} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"abd5430f1508af0bc008143ea2566277ef714ec2\",\n git_commit_timestamp: \"2025-01-17T20:55:03.000000000Z\",\n git_branch: \"alan/tokio-to-140\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:19.741258488Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1222} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:19.741275658Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1222} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:19.741290207Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1222} {"msg":"Crucible 26179e22-a611-4d85-8ff4-4b14e97f2738 has session id: 7505967b-6060-4136-a156-61f0e8ecccce","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:19.741961939Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1222,"session_id":"7505967b-6060-4136-a156-61f0e8ecccce"} {"msg":"Upstairs opts: Upstairs UUID: 26179e22-a611-4d85-8ff4-4b14e97f2738, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:19.742008757Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1222,"session_id":"7505967b-6060-4136-a156-61f0e8ecccce"} {"msg":"Crucible stats registered with UUID: 26179e22-a611-4d85-8ff4-4b14e97f2738","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:19.742037073Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1222,"session_id":"7505967b-6060-4136-a156-61f0e8ecccce"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:11:19.742063348Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1222,"session_id":"7505967b-6060-4136-a156-61f0e8ecccce"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:19.74249287Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1222,"task":"crutest"} {"msg":"The guest has requested activation with gen:10","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:19.742524528Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1222} {"msg":"26179e22-a611-4d85-8ff4-4b14e97f2738 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:19.742587875Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1222,"session_id":"7505967b-6060-4136-a156-61f0e8ecccce"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:19.742618103Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1222,"client":"0","":"downstairs","session_id":"7505967b-6060-4136-a156-61f0e8ecccce"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:19.742641637Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1222,"client":"0","":"downstairs","session_id":"7505967b-6060-4136-a156-61f0e8ecccce"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:19.742655345Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1222,"client":"1","":"downstairs","session_id":"7505967b-6060-4136-a156-61f0e8ecccce"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:19.742669533Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1222,"client":"1","":"downstairs","session_id":"7505967b-6060-4136-a156-61f0e8ecccce"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:19.74268238Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1222,"client":"2","":"downstairs","session_id":"7505967b-6060-4136-a156-61f0e8ecccce"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:19.742696118Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1222,"client":"2","":"downstairs","session_id":"7505967b-6060-4136-a156-61f0e8ecccce"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:19.742712188Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1222,"":"io task","client":"2","":"downstairs","session_id":"7505967b-6060-4136-a156-61f0e8ecccce"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:19.742735151Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1222,"":"io task","client":"0","":"downstairs","session_id":"7505967b-6060-4136-a156-61f0e8ecccce"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:19.742760656Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1222,"":"io task","client":"1","":"downstairs","session_id":"7505967b-6060-4136-a156-61f0e8ecccce"} {"msg":"ds_connection connected from Ok(127.0.0.1:63718)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:19.742786361Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1222,"":"io task","client":"0","":"downstairs","session_id":"7505967b-6060-4136-a156-61f0e8ecccce"} {"msg":"ds_connection connected from Ok(127.0.0.1:34284)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:19.742814638Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1222,"":"io task","client":"2","":"downstairs","session_id":"7505967b-6060-4136-a156-61f0e8ecccce"} {"msg":"ds_connection connected from Ok(127.0.0.1:52519)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:19.742842204Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1222,"":"io task","client":"1","":"downstairs","session_id":"7505967b-6060-4136-a156-61f0e8ecccce"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:19.743609142Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1222,"client":"2","":"downstairs","session_id":"7505967b-6060-4136-a156-61f0e8ecccce"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:19.743649255Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1222,"client":"0","":"downstairs","session_id":"7505967b-6060-4136-a156-61f0e8ecccce"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:19.743679223Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1222,"client":"1","":"downstairs","session_id":"7505967b-6060-4136-a156-61f0e8ecccce"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:19.744287138Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1222,"client":"2","":"downstairs","session_id":"7505967b-6060-4136-a156-61f0e8ecccce"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:19.744360791Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1222,"client":"1","":"downstairs","session_id":"7505967b-6060-4136-a156-61f0e8ecccce"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:19.744430442Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1222,"client":"0","":"downstairs","session_id":"7505967b-6060-4136-a156-61f0e8ecccce"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:19.744921469Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1222,"session_id":"7505967b-6060-4136-a156-61f0e8ecccce"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:19.744955079Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1222,"session_id":"7505967b-6060-4136-a156-61f0e8ecccce"} {"msg":"[0]R flush_numbers[0..12]: [8, 9, 7, 4, 8, 7, 12, 12, 11, 10, 11, 7]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:19.745001926Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1222,"":"downstairs","":"downstairs","session_id":"7505967b-6060-4136-a156-61f0e8ecccce"} {"msg":"[0]R generation[0..12]: [7, 8, 6, 4, 7, 6, 9, 9, 8, 8, 8, 6]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:19.745020157Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1222,"":"downstairs","":"downstairs","session_id":"7505967b-6060-4136-a156-61f0e8ecccce"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:19.745034705Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1222,"":"downstairs","":"downstairs","session_id":"7505967b-6060-4136-a156-61f0e8ecccce"} {"msg":"[1]R flush_numbers[0..12]: [8, 9, 7, 4, 8, 7, 12, 12, 11, 10, 11, 7]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:19.745048343Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1222,"":"downstairs","":"downstairs","session_id":"7505967b-6060-4136-a156-61f0e8ecccce"} {"msg":"[1]R generation[0..12]: [7, 8, 6, 4, 7, 6, 9, 9, 8, 8, 8, 6]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:19.745061581Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1222,"":"downstairs","":"downstairs","session_id":"7505967b-6060-4136-a156-61f0e8ecccce"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:19.745075099Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1222,"":"downstairs","":"downstairs","session_id":"7505967b-6060-4136-a156-61f0e8ecccce"} {"msg":"[2]R flush_numbers[0..12]: [8, 9, 7, 4, 8, 7, 12, 12, 11, 10, 11, 7]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:19.745088687Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1222,"":"downstairs","":"downstairs","session_id":"7505967b-6060-4136-a156-61f0e8ecccce"} {"msg":"[2]R generation[0..12]: [7, 8, 6, 4, 7, 6, 9, 9, 8, 8, 8, 6]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:19.745101984Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1222,"":"downstairs","":"downstairs","session_id":"7505967b-6060-4136-a156-61f0e8ecccce"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:19.745115652Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1222,"":"downstairs","":"downstairs","session_id":"7505967b-6060-4136-a156-61f0e8ecccce"} {"msg":"Max found gen is 10","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:19.74512831Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1222,"":"downstairs","":"downstairs","session_id":"7505967b-6060-4136-a156-61f0e8ecccce"} {"msg":"Generation requested: 10 >= found:10","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:19.745141077Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1222,"":"downstairs","":"downstairs","session_id":"7505967b-6060-4136-a156-61f0e8ecccce"} {"msg":"Next flush: 13","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:19.745153485Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1222,"":"downstairs","":"downstairs","session_id":"7505967b-6060-4136-a156-61f0e8ecccce"}Wait for a query_work_queue command to finish before sending IO {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:19.745172456Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1222,"":"downstairs","":"downstairs","session_id":"7505967b-6060-4136-a156-61f0e8ecccce"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:19.745191127Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1222,"session_id":"7505967b-6060-4136-a156-61f0e8ecccce"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:19.745215471Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1222,"client":"0","":"downstairs","session_id":"7505967b-6060-4136-a156-61f0e8ecccce"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:19.74523121Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1222,"client":"1","":"downstairs","session_id":"7505967b-6060-4136-a156-61f0e8ecccce"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:19.745244118Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1222,"client":"2","":"downstairs","session_id":"7505967b-6060-4136-a156-61f0e8ecccce"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:19.745257235Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1222,"session_id":"7505967b-6060-4136-a156-61f0e8ecccce"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:19.745269963Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1222,"session_id":"7505967b-6060-4136-a156-61f0e8ecccce"} {"msg":"26179e22-a611-4d85-8ff4-4b14e97f2738 is now active with session: 7505967b-6060-4136-a156-61f0e8ecccce","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:19.745291465Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1222,"session_id":"7505967b-6060-4136-a156-61f0e8ecccce"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:19.745310356Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1222,"session_id":"7505967b-6060-4136-a156-61f0e8ecccce"} {"msg":"The guest has finished waiting for activation with:10","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:19.745322744Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1222} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 180 len 8192 data: 2 2 02/30 Read block 581 len 16384 03/30 Flush 04/30 Write block 10 len 8192 data: 2 2 05/30 Read block 83 len 24576 06/30 Write block 335 len 40960 data: 2 2 2 2 2 2 2 2 2 2 07/30 Read block 8 len 12288 08/30 Read block 49 len 12288 09/30 Write block 241 len 40960 data: 2 2 2 2 2 2 2 2 2 2 10/30 Write block 65 len 28672 data: 2 2 2 2 2 2 2 11/30 Write block 26 len 28672 data: 2 2 2 2 2 2 3 12/30 Read block 182 len 8192 13/30 Write block 343 len 12288 data: 3 3 2 14/30 Flush 15/30 Write block 177 len 32768 data: 2 2 2 3 3 2 2 2 16/30 Read block 209 len 28672 17/30 Write block 546 len 28672 data: 2 2 2 2 2 2 2 18/30 Read block 310 len 40960 19/30 Read block 394 len 12288 20/30 Read block 345 len 12288 21/30 Read block 297 len 40960 22/30 Write block 404 len 8192 data: 3 3 23/30 Flush 24/30 Write block 136 len 40960 data: 2 2 2 3 3 3 3 2 2 2 25/30 Read block 360 len 36864 26/30 Read block 130 len 32768 27/30 Write block 242 len 28672 data: 3 3 3 3 3 3 3 28/30 Write block 431 len 20480 data: 3 3 2 2 2 29/30 Read block 590 len 36864 30/30 Read block 568 len 24576 ---------------------------------------------------------------- Crucible gen:10 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 10 Done Done Done false 1024 Acked Read 9 Done Done Done false 1025 Acked Read 8 Done Done Done false 1026 Acked Write 7 Done Done Done false 1027 Acked Write 5 Done Done Done false 1028 Acked Read 9 Done Done Done false 1029 Acked Read 6 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 0 1 Done 6 7 7 20 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:11:19.766133117Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1222} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 10 10 10 14 14 14 F F F 1 020-039 10 10 10 14 14 14 F F F 2 040-059 6 6 6 7 7 7 F F F 3 060-079 10 10 10 14 14 14 F F F 4 080-099 7 7 7 8 8 8 F F F 5 100-119 6 6 6 7 7 7 F F F 6 120-139 9 9 9 12 12 12 T T T 7 140-159 9 9 9 12 12 12 T T T 8 160-179 10 10 10 15 15 15 F F F 9 180-199 10 10 10 15 15 15 F F F 10 200-219 8 8 8 11 11 11 F F F 11 220-239 6 6 6 7 7 7 F F F 12 240-259 10 10 10 14 14 14 T T T 13 260-279 6 6 6 7 7 7 F F F 14 280-299 1 1 1 1 1 1 F F F 15 300-319 9 9 9 12 12 12 F F F 16 320-339 10 10 10 14 14 14 F F F 17 340-359 10 10 10 14 14 14 F F F 18 360-379 6 6 6 7 7 7 F F F 19 380-399 7 7 7 8 8 8 F F F 20 400-419 10 10 10 15 15 15 F F F 21 420-439 7 7 7 8 8 8 T T T 22 440-459 6 6 6 7 7 7 F F F 23 460-479 8 8 8 9 9 9 F F F 24 480-499 3 3 3 3 3 3 F F F 25 500-519 9 9 9 12 12 12 F F F 26 520-539 2 2 2 2 2 2 F F F 27 540-559 10 10 10 15 15 15 F F F 28 560-579 8 8 8 9 9 9 F F F 29 580-599 6 6 6 7 7 7 F F F Max gen: 10, Max flush: 15 A Difference in extent metadata is expected here On loop 5 Verifying data now No change after verify Loop: 5 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 10 10 10 14 14 14 F F F 1 020-039 10 10 10 14 14 14 F F F 2 040-059 6 6 6 7 7 7 F F F 3 060-079 10 10 10 14 14 14 F F F 4 080-099 7 7 7 8 8 8 F F F 5 100-119 6 6 6 7 7 7 F F F 6 120-139 11 11 11 16 16 16 F F F 7 140-159 11 11 11 16 16 16 F F F 8 160-179 10 10 10 15 15 15 F F F 9 180-199 10 10 10 15 15 15 F F F 10 200-219 8 8 8 11 11 11 F F F 11 220-239 6 6 6 7 7 7 F F F 12 240-259 11 11 11 16 16 16 F F F 13 260-279 6 6 6 7 7 7 F F F 14 280-299 1 1 1 1 1 1 F F F 15 300-319 9 9 9 12 12 12 F F F 16 320-339 10 10 10 14 14 14 F F F 17 340-359 10 10 10 14 14 14 F F F 18 360-379 6 6 6 7 7 7 F F F 19 380-399 7 7 7 8 8 8 F F F 20 400-419 10 10 10 15 15 15 F F F 21 420-439 11 11 11 16 16 16 F F F 22 440-459 6 6 6 7 7 7 F F F 23 460-479 8 8 8 9 9 9 F F F 24 480-499 3 3 3 3 3 3 F F F 25 500-519 9 9 9 12 12 12 F F F 26 520-539 2 2 2 2 2 2 F F F 27 540-559 10 10 10 15 15 15 F F F 28 560-579 8 8 8 9 9 9 F F F 29 580-599 6 6 6 7 7 7 F F F Max gen: 11, Max flush: 16 Begin loop 6 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:24.033409706Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1230,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:24.077239264Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1230,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:24.077881689Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1230} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"abd5430f1508af0bc008143ea2566277ef714ec2\",\n git_commit_timestamp: \"2025-01-17T20:55:03.000000000Z\",\n git_branch: \"alan/tokio-to-140\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:24.077921282Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1230} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:24.077935911Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1230} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:24.077949909Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1230} {"msg":"Crucible 21392b3b-570e-4eed-9f99-3e75fa703e38 has session id: e5de9631-8a44-45ed-9285-77579a6d2566","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:24.078388996Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1230,"session_id":"e5de9631-8a44-45ed-9285-77579a6d2566"} {"msg":"Upstairs opts: Upstairs UUID: 21392b3b-570e-4eed-9f99-3e75fa703e38, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:24.078439655Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1230,"session_id":"e5de9631-8a44-45ed-9285-77579a6d2566"} {"msg":"Crucible stats registered with UUID: 21392b3b-570e-4eed-9f99-3e75fa703e38","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:24.078457516Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1230,"session_id":"e5de9631-8a44-45ed-9285-77579a6d2566"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:11:24.078470443Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1230,"session_id":"e5de9631-8a44-45ed-9285-77579a6d2566"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:24.078989516Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1230,"task":"crutest"} {"msg":"The guest has requested activation with gen:12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:24.079043028Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1230} {"msg":"21392b3b-570e-4eed-9f99-3e75fa703e38 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:24.079101442Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1230,"session_id":"e5de9631-8a44-45ed-9285-77579a6d2566"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:24.079119092Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1230,"client":"0","":"downstairs","session_id":"e5de9631-8a44-45ed-9285-77579a6d2566"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:24.079143036Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1230,"client":"0","":"downstairs","session_id":"e5de9631-8a44-45ed-9285-77579a6d2566"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:24.079160997Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1230,"client":"1","":"downstairs","session_id":"e5de9631-8a44-45ed-9285-77579a6d2566"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:24.079175115Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1230,"client":"1","":"downstairs","session_id":"e5de9631-8a44-45ed-9285-77579a6d2566"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:24.079188072Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1230,"client":"2","":"downstairs","session_id":"e5de9631-8a44-45ed-9285-77579a6d2566"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:24.079202001Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1230,"client":"2","":"downstairs","session_id":"e5de9631-8a44-45ed-9285-77579a6d2566"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:24.079214758Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1230,"":"io task","client":"2","":"downstairs","session_id":"e5de9631-8a44-45ed-9285-77579a6d2566"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:24.07923553Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1230,"":"io task","client":"0","":"downstairs","session_id":"e5de9631-8a44-45ed-9285-77579a6d2566"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:24.079260885Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1230,"":"io task","client":"1","":"downstairs","session_id":"e5de9631-8a44-45ed-9285-77579a6d2566"} {"msg":"ds_connection connected from Ok(127.0.0.1:57544)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:24.079295575Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1230,"":"io task","client":"2","":"downstairs","session_id":"e5de9631-8a44-45ed-9285-77579a6d2566"} {"msg":"ds_connection connected from Ok(127.0.0.1:51553)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:24.079313736Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1230,"":"io task","client":"0","":"downstairs","session_id":"e5de9631-8a44-45ed-9285-77579a6d2566"} {"msg":"ds_connection connected from Ok(127.0.0.1:48140)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:24.079381196Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1230,"":"io task","client":"1","":"downstairs","session_id":"e5de9631-8a44-45ed-9285-77579a6d2566"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:24.079977384Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1230,"client":"0","":"downstairs","session_id":"e5de9631-8a44-45ed-9285-77579a6d2566"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:24.080006201Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1230,"client":"1","":"downstairs","session_id":"e5de9631-8a44-45ed-9285-77579a6d2566"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:24.080661144Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1230,"client":"1","":"downstairs","session_id":"e5de9631-8a44-45ed-9285-77579a6d2566"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:24.080700857Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1230,"client":"0","":"downstairs","session_id":"e5de9631-8a44-45ed-9285-77579a6d2566"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:24.081256341Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1230,"session_id":"e5de9631-8a44-45ed-9285-77579a6d2566"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:24.081282016Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1230,"session_id":"e5de9631-8a44-45ed-9285-77579a6d2566"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:24.081434465Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1230,"client":"2","":"downstairs","session_id":"e5de9631-8a44-45ed-9285-77579a6d2566"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:24.081975732Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1230,"client":"2","":"downstairs","session_id":"e5de9631-8a44-45ed-9285-77579a6d2566"} {"msg":"[0]R flush_numbers[0..12]: [14, 14, 7, 14, 8, 7, 16, 16, 15, 15, 11, 7]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:24.082458524Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1230,"":"downstairs","":"downstairs","session_id":"e5de9631-8a44-45ed-9285-77579a6d2566"} {"msg":"[0]R generation[0..12]: [10, 10, 6, 10, 7, 6, 11, 11, 10, 10, 8, 6]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:24.082481838Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1230,"":"downstairs","":"downstairs","session_id":"e5de9631-8a44-45ed-9285-77579a6d2566"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:24.082497037Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1230,"":"downstairs","":"downstairs","session_id":"e5de9631-8a44-45ed-9285-77579a6d2566"} {"msg":"[1]R flush_numbers[0..12]: [14, 14, 7, 14, 8, 7, 16, 16, 15, 15, 11, 7]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:24.082510794Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1230,"":"downstairs","":"downstairs","session_id":"e5de9631-8a44-45ed-9285-77579a6d2566"} {"msg":"[1]R generation[0..12]: [10, 10, 6, 10, 7, 6, 11, 11, 10, 10, 8, 6]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:24.082524162Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1230,"":"downstairs","":"downstairs","session_id":"e5de9631-8a44-45ed-9285-77579a6d2566"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:24.08253803Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1230,"":"downstairs","":"downstairs","session_id":"e5de9631-8a44-45ed-9285-77579a6d2566"} {"msg":"[2]R flush_numbers[0..12]: [14, 14, 7, 14, 8, 7, 16, 16, 15, 15, 11, 7]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:24.082551669Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1230,"":"downstairs","":"downstairs","session_id":"e5de9631-8a44-45ed-9285-77579a6d2566"} {"msg":"[2]R generation[0..12]: [10, 10, 6, 10, 7, 6, 11, 11, 10, 10, 8, 6]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:24.082564976Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1230,"":"downstairs","":"downstairs","session_id":"e5de9631-8a44-45ed-9285-77579a6d2566"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:24.082578494Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1230,"":"downstairs","":"downstairs","session_id":"e5de9631-8a44-45ed-9285-77579a6d2566"} {"msg":"Max found gen is 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:24.082591012Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1230,"":"downstairs","":"downstairs","session_id":"e5de9631-8a44-45ed-9285-77579a6d2566"} {"msg":"Generation requested: 12 >= found:12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:24.082606731Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1230,"":"downstairs","":"downstairs","session_id":"e5de9631-8a44-45ed-9285-77579a6d2566"} {"msg":"Next flush: 17","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-17T21:11:24.082619828Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1230,"":"downstairs","":"downstairs","session_id":"e5de9631-8a44-45ed-9285-77579a6d2566"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:24.08263904Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1230,"":"downstairs","":"downstairs","session_id":"e5de9631-8a44-45ed-9285-77579a6d2566"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:24.082652257Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1230,"session_id":"e5de9631-8a44-45ed-9285-77579a6d2566"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:24.082664024Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1230,"client":"0","":"downstairs","session_id":"e5de9631-8a44-45ed-9285-77579a6d2566"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:24.082677022Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1230,"client":"1","":"downstairs","session_id":"e5de9631-8a44-45ed-9285-77579a6d2566"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:24.082689679Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1230,"client":"2","":"downstairs","session_id":"e5de9631-8a44-45ed-9285-77579a6d2566"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:24.082702557Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1230,"session_id":"e5de9631-8a44-45ed-9285-77579a6d2566"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:24.082714554Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1230,"session_id":"e5de9631-8a44-45ed-9285-77579a6d2566"} {"msg":"21392b3b-570e-4eed-9f99-3e75fa703e38 is now active with session: e5de9631-8a44-45ed-9285-77579a6d2566","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:24.082727201Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1230,"session_id":"e5de9631-8a44-45ed-9285-77579a6d2566"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:24.082738818Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1230,"session_id":"e5de9631-8a44-45ed-9285-77579a6d2566"} {"msg":"The guest has finished waiting for activation with:12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:24.082750735Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1230} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 536 len 24576 data: 2 2 2 2 2 2 02/30 Write block 561 len 32768 data: 3 3 3 2 2 3 2 2 03/30 Write block 574 len 16384 data: 2 2 2 2 04/30 Read block 152 len 40960 05/30 Write block 280 len 24576 data: 2 2 2 2 2 2 06/30 Write block 566 len 20480 data: 4 3 3 2 2 07/30 Read block 28 len 20480 08/30 Read block 398 len 36864 09/30 Read block 310 len 40960 10/30 Read block 68 len 28672 11/30 Read block 45 len 20480 12/30 Read block 264 len 28672 13/30 Read block 197 len 12288 14/30 Read block 174 len 20480 15/30 Write block 12 len 12288 data: 2 2 2 16/30 Read block 257 len 28672 17/30 Write block 464 len 20480 data: 3 3 2 3 3 18/30 Write block 87 len 8192 data: 2 2 19/30 Read block 216 len 12288 20/30 Read block 463 len 12288 21/30 Flush 22/30 Write block 542 len 40960 data: 2 2 2 2 3 3 3 3 3 3 23/30 Read block 477 len 24576 24/30 Write block 143 len 36864 data: 3 3 3 2 2 2 2 2 2 25/30 Write block 60 len 4096 data: 2 26/30 Read block 413 len 20480 27/30 Write block 256 len 36864 data: 2 2 2 2 2 2 2 2 2 28/30 Read block 263 len 40960 29/30 Read block 226 len 28672 30/30 Read block 107 len 16384 ---------------------------------------------------------------- Crucible gen:12 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 10 Done Done Done false 1022 Acked Read 6 Done Done Done false 1023 Acked Write 9 Done Done Done false 1024 Acked Write 1 Done Done Done false 1025 Acked Read 5 Done Done Done false 1026 Acked Write 9 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Read 7 Done Done Done false 1029 Acked Read 4 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 0 1 Done 8 9 9 26 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1020 1020 1020 Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:11:24.103103961Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1230} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 12 12 12 17 17 17 F F F 1 020-039 10 10 10 14 14 14 F F F 2 040-059 6 6 6 7 7 7 F F F 3 060-079 10 10 10 14 14 14 T T T 4 080-099 12 12 12 17 17 17 F F F 5 100-119 6 6 6 7 7 7 F F F 6 120-139 11 11 11 16 16 16 F F F 7 140-159 11 11 11 16 16 16 T T T 8 160-179 10 10 10 15 15 15 F F F 9 180-199 10 10 10 15 15 15 F F F 10 200-219 8 8 8 11 11 11 F F F 11 220-239 6 6 6 7 7 7 F F F 12 240-259 11 11 11 16 16 16 T T T 13 260-279 6 6 6 7 7 7 T T T 14 280-299 12 12 12 17 17 17 F F F 15 300-319 9 9 9 12 12 12 F F F 16 320-339 10 10 10 14 14 14 F F F 17 340-359 10 10 10 14 14 14 F F F 18 360-379 6 6 6 7 7 7 F F F 19 380-399 7 7 7 8 8 8 F F F 20 400-419 10 10 10 15 15 15 F F F 21 420-439 11 11 11 16 16 16 F F F 22 440-459 6 6 6 7 7 7 F F F 23 460-479 12 12 12 17 17 17 F F F 24 480-499 3 3 3 3 3 3 F F F 25 500-519 9 9 9 12 12 12 F F F 26 520-539 12 12 12 17 17 17 F F F 27 540-559 12 12 12 17 17 17 T T T 28 560-579 12 12 12 17 17 17 F F F 29 580-599 6 6 6 7 7 7 F F F Max gen: 12, Max flush: 17 A Difference in extent metadata is expected here On loop 6 Verifying data now No change after verify Loop: 6 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 12 12 12 17 17 17 F F F 1 020-039 10 10 10 14 14 14 F F F 2 040-059 6 6 6 7 7 7 F F F 3 060-079 13 13 13 18 18 18 F F F 4 080-099 12 12 12 17 17 17 F F F 5 100-119 6 6 6 7 7 7 F F F 6 120-139 11 11 11 16 16 16 F F F 7 140-159 13 13 13 18 18 18 F F F 8 160-179 10 10 10 15 15 15 F F F 9 180-199 10 10 10 15 15 15 F F F 10 200-219 8 8 8 11 11 11 F F F 11 220-239 6 6 6 7 7 7 F F F 12 240-259 13 13 13 18 18 18 F F F 13 260-279 13 13 13 18 18 18 F F F 14 280-299 12 12 12 17 17 17 F F F 15 300-319 9 9 9 12 12 12 F F F 16 320-339 10 10 10 14 14 14 F F F 17 340-359 10 10 10 14 14 14 F F F 18 360-379 6 6 6 7 7 7 F F F 19 380-399 7 7 7 8 8 8 F F F 20 400-419 10 10 10 15 15 15 F F F 21 420-439 11 11 11 16 16 16 F F F 22 440-459 6 6 6 7 7 7 F F F 23 460-479 12 12 12 17 17 17 F F F 24 480-499 3 3 3 3 3 3 F F F 25 500-519 9 9 9 12 12 12 F F F 26 520-539 12 12 12 17 17 17 F F F 27 540-559 13 13 13 18 18 18 F F F 28 560-579 12 12 12 17 17 17 F F F 29 580-599 6 6 6 7 7 7 F F F Max gen: 13, Max flush: 18 Begin loop 7 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:28.342380427Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1238,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:28.387518258Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1238,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:28.388163836Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1238} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"abd5430f1508af0bc008143ea2566277ef714ec2\",\n git_commit_timestamp: \"2025-01-17T20:55:03.000000000Z\",\n git_branch: \"alan/tokio-to-140\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:28.388204229Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1238} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:28.388218598Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1238} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:28.388232776Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1238} {"msg":"Crucible 8cc13f5a-0f30-4bae-9ece-d1531f24c11e has session id: 182d9708-ad0d-43ab-b754-c5ade9e0ab78","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:28.388772842Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1238,"session_id":"182d9708-ad0d-43ab-b754-c5ade9e0ab78"} {"msg":"Upstairs opts: Upstairs UUID: 8cc13f5a-0f30-4bae-9ece-d1531f24c11e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:28.388813605Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1238,"session_id":"182d9708-ad0d-43ab-b754-c5ade9e0ab78"} {"msg":"Crucible stats registered with UUID: 8cc13f5a-0f30-4bae-9ece-d1531f24c11e","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:28.388840241Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1238,"session_id":"182d9708-ad0d-43ab-b754-c5ade9e0ab78"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:11:28.388863795Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1238,"session_id":"182d9708-ad0d-43ab-b754-c5ade9e0ab78"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:28.389497715Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1238,"task":"crutest"} {"msg":"The guest has requested activation with gen:14","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:28.38952328Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1238} {"msg":"8cc13f5a-0f30-4bae-9ece-d1531f24c11e active request set","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:28.389611642Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1238,"session_id":"182d9708-ad0d-43ab-b754-c5ade9e0ab78"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:28.389637007Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1238,"client":"0","":"downstairs","session_id":"182d9708-ad0d-43ab-b754-c5ade9e0ab78"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:28.389662361Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1238,"client":"0","":"downstairs","session_id":"182d9708-ad0d-43ab-b754-c5ade9e0ab78"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:28.38967634Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1238,"client":"1","":"downstairs","session_id":"182d9708-ad0d-43ab-b754-c5ade9e0ab78"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:28.389695891Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1238,"client":"1","":"downstairs","session_id":"182d9708-ad0d-43ab-b754-c5ade9e0ab78"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:28.389722357Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1238,"client":"2","":"downstairs","session_id":"182d9708-ad0d-43ab-b754-c5ade9e0ab78"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:28.389752364Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1238,"client":"2","":"downstairs","session_id":"182d9708-ad0d-43ab-b754-c5ade9e0ab78"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:28.389778419Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1238,"":"io task","client":"2","":"downstairs","session_id":"182d9708-ad0d-43ab-b754-c5ade9e0ab78"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:28.389805685Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1238,"":"io task","client":"0","":"downstairs","session_id":"182d9708-ad0d-43ab-b754-c5ade9e0ab78"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:28.389831711Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1238,"":"io task","client":"1","":"downstairs","session_id":"182d9708-ad0d-43ab-b754-c5ade9e0ab78"} {"msg":"ds_connection connected from Ok(127.0.0.1:36369)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:28.389856575Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1238,"":"io task","client":"2","":"downstairs","session_id":"182d9708-ad0d-43ab-b754-c5ade9e0ab78"} {"msg":"ds_connection connected from Ok(127.0.0.1:49644)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:28.389881289Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1238,"":"io task","client":"0","":"downstairs","session_id":"182d9708-ad0d-43ab-b754-c5ade9e0ab78"} {"msg":"ds_connection connected from Ok(127.0.0.1:54589)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:28.389907135Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1238,"":"io task","client":"1","":"downstairs","session_id":"182d9708-ad0d-43ab-b754-c5ade9e0ab78"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:28.390622383Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1238,"client":"0","":"downstairs","session_id":"182d9708-ad0d-43ab-b754-c5ade9e0ab78"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:28.390661365Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1238,"client":"1","":"downstairs","session_id":"182d9708-ad0d-43ab-b754-c5ade9e0ab78"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:28.390780035Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1238,"client":"2","":"downstairs","session_id":"182d9708-ad0d-43ab-b754-c5ade9e0ab78"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:28.391255953Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1238,"client":"1","":"downstairs","session_id":"182d9708-ad0d-43ab-b754-c5ade9e0ab78"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:28.391296867Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1238,"client":"0","":"downstairs","session_id":"182d9708-ad0d-43ab-b754-c5ade9e0ab78"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:28.391342253Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1238,"client":"2","":"downstairs","session_id":"182d9708-ad0d-43ab-b754-c5ade9e0ab78"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:28.39179895Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1238,"session_id":"182d9708-ad0d-43ab-b754-c5ade9e0ab78"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:28.391890244Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1238,"session_id":"182d9708-ad0d-43ab-b754-c5ade9e0ab78"} {"msg":"[0]R flush_numbers[0..12]: [17, 14, 7, 18, 17, 7, 16, 18, 15, 15, 11, 7]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:28.391964927Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1238,"":"downstairs","":"downstairs","session_id":"182d9708-ad0d-43ab-b754-c5ade9e0ab78"} {"msg":"[0]R generation[0..12]: [12, 10, 6, 13, 12, 6, 11, 13, 10, 10, 8, 6]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:28.391983118Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1238,"":"downstairs","":"downstairs","session_id":"182d9708-ad0d-43ab-b754-c5ade9e0ab78"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:28.391997436Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1238,"":"downstairs","":"downstairs","session_id":"182d9708-ad0d-43ab-b754-c5ade9e0ab78"} {"msg":"[1]R flush_numbers[0..12]: [17, 14, 7, 18, 17, 7, 16, 18, 15, 15, 11, 7]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:28.392011204Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1238,"":"downstairs","":"downstairs","session_id":"182d9708-ad0d-43ab-b754-c5ade9e0ab78"} {"msg":"[1]R generation[0..12]: [12, 10, 6, 13, 12, 6, 11, 13, 10, 10, 8, 6]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:28.392025713Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1238,"":"downstairs","":"downstairs","session_id":"182d9708-ad0d-43ab-b754-c5ade9e0ab78"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:28.392039411Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1238,"":"downstairs","":"downstairs","session_id":"182d9708-ad0d-43ab-b754-c5ade9e0ab78"} {"msg":"[2]R flush_numbers[0..12]: [17, 14, 7, 18, 17, 7, 16, 18, 15, 15, 11, 7]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:28.392056791Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1238,"":"downstairs","":"downstairs","session_id":"182d9708-ad0d-43ab-b754-c5ade9e0ab78"} {"msg":"[2]R generation[0..12]: [12, 10, 6, 13, 12, 6, 11, 13, 10, 10, 8, 6]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:28.392079574Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1238,"":"downstairs","":"downstairs","session_id":"182d9708-ad0d-43ab-b754-c5ade9e0ab78"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:28.392093402Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1238,"":"downstairs","":"downstairs","session_id":"182d9708-ad0d-43ab-b754-c5ade9e0ab78"} {"msg":"Max found gen is 14","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:28.39210594Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1238,"":"downstairs","":"downstairs","session_id":"182d9708-ad0d-43ab-b754-c5ade9e0ab78"} {"msg":"Generation requested: 14 >= found:14","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:28.392118627Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1238,"":"downstairs","":"downstairs","session_id":"182d9708-ad0d-43ab-b754-c5ade9e0ab78"} {"msg":"Next flush: 19","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:28.392131284Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1238,"":"downstairs","":"downstairs","session_id":"182d9708-ad0d-43ab-b754-c5ade9e0ab78"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:28.392143962Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1238,"":"downstairs","":"downstairs","session_id":"182d9708-ad0d-43ab-b754-c5ade9e0ab78"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:28.39215697Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1238,"session_id":"182d9708-ad0d-43ab-b754-c5ade9e0ab78"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:28.392168576Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1238,"client":"0","":"downstairs","session_id":"182d9708-ad0d-43ab-b754-c5ade9e0ab78"} {"msg":"Transition from WaitQuorum to Active","v":0Wait for a query_work_queue command to finish before sending IO ,"name":"crucible","level":30,"time":"2025-01-17T21:11:28.392189869Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1238,"client":"1","":"downstairs","session_id":"182d9708-ad0d-43ab-b754-c5ade9e0ab78"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:28.392202556Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1238,"client":"2","":"downstairs","session_id":"182d9708-ad0d-43ab-b754-c5ade9e0ab78"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:28.392215224Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1238,"session_id":"182d9708-ad0d-43ab-b754-c5ade9e0ab78"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:28.39222704Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1238,"session_id":"182d9708-ad0d-43ab-b754-c5ade9e0ab78"} {"msg":"8cc13f5a-0f30-4bae-9ece-d1531f24c11e is now active with session: 182d9708-ad0d-43ab-b754-c5ade9e0ab78","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:28.392239318Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1238,"session_id":"182d9708-ad0d-43ab-b754-c5ade9e0ab78"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:28.392250865Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1238,"session_id":"182d9708-ad0d-43ab-b754-c5ade9e0ab78"} {"msg":"The guest has finished waiting for activation with:14","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:28.392262752Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1238} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 307 len 12288 data: 3 3 3 02/30 Read block 331 len 40960 03/30 Write block 38 len 16384 data: 3 3 2 2 04/30 Write block 319 len 20480 data: 3 3 3 2 2 05/30 Read block 276 len 20480 06/30 Write block 175 len 20480 data: 3 3 3 3 3 07/30 Read block 275 len 16384 08/30 Write block 343 len 4096 data: 4 09/30 Read block 532 len 40960 10/30 Read block 305 len 28672 11/30 Read block 113 len 40960 12/30 Read block 586 len 28672 13/30 Write block 1 len 4096 data: 3 14/30 Write block 179 len 20480 data: 4 4 4 3 3 15/30 Read block 126 len 20480 16/30 Flush 17/30 Write block 12 len 4096 data: 3 18/30 Read block 398 len 16384 19/30 Read block 39 len 8192 20/30 Read block 201 len 8192 21/30 Write block 16 len 16384 data: 2 2 2 2 22/30 Flush 23/30 Flush 24/30 Write block 165 len 24576 data: 2 2 2 2 2 2 25/30 Flush 26/30 Write block 224 len 28672 data: 2 2 2 2 2 2 2 27/30 Read block 416 len 16384 28/30 Write block 190 len 20480 data: 4 4 4 4 4 29/30 Read block 344 len 8192 30/30 Read block 233 len 16384 ---------------------------------------------------------------- Crucible gen:14 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 7 Done Done Done false 1026 Acked Read 4 Done Done Done false 1027 Acked Write 5 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Read 4 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 1 2 Done 4 5 4 13 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:11:28.411277675Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1238} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 14 14 14 20 20 20 F F F 1 020-039 14 14 14 19 19 19 F F F 2 040-059 14 14 14 19 19 19 F F F 3 060-079 13 13 13 18 18 18 F F F 4 080-099 12 12 12 17 17 17 F F F 5 100-119 6 6 6 7 7 7 F F F 6 120-139 11 11 11 16 16 16 F F F 7 140-159 13 13 13 18 18 18 F F F 8 160-179 14 14 14 22 22 22 F F F 9 180-199 14 14 14 19 19 19 T T T 10 200-219 8 8 8 11 11 11 F F F 11 220-239 6 6 6 7 7 7 T T T 12 240-259 13 13 13 18 18 18 F F F 13 260-279 13 13 13 18 18 18 F F F 14 280-299 12 12 12 17 17 17 F F F 15 300-319 14 14 14 19 19 19 F F F 16 320-339 14 14 14 19 19 19 F F F 17 340-359 14 14 14 19 19 19 F F F 18 360-379 6 6 6 7 7 7 F F F 19 380-399 7 7 7 8 8 8 F F F 20 400-419 10 10 10 15 15 15 F F F 21 420-439 11 11 11 16 16 16 F F F 22 440-459 6 6 6 7 7 7 F F F 23 460-479 12 12 12 17 17 17 F F F 24 480-499 3 3 3 3 3 3 F F F 25 500-519 9 9 9 12 12 12 F F F 26 520-539 12 12 12 17 17 17 F F F 27 540-559 13 13 13 18 18 18 F F F 28 560-579 12 12 12 17 17 17 F F F 29 580-599 6 6 6 7 7 7 F F F Max gen: 14, Max flush: 22 A Difference in extent metadata is expected here On loop 7 Verifying data now No change after verify Loop: 7 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 14 14 14 20 20 20 F F F 1 020-039 14 14 14 19 19 19 F F F 2 040-059 14 14 14 19 19 19 F F F 3 060-079 13 13 13 18 18 18 F F F 4 080-099 12 12 12 17 17 17 F F F 5 100-119 6 6 6 7 7 7 F F F 6 120-139 11 11 11 16 16 16 F F F 7 140-159 13 13 13 18 18 18 F F F 8 160-179 14 14 14 22 22 22 F F F 9 180-199 15 15 15 23 23 23 F F F 10 200-219 8 8 8 11 11 11 F F F 11 220-239 15 15 15 23 23 23 F F F 12 240-259 13 13 13 18 18 18 F F F 13 260-279 13 13 13 18 18 18 F F F 14 280-299 12 12 12 17 17 17 F F F 15 300-319 14 14 14 19 19 19 F F F 16 320-339 14 14 14 19 19 19 F F F 17 340-359 14 14 14 19 19 19 F F F 18 360-379 6 6 6 7 7 7 F F F 19 380-399 7 7 7 8 8 8 F F F 20 400-419 10 10 10 15 15 15 F F F 21 420-439 11 11 11 16 16 16 F F F 22 440-459 6 6 6 7 7 7 F F F 23 460-479 12 12 12 17 17 17 F F F 24 480-499 3 3 3 3 3 3 F F F 25 500-519 9 9 9 12 12 12 F F F 26 520-539 12 12 12 17 17 17 F F F 27 540-559 13 13 13 18 18 18 F F F 28 560-579 12 12 12 17 17 17 F F F 29 580-599 6 6 6 7 7 7 F F F Max gen: 15, Max flush: 23 Begin loop 8 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:32.633180757Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1246,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:11:32.676957213Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1246,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:32.677392107Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1246,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:32.709842704Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1246,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:32.710490733Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1246} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"abd5430f1508af0bc008143ea2566277ef714ec2\",\n git_commit_timestamp: \"2025-01-17T20:55:03.000000000Z\",\n git_branch: \"alan/tokio-to-140\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:32.710532977Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1246} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:32.710549587Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1246} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:32.710567217Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1246} {"msg":"Crucible 34b410f7-efcb-496d-982c-3b1c56a3eddf has session id: be520ec1-7d7a-4c26-a945-7f54756b3e5e","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:32.711182007Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1246,"session_id":"be520ec1-7d7a-4c26-a945-7f54756b3e5e"} {"msg":"Upstairs opts: Upstairs UUID: 34b410f7-efcb-496d-982c-3b1c56a3eddf, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:32.711216247Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1246,"session_id":"be520ec1-7d7a-4c26-a945-7f54756b3e5e"} {"msg":"Crucible stats registered with UUID: 34b410f7-efcb-496d-982c-3b1c56a3eddf","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:32.711233897Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1246,"session_id":"be520ec1-7d7a-4c26-a945-7f54756b3e5e"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:11:32.711249346Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1246,"session_id":"be520ec1-7d7a-4c26-a945-7f54756b3e5e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:32.711687382Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1246,"task":"crutest"} {"msg":"The guest has requested activation with gen:16","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:32.711705993Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1246} {"msg":"34b410f7-efcb-496d-982c-3b1c56a3eddf active request set","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:32.711776684Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1246,"session_id":"be520ec1-7d7a-4c26-a945-7f54756b3e5e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:32.711793764Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1246,"client":"0","":"downstairs","session_id":"be520ec1-7d7a-4c26-a945-7f54756b3e5e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:32.711817348Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1246,"client":"0","":"downstairs","session_id":"be520ec1-7d7a-4c26-a945-7f54756b3e5e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:32.711830986Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1246,"client":"1","":"downstairs","session_id":"be520ec1-7d7a-4c26-a945-7f54756b3e5e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:32.711844784Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1246,"client":"1","":"downstairs","session_id":"be520ec1-7d7a-4c26-a945-7f54756b3e5e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:32.711857621Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1246,"client":"2","":"downstairs","session_id":"be520ec1-7d7a-4c26-a945-7f54756b3e5e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:32.7118715Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1246,"client":"2","":"downstairs","session_id":"be520ec1-7d7a-4c26-a945-7f54756b3e5e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:32.711885568Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1246,"":"io task","client":"2","":"downstairs","session_id":"be520ec1-7d7a-4c26-a945-7f54756b3e5e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:32.711910332Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1246,"":"io task","client":"0","":"downstairs","session_id":"be520ec1-7d7a-4c26-a945-7f54756b3e5e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:32.711936378Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1246,"":"io task","client":"1","":"downstairs","session_id":"be520ec1-7d7a-4c26-a945-7f54756b3e5e"} {"msg":"ds_connection connected from Ok(127.0.0.1:48142)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:32.71197491Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1246,"":"io task","client":"0","":"downstairs","session_id":"be520ec1-7d7a-4c26-a945-7f54756b3e5e"} {"msg":"ds_connection connected from Ok(127.0.0.1:35407)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:32.712181431Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1246,"":"io task","client":"1","":"downstairs","session_id":"be520ec1-7d7a-4c26-a945-7f54756b3e5e"} {"msg":"ds_connection connected from Ok(127.0.0.1:37687)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:32.712254444Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1246,"":"io task","client":"2","":"downstairs","session_id":"be520ec1-7d7a-4c26-a945-7f54756b3e5e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:32.712748562Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1246,"client":"2","":"downstairs","session_id":"be520ec1-7d7a-4c26-a945-7f54756b3e5e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:32.712775018Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1246,"client":"1","":"downstairs","session_id":"be520ec1-7d7a-4c26-a945-7f54756b3e5e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:32.712920483Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1246,"client":"0","":"downstairs","session_id":"be520ec1-7d7a-4c26-a945-7f54756b3e5e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:32.71341241Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1246,"client":"1","":"downstairs","session_id":"be520ec1-7d7a-4c26-a945-7f54756b3e5e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:32.713440967Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1246,"client":"2","":"downstairs","session_id":"be520ec1-7d7a-4c26-a945-7f54756b3e5e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:32.713456706Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1246,"client":"0","":"downstairs","session_id":"be520ec1-7d7a-4c26-a945-7f54756b3e5e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:32.713992809Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1246,"session_id":"be520ec1-7d7a-4c26-a945-7f54756b3e5e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:32.71402811Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1246,"session_id":"be520ec1-7d7a-4c26-a945-7f54756b3e5e"} {"msg":"[0]R flush_numbers[0..12]: [20, 19, 19, 18, 17, 7, 16, 18, 22, 23, 11, 23]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:32.714106916Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1246,"":"downstairs","":"downstairs","session_id":"be520ec1-7d7a-4c26-a945-7f54756b3e5e"} {"msg":"[0]R generation[0..12]: [14, 14, 14, 13, 12, 6, 11, 13, 14, 15, 8, 15]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:32.714133962Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1246,"":"downstairs","":"downstairs","session_id":"be520ec1-7d7a-4c26-a945-7f54756b3e5e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:32.714153053Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1246,"":"downstairs","":"downstairs","session_id":"be520ec1-7d7a-4c26-a945-7f54756b3e5e"} {"msg":"[1]R flush_numbers[0..12]: [20, 19, 19, 18, 17, 7, 16, 18, 22, 23, 11, 23]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:32.714167462Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1246,"":"downstairs","":"downstairs","session_id":"be520ec1-7d7a-4c26-a945-7f54756b3e5e"} {"msg":"[1]R generation[0..12]: [14, 14, 14, 13, 12, 6, 11, 13, 14, 15, 8, 15]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:32.714180799Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1246,"":"downstairs","":"downstairs","session_id":"be520ec1-7d7a-4c26-a945-7f54756b3e5e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:32.714195648Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1246,"":"downstairs","":"downstairs","session_id":"be520ec1-7d7a-4c26-a945-7f54756b3e5e"} {"msg":"[2]R flush_numbers[0..12]: [20, 19, 19, 18, 17, 7, 16, 18, 22, 23, 11, 23]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:32.714209146Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1246,"":"downstairs","":"downstairs","session_id":"be520ec1-7d7a-4c26-a945-7f54756b3e5e"} {"msg":"[2]R generation[0..12]: [14, 14, 14, 13, 12, 6, 11, 13, 14, 15, 8, 15]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:32.714222254Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1246,"":"downstairs","":"downstairs","session_id":"be520ec1-7d7a-4c26-a945-7f54756b3e5e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:32.714235771Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1246,"":"downstairs","":"downstairs","session_id":"be520ec1-7d7a-4c26-a945-7f54756b3e5e"} {"msg":"Max found gen is 16","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:32.714248389Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1246,"":"downstairs","":"downstairs","session_id":"be520ec1-7d7a-4c26-a945-7f54756b3e5e"} {"msg":"Generation requested: 16 >= found:16","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:32.714261266Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1246,"":"downstairs","":"downstairs","session_id":"be520ec1-7d7a-4c26-a945-7f54756b3e5e"} Wait for a query_work_queue command to finish before sending IO {"msg":"Next flush: 24","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:32.714279937Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1246,"":"downstairs","":"downstairs","session_id":"be520ec1-7d7a-4c26-a945-7f54756b3e5e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:32.714295116Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1246,"":"downstairs","":"downstairs","session_id":"be520ec1-7d7a-4c26-a945-7f54756b3e5e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:32.714317709Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1246,"session_id":"be520ec1-7d7a-4c26-a945-7f54756b3e5e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:32.714330547Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1246,"client":"0","":"downstairs","session_id":"be520ec1-7d7a-4c26-a945-7f54756b3e5e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:32.714343645Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1246,"client":"1","":"downstairs","session_id":"be520ec1-7d7a-4c26-a945-7f54756b3e5e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:32.714358523Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1246,"client":"2","":"downstairs","session_id":"be520ec1-7d7a-4c26-a945-7f54756b3e5e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:32.714371671Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1246,"session_id":"be520ec1-7d7a-4c26-a945-7f54756b3e5e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:32.714383768Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1246,"session_id":"be520ec1-7d7a-4c26-a945-7f54756b3e5e"} {"msg":"34b410f7-efcb-496d-982c-3b1c56a3eddf is now active with session: be520ec1-7d7a-4c26-a945-7f54756b3e5e","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:32.714396155Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1246,"session_id":"be520ec1-7d7a-4c26-a945-7f54756b3e5e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:32.714407572Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1246,"session_id":"be520ec1-7d7a-4c26-a945-7f54756b3e5e"} {"msg":"The guest has finished waiting for activation with:16","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:32.714419319Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1246} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 76 len 8192 data: 3 3 02/30 Write block 192 len 40960 data: 5 5 5 3 3 2 3 3 3 3 03/30 Flush 04/30 Write block 505 len 20480 data: 3 3 3 3 3 05/30 Write block 52 len 28672 data: 4 3 3 3 3 3 3 06/30 Flush 07/30 Write block 315 len 40960 data: 3 3 3 3 4 4 4 3 3 2 08/30 Write block 523 len 4096 data: 2 09/30 Read block 528 len 36864 10/30 Read block 342 len 4096 11/30 Read block 90 len 20480 12/30 Read block 108 len 20480 13/30 Read block 145 len 24576 14/30 Write block 330 len 8192 data: 2 2 15/30 Write block 134 len 28672 data: 2 2 3 3 3 4 4 16/30 Flush 17/30 Write block 222 len 40960 data: 3 3 3 3 3 3 3 3 3 2 18/30 Read block 92 len 16384 19/30 Write block 11 len 24576 data: 3 4 3 3 2 3 20/30 Write block 112 len 16384 data: 2 2 3 3 21/30 Read block 320 len 36864 22/30 Write block 515 len 28672 data: 3 3 2 2 2 2 3 23/30 Read block 231 len 20480 24/30 Write block 37 len 32768 data: 3 4 4 3 3 2 2 2 25/30 Read block 283 len 32768 26/30 Read block 536 len 36864 27/30 Write block 181 len 20480 data: 5 4 4 3 2 28/30 Read block 234 len 40960 29/30 Write block 127 len 16384 data: 2 2 2 2 30/30 Read block 481 len 12288 ---------------------------------------------------------------- Crucible gen:16 GIO:true work queues: Upstairs:0 downstairs:14 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1016 Acked Write 10 Done Done Done false 1017 Acked Read 4 Done Done Done false 1018 Acked Write 6 Done Done Done false 1019 Acked Write 4 Done Done Done false 1020 Acked Read 9 Done Done Done false 1021 Acked Write 7 Done Done Done false 1022 Acked Read 5 Done Done Done false 1023 Acked Write 8 Done Done Done false 1024 Acked Read 8 Done Done Done false 1025 Acked Read 9 Done Done Done false 1026 Acked Write 5 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Write 4 Done Sent Sent false 1029 Acked Read 3 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 2 2 4 Done 14 12 12 38 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1015 1015 1015 Downstairs last five completed: 1015 1014 1013 1012 1011 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:11:32.735015997Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1246} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 14 14 14 20 20 20 T T T 1 020-039 14 14 14 19 19 19 T T T 2 040-059 16 16 16 25 25 25 T T T 3 060-079 16 16 16 24 24 24 F F F 4 080-099 12 12 12 17 17 17 F F F 5 100-119 6 6 6 7 7 7 T T T 6 120-139 16 16 16 26 26 26 T T T 7 140-159 16 16 16 26 26 26 F F F 8 160-179 14 14 14 22 22 22 F F F 9 180-199 16 16 16 24 24 24 T T T 10 200-219 16 16 16 24 24 24 F F F 11 220-239 15 15 15 23 23 23 T T T 12 240-259 13 13 13 18 18 18 F F F 13 260-279 13 13 13 18 18 18 F F F 14 280-299 12 12 12 17 17 17 F F F 15 300-319 16 16 16 26 26 26 F F F 16 320-339 16 16 16 26 26 26 F F F 17 340-359 14 14 14 19 19 19 F F F 18 360-379 6 6 6 7 7 7 F F F 19 380-399 7 7 7 8 8 8 F F F 20 400-419 10 10 10 15 15 15 F F F 21 420-439 11 11 11 16 16 16 F F F 22 440-459 6 6 6 7 7 7 F F F 23 460-479 12 12 12 17 17 17 F F F 24 480-499 3 3 3 3 3 3 F F F 25 500-519 16 16 16 25 25 25 T T T 26 520-539 16 16 16 26 26 26 T T T 27 540-559 13 13 13 18 18 18 F F F 28 560-579 12 12 12 17 17 17 F F F 29 580-599 6 6 6 7 7 7 F F F Max gen: 16, Max flush: 26 A Difference in extent metadata is expected here On loop 8 Verifying data now No change after verify Loop: 8 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 17 17 17 27 27 27 F F F 1 020-039 17 17 17 27 27 27 F F F 2 040-059 17 17 17 27 27 27 F F F 3 060-079 16 16 16 24 24 24 F F F 4 080-099 12 12 12 17 17 17 F F F 5 100-119 17 17 17 27 27 27 F F F 6 120-139 17 17 17 27 27 27 F F F 7 140-159 16 16 16 26 26 26 F F F 8 160-179 14 14 14 22 22 22 F F F 9 180-199 17 17 17 27 27 27 F F F 10 200-219 16 16 16 24 24 24 F F F 11 220-239 17 17 17 27 27 27 F F F 12 240-259 13 13 13 18 18 18 F F F 13 260-279 13 13 13 18 18 18 F F F 14 280-299 12 12 12 17 17 17 F F F 15 300-319 16 16 16 26 26 26 F F F 16 320-339 16 16 16 26 26 26 F F F 17 340-359 14 14 14 19 19 19 F F F 18 360-379 6 6 6 7 7 7 F F F 19 380-399 7 7 7 8 8 8 F F F 20 400-419 10 10 10 15 15 15 F F F 21 420-439 11 11 11 16 16 16 F F F 22 440-459 6 6 6 7 7 7 F F F 23 460-479 12 12 12 17 17 17 F F F 24 480-499 3 3 3 3 3 3 F F F 25 500-519 17 17 17 27 27 27 F F F 26 520-539 17 17 17 27 27 27 F F F 27 540-559 13 13 13 18 18 18 F F F 28 560-579 12 12 12 17 17 17 F F F 29 580-599 6 6 6 7 7 7 F F F Max gen: 17, Max flush: 27 Begin loop 9 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:37.02645704Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1254,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:11:37.070724233Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1254,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:37.071193077Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1254,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:37.103711233Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1254,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:37.10432275Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1254} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"abd5430f1508af0bc008143ea2566277ef714ec2\",\n git_commit_timestamp: \"2025-01-17T20:55:03.000000000Z\",\n git_branch: \"alan/tokio-to-140\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:37.10435571Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1254} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:37.104369098Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1254} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:37.104383426Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1254} {"msg":"Crucible 4a18e9a8-12c6-4797-b8a3-7774c81c683a has session id: da71bc00-95c6-481d-b347-ba61590ed92c","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:37.104966757Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1254,"session_id":"da71bc00-95c6-481d-b347-ba61590ed92c"} {"msg":"Upstairs opts: Upstairs UUID: 4a18e9a8-12c6-4797-b8a3-7774c81c683a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:37.105000757Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1254,"session_id":"da71bc00-95c6-481d-b347-ba61590ed92c"} {"msg":"Crucible stats registered with UUID: 4a18e9a8-12c6-4797-b8a3-7774c81c683a","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:37.105025872Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1254,"session_id":"da71bc00-95c6-481d-b347-ba61590ed92c"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:11:37.105049555Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1254,"session_id":"da71bc00-95c6-481d-b347-ba61590ed92c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:37.105474774Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1254,"task":"crutest"} {"msg":"The guest has requested activation with gen:18","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:37.105506753Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1254} {"msg":"4a18e9a8-12c6-4797-b8a3-7774c81c683a active request set","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:37.105559904Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1254,"session_id":"da71bc00-95c6-481d-b347-ba61590ed92c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:37.105588761Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1254,"client":"0","":"downstairs","session_id":"da71bc00-95c6-481d-b347-ba61590ed92c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:37.105614556Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1254,"client":"0","":"downstairs","session_id":"da71bc00-95c6-481d-b347-ba61590ed92c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:37.105629104Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1254,"client":"1","":"downstairs","session_id":"da71bc00-95c6-481d-b347-ba61590ed92c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:37.105643172Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1254,"client":"1","":"downstairs","session_id":"da71bc00-95c6-481d-b347-ba61590ed92c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:37.10565594Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1254,"client":"2","":"downstairs","session_id":"da71bc00-95c6-481d-b347-ba61590ed92c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:37.105679283Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1254,"client":"2","":"downstairs","session_id":"da71bc00-95c6-481d-b347-ba61590ed92c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:37.105703528Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1254,"":"io task","client":"2","":"downstairs","session_id":"da71bc00-95c6-481d-b347-ba61590ed92c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:37.105727111Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1254,"":"io task","client":"0","":"downstairs","session_id":"da71bc00-95c6-481d-b347-ba61590ed92c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:37.105751145Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1254,"":"io task","client":"1","":"downstairs","session_id":"da71bc00-95c6-481d-b347-ba61590ed92c"} {"msg":"ds_connection connected from Ok(127.0.0.1:62907)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:37.105795231Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1254,"":"io task","client":"2","":"downstairs","session_id":"da71bc00-95c6-481d-b347-ba61590ed92c"} {"msg":"ds_connection connected from Ok(127.0.0.1:58833)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:37.106096707Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1254,"":"io task","client":"1","":"downstairs","session_id":"da71bc00-95c6-481d-b347-ba61590ed92c"} {"msg":"ds_connection connected from Ok(127.0.0.1:41928)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:37.10616973Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1254,"":"io task","client":"0","":"downstairs","session_id":"da71bc00-95c6-481d-b347-ba61590ed92c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:37.106469205Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1254,"client":"2","":"downstairs","session_id":"da71bc00-95c6-481d-b347-ba61590ed92c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:37.106713688Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1254,"client":"1","":"downstairs","session_id":"da71bc00-95c6-481d-b347-ba61590ed92c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:37.106795986Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1254,"client":"0","":"downstairs","session_id":"da71bc00-95c6-481d-b347-ba61590ed92c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:37.107191758Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1254,"client":"2","":"downstairs","session_id":"da71bc00-95c6-481d-b347-ba61590ed92c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:37.107220024Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1254,"client":"1","":"downstairs","session_id":"da71bc00-95c6-481d-b347-ba61590ed92c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:37.107320933Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1254,"client":"0","":"downstairs","session_id":"da71bc00-95c6-481d-b347-ba61590ed92c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:37.107785565Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1254,"session_id":"da71bc00-95c6-481d-b347-ba61590ed92c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:37.107820325Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1254,"session_id":"da71bc00-95c6-481d-b347-ba61590ed92c"} {"msg":"[0]R flush_numbers[0..12]: [27, 27, 27, 24, 17, 27, 27, 26, 22, 27, 24, 27]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:37.10791506Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1254,"":"downstairs","":"downstairs","session_id":"da71bc00-95c6-481d-b347-ba61590ed92c"} {"msg":"[0]R generation[0..12]: [17, 17, 17, 16, 12, 17, 17, 16, 14, 17, 16, 17]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:37.107935142Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1254,"":"downstairs","":"downstairs","session_id":"da71bc00-95c6-481d-b347-ba61590ed92c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:37.107949781Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1254,"":"downstairs","":"downstairs","session_id":"da71bc00-95c6-481d-b347-ba61590ed92c"} {"msg":"[1]R flush_numbers[0..12]: [27, 27, 27, 24, 17, 27, 27, 26, 22, 27, 24, 27]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:37.107963569Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1254,"":"downstairs","":"downstairs","session_id":"da71bc00-95c6-481d-b347-ba61590ed92c"} {"msg":"[1]R generation[0..12]: [17, 17, 17, 16, 12, 17, 17, 16, 14, 17, 16, 17]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:37.107976886Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1254,"":"downstairs","":"downstairs","session_id":"da71bc00-95c6-481d-b347-ba61590ed92c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:37.107990534Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1254,"":"downstairs","":"downstairs","session_id":"da71bc00-95c6-481d-b347-ba61590ed92c"} {"msg":"[2]R flush_numbers[0..12]: [27, 27, 27, 24, 17, 27, 27, 26, 22, 27, 24, 27]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:37.10801825Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1254,"":"downstairs","":"downstairs","session_id":"da71bc00-95c6-481d-b347-ba61590ed92c"} {"msg":"[2]R generation[0..12]: [17, 17, 17, 16, 12, 17, 17, 16, 14, 17, 16, 17]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:37.108032069Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1254,"":"downstairs","":"downstairs","session_id":"da71bc00-95c6-481d-b347-ba61590ed92c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:37.108045566Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1254,"":"downstairs","":"downstairs","session_id":"da71bc00-95c6-481d-b347-ba61590ed92c"} {"msg":"Max found gen is 18","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:37.108061786Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1254,"":"downstairs","":"downstairs","session_id":"da71bc00-95c6-481d-b347-ba61590ed92c"} {"msg":"Generation requested: 18 >= found:18","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:37.108075084Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1254,"":"downstairs","":"downstairs","session_id":"da71bc00-95c6-481d-b347-ba61590ed92c"} {"msg":"Next flush: 28","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:37.108087701Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1254,"":"downstairs","":"downstairs","session_id":"da71bc00-95c6-481d-b347-ba61590ed92c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:37.108100218Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1254,"":"downstairs","":"downstairs","session_id":"da71bc00-95c6-481d-b347-ba61590ed92cWait for a query_work_queue command to finish before sending IO "} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:37.108121751Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1254,"session_id":"da71bc00-95c6-481d-b347-ba61590ed92c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:37.108133278Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1254,"client":"0","":"downstairs","session_id":"da71bc00-95c6-481d-b347-ba61590ed92c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:37.108146175Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1254,"client":"1","":"downstairs","session_id":"da71bc00-95c6-481d-b347-ba61590ed92c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:37.108158773Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1254,"client":"2","":"downstairs","session_id":"da71bc00-95c6-481d-b347-ba61590ed92c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:37.10817154Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1254,"session_id":"da71bc00-95c6-481d-b347-ba61590ed92c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:37.108183487Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1254,"session_id":"da71bc00-95c6-481d-b347-ba61590ed92c"} {"msg":"4a18e9a8-12c6-4797-b8a3-7774c81c683a is now active with session: da71bc00-95c6-481d-b347-ba61590ed92c","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:37.108195814Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1254,"session_id":"da71bc00-95c6-481d-b347-ba61590ed92c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:37.108207241Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1254,"session_id":"da71bc00-95c6-481d-b347-ba61590ed92c"} {"msg":"The guest has finished waiting for activation with:18","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:37.108218938Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1254} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 426 len 8192 data: 2 2 02/30 Read block 336 len 12288 03/30 Write block 332 len 20480 data: 2 2 2 3 3 04/30 Write block 476 len 28672 data: 2 2 2 2 3 3 3 05/30 Read block 369 len 40960 06/30 Read block 390 len 28672 07/30 Write block 107 len 24576 data: 2 2 2 2 2 3 08/30 Flush 09/30 Write block 513 len 32768 data: 3 3 4 4 3 3 3 3 10/30 Read block 64 len 20480 11/30 Read block 168 len 32768 12/30 Write block 372 len 28672 data: 3 3 4 4 4 6 6 13/30 Flush 14/30 Write block 365 len 4096 data: 3 15/30 Read block 184 len 24576 16/30 Read block 216 len 4096 17/30 Write block 230 len 32768 data: 4 3 2 2 2 2 2 2 18/30 Read block 347 len 24576 19/30 Read block 502 len 20480 20/30 Write block 469 len 16384 data: 3 3 3 3 21/30 Flush 22/30 Write block 343 len 36864 data: 5 4 3 2 2 2 2 2 2 23/30 Read block 347 len 28672 24/30 Read block 171 len 12288 25/30 Write block 579 len 28672 data: 2 2 2 2 2 2 2 26/30 Read block 184 len 40960 27/30 Read block 53 len 4096 28/30 Write block 53 len 28672 data: 4 4 4 4 4 4 3 29/30 Read block 335 len 32768 30/30 Write block 332 len 20480 data: 3 3 3 4 4 ---------------------------------------------------------------- Crucible gen:18 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 9 Done Done Done false 1022 Acked Read 7 Done Done Done false 1023 Acked Read 3 Done Done Done false 1024 Acked Write 7 Done Done Done false 1025 Acked Read 10 Done Done Done false 1026 Acked Read 1 Done Done Done false 1027 Acked Write 7 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 8 8 8 24 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1020 1020 1020 Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:11:37.129375913Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1254} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 17 17 17 27 27 27 F F F 1 020-039 17 17 17 27 27 27 F F F 2 040-059 17 17 17 27 27 27 T T T 3 060-079 16 16 16 24 24 24 F F F 4 080-099 12 12 12 17 17 17 F F F 5 100-119 18 18 18 28 28 28 F F F 6 120-139 17 17 17 27 27 27 F F F 7 140-159 16 16 16 26 26 26 F F F 8 160-179 14 14 14 22 22 22 F F F 9 180-199 17 17 17 27 27 27 F F F 10 200-219 16 16 16 24 24 24 F F F 11 220-239 18 18 18 30 30 30 F F F 12 240-259 13 13 13 18 18 18 F F F 13 260-279 13 13 13 18 18 18 F F F 14 280-299 12 12 12 17 17 17 F F F 15 300-319 16 16 16 26 26 26 F F F 16 320-339 18 18 18 28 28 28 T T T 17 340-359 14 14 14 19 19 19 T T T 18 360-379 18 18 18 30 30 30 F F F 19 380-399 7 7 7 8 8 8 F F F 20 400-419 10 10 10 15 15 15 F F F 21 420-439 18 18 18 28 28 28 F F F 22 440-459 6 6 6 7 7 7 F F F 23 460-479 18 18 18 30 30 30 F F F 24 480-499 18 18 18 28 28 28 F F F 25 500-519 18 18 18 29 29 29 F F F 26 520-539 18 18 18 29 29 29 F F F 27 540-559 13 13 13 18 18 18 F F F 28 560-579 12 12 12 17 17 17 T T T 29 580-599 6 6 6 7 7 7 T T T Max gen: 18, Max flush: 30 A Difference in extent metadata is expected here On loop 9 Verifying data now No change after verify Loop: 9 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 17 17 17 27 27 27 F F F 1 020-039 17 17 17 27 27 27 F F F 2 040-059 19 19 19 31 31 31 F F F 3 060-079 16 16 16 24 24 24 F F F 4 080-099 12 12 12 17 17 17 F F F 5 100-119 18 18 18 28 28 28 F F F 6 120-139 17 17 17 27 27 27 F F F 7 140-159 16 16 16 26 26 26 F F F 8 160-179 14 14 14 22 22 22 F F F 9 180-199 17 17 17 27 27 27 F F F 10 200-219 16 16 16 24 24 24 F F F 11 220-239 18 18 18 30 30 30 F F F 12 240-259 13 13 13 18 18 18 F F F 13 260-279 13 13 13 18 18 18 F F F 14 280-299 12 12 12 17 17 17 F F F 15 300-319 16 16 16 26 26 26 F F F 16 320-339 19 19 19 31 31 31 F F F 17 340-359 19 19 19 31 31 31 F F F 18 360-379 18 18 18 30 30 30 F F F 19 380-399 7 7 7 8 8 8 F F F 20 400-419 10 10 10 15 15 15 F F F 21 420-439 18 18 18 28 28 28 F F F 22 440-459 6 6 6 7 7 7 F F F 23 460-479 18 18 18 30 30 30 F F F 24 480-499 18 18 18 28 28 28 F F F 25 500-519 18 18 18 29 29 29 F F F 26 520-539 18 18 18 29 29 29 F F F 27 540-559 13 13 13 18 18 18 F F F 28 560-579 19 19 19 31 31 31 F F F 29 580-599 19 19 19 31 31 31 F F F Max gen: 19, Max flush: 31 Begin loop 10 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:41.397348612Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1262,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:41.441952231Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1262,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:41.442637632Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1262} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"abd5430f1508af0bc008143ea2566277ef714ec2\",\n git_commit_timestamp: \"2025-01-17T20:55:03.000000000Z\",\n git_branch: \"alan/tokio-to-140\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:41.442689082Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1262} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:41.442710044Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1262} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:41.442732928Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1262} {"msg":"Crucible 3884ed2b-f379-4487-9dac-cad67ab0dbf3 has session id: 3ba60221-d4a1-4f7e-aec0-b728335fcc58","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:41.443246908Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1262,"session_id":"3ba60221-d4a1-4f7e-aec0-b728335fcc58"} {"msg":"Upstairs opts: Upstairs UUID: 3884ed2b-f379-4487-9dac-cad67ab0dbf3, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:41.443297237Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1262,"session_id":"3ba60221-d4a1-4f7e-aec0-b728335fcc58"} {"msg":"Crucible stats registered with UUID: 3884ed2b-f379-4487-9dac-cad67ab0dbf3","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:41.443331588Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1262,"session_id":"3ba60221-d4a1-4f7e-aec0-b728335fcc58"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:11:41.443355812Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1262,"session_id":"3ba60221-d4a1-4f7e-aec0-b728335fcc58"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:41.443799581Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1262,"task":"crutest"} {"msg":"The guest has requested activation with gen:20","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:41.443824656Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1262} {"msg":"3884ed2b-f379-4487-9dac-cad67ab0dbf3 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:41.443903862Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1262,"session_id":"3ba60221-d4a1-4f7e-aec0-b728335fcc58"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:41.443921813Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1262,"client":"0","":"downstairs","session_id":"3ba60221-d4a1-4f7e-aec0-b728335fcc58"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:41.443945516Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1262,"client":"0","":"downstairs","session_id":"3ba60221-d4a1-4f7e-aec0-b728335fcc58"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:41.443959415Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1262,"client":"1","":"downstairs","session_id":"3ba60221-d4a1-4f7e-aec0-b728335fcc58"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:41.443973533Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1262,"client":"1","":"downstairs","session_id":"3ba60221-d4a1-4f7e-aec0-b728335fcc58"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:41.443988892Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1262,"client":"2","":"downstairs","session_id":"3ba60221-d4a1-4f7e-aec0-b728335fcc58"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:41.444013766Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1262,"client":"2","":"downstairs","session_id":"3ba60221-d4a1-4f7e-aec0-b728335fcc58"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:41.444038861Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1262,"":"io task","client":"2","":"downstairs","session_id":"3ba60221-d4a1-4f7e-aec0-b728335fcc58"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:41.444066667Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1262,"":"io task","client":"0","":"downstairs","session_id":"3ba60221-d4a1-4f7e-aec0-b728335fcc58"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:41.444095384Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1262,"":"io task","client":"1","":"downstairs","session_id":"3ba60221-d4a1-4f7e-aec0-b728335fcc58"} {"msg":"ds_connection connected from Ok(127.0.0.1:32907)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:41.444142531Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1262,"":"io task","client":"1","":"downstairs","session_id":"3ba60221-d4a1-4f7e-aec0-b728335fcc58"} {"msg":"ds_connection connected from Ok(127.0.0.1:40428)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:41.444171719Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1262,"":"io task","client":"0","":"downstairs","session_id":"3ba60221-d4a1-4f7e-aec0-b728335fcc58"} {"msg":"ds_connection connected from Ok(127.0.0.1:62082)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:41.444356416Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1262,"":"io task","client":"2","":"downstairs","session_id":"3ba60221-d4a1-4f7e-aec0-b728335fcc58"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:41.444865604Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1262,"client":"1","":"downstairs","session_id":"3ba60221-d4a1-4f7e-aec0-b728335fcc58"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:41.444893481Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1262,"client":"0","":"downstairs","session_id":"3ba60221-d4a1-4f7e-aec0-b728335fcc58"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:41.445401397Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1262,"client":"2","":"downstairs","session_id":"3ba60221-d4a1-4f7e-aec0-b728335fcc58"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:41.445538177Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1262,"client":"1","":"downstairs","session_id":"3ba60221-d4a1-4f7e-aec0-b728335fcc58"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:41.445566824Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1262,"client":"0","":"downstairs","session_id":"3ba60221-d4a1-4f7e-aec0-b728335fcc58"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:41.44607262Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1262,"session_id":"3ba60221-d4a1-4f7e-aec0-b728335fcc58"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:41.446110652Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1262,"client":"2","":"downstairs","session_id":"3ba60221-d4a1-4f7e-aec0-b728335fcc58"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:41.446140599Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1262,"session_id":"3ba60221-d4a1-4f7e-aec0-b728335fcc58"} {"msg":"[0]R flush_numbers[0..12]: [27, 27, 31, 24, 17, 28, 27, 26, 22, 27, 24, 30]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:41.446574723Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1262,"":"downstairs","":"downstairs","session_id":"3ba60221-d4a1-4f7e-aec0-b728335fcc58"} {"msg":"[0]R generation[0..12]: [17, 17, 19, 16, 12, 18, 17, 16, 14, 17, 16, 18]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:41.446594575Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1262,"":"downstairs","":"downstairs","session_id":"3ba60221-d4a1-4f7e-aec0-b728335fcc58"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:41.446609764Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1262,"":"downstairs","":"downstairs","session_id":"3ba60221-d4a1-4f7e-aec0-b728335fcc58"} {"msg":"[1]R flush_numbers[0..12]: [27, 27, 31, 24, 17, 28, 27, 26, 22, 27, 24, 30]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:41.446623201Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1262,"":"downstairs","":"downstairs","session_id":"3ba60221-d4a1-4f7e-aec0-b728335fcc58"} {"msg":"[1]R generation[0..12]: [17, 17, 19, 16, 12, 18, 17, 16, 14, 17, 16, 18]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:41.446636409Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1262,"":"downstairs","":"downstairs","session_id":"3ba60221-d4a1-4f7e-aec0-b728335fcc58"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:41.446653499Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1262,"":"downstairs","":"downstairs","session_id":"3ba60221-d4a1-4f7e-aec0-b728335fcc58"} {"msg":"[2]R flush_numbers[0..12]: [27, 27, 31, 24, 17, 28, 27, 26, 22, 27, 24, 30]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:41.446667387Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1262,"":"downstairs","":"downstairs","session_id":"3ba60221-d4a1-4f7e-aec0-b728335fcc58"} {"msg":"[2]R generation[0..12]: [17, 17, 19, 16, 12, 18, 17, 16, 14, 17, 16, 18]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:41.446680625Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1262,"":"downstairs","":"downstairs","session_id":"3ba60221-d4a1-4f7e-aec0-b728335fcc58"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:41.446694063Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1262,"":"downstairs","":"downstairs","session_id":"3ba60221-d4a1-4f7e-aec0-b728335fcc58"} {"msg":"Max found gen is 20","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:41.44670666Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1262,"":"downstairs","":"downstairs","session_id":"3ba60221-d4a1-4f7e-aec0-b728335fcc58"} {"msg":"Generation requested: 20 >= found:20","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:41.446719548Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1262,"":"downstairs","":"downstairs","session_id":"3ba60221-d4a1-4f7e-aec0-b728335fcc58"} {"msg":"Next flush: 32","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-17T21:11:41.446732075Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1262,"":"downstairs","":"downstairs","session_id":"3ba60221-d4a1-4f7e-aec0-b728335fcc58"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:41.44675825Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1262,"":"downstairs","":"downstairs","session_id":"3ba60221-d4a1-4f7e-aec0-b728335fcc58"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:41.446783905Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1262,"session_id":"3ba60221-d4a1-4f7e-aec0-b728335fcc58"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:41.44680826Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1262,"client":"0","":"downstairs","session_id":"3ba60221-d4a1-4f7e-aec0-b728335fcc58"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:41.446833644Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1262,"client":"1","":"downstairs","session_id":"3ba60221-d4a1-4f7e-aec0-b728335fcc58"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:41.44686025Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1262,"client":"2","":"downstairs","session_id":"3ba60221-d4a1-4f7e-aec0-b728335fcc58"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:41.446887276Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1262,"session_id":"3ba60221-d4a1-4f7e-aec0-b728335fcc58"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:41.44691137Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1262,"session_id":"3ba60221-d4a1-4f7e-aec0-b728335fcc58"} {"msg":"3884ed2b-f379-4487-9dac-cad67ab0dbf3 is now active with session: 3ba60221-d4a1-4f7e-aec0-b728335fcc58","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:41.446937185Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1262,"session_id":"3ba60221-d4a1-4f7e-aec0-b728335fcc58"} {"msg":"Set Active after no reconciliation"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:41.446968903Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1262,"session_id":"3ba60221-d4a1-4f7e-aec0-b728335fcc58"} {"msg":"The guest has finished waiting for activation with:20","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:41.446994148Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1262} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 310 len 4096 data: 3 02/30 Read block 472 len 20480 03/30 Write block 109 len 28672 data: 3 3 3 4 3 4 4 04/30 Read block 315 len 12288 05/30 Write block 295 len 28672 data: 2 2 2 2 2 2 2 06/30 Read block 555 len 8192 07/30 Write block 472 len 8192 data: 4 2 08/30 Write block 51 len 28672 data: 4 5 5 5 5 5 5 09/30 Write block 504 len 28672 data: 4 4 4 4 4 4 4 10/30 Write block 120 len 16384 data: 4 3 3 3 11/30 Write block 403 len 20480 data: 4 4 4 4 3 12/30 Read block 226 len 40960 13/30 Write block 542 len 28672 data: 3 3 3 3 4 4 4 14/30 Read block 230 len 16384 15/30 Read block 335 len 32768 16/30 Write block 151 len 8192 data: 3 2 17/30 Write block 500 len 32768 data: 2 2 2 3 5 5 5 5 18/30 Read block 233 len 32768 19/30 Read block 231 len 36864 20/30 Write block 444 len 32768 data: 2 2 2 2 2 3 3 3 21/30 Read block 579 len 8192 22/30 Read block 196 len 16384 23/30 Read block 528 len 40960 24/30 Write block 458 len 4096 data: 3 25/30 Read block 522 len 8192 26/30 Read block 427 len 28672 27/30 Read block 366 len 32768 28/30 Read block 485 len 24576 29/30 Write block 193 len 12288 data: 6 6 4 30/30 Write block 244 len 36864 data: 4 4 4 4 4 3 3 2 2 ---------------------------------------------------------------- Crucible gen:20 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Done Done Done false 1001 Acked Read 5 Done Done Done false 1002 Acked Write 7 Done Done Done false 1003 Acked Read 3 Done Done Done false 1004 Acked Write 7 Done Done Done false 1005 Acked Read 2 Done Done Done false 1006 Acked Write 2 Done Done Done false 1007 Acked Write 7 Done Done Done false 1008 Acked Write 7 Done Done Done false 1009 Acked Write 4 Done Done Done false 1010 Acked Write 5 Done Done Done false 1011 Acked Read 10 Done Done Done false 1012 Acked Write 7 Done Done Done false 1013 Acked Read 4 Done Done Done false 1014 Acked Read 8 Done Done Done false 1015 Acked Write 2 Done Done Done false 1016 Acked Write 8 Done Done Done false 1017 Acked Read 8 Done Done Done false 1018 Acked Read 9 Done Done Done false 1019 Acked Write 8 Done Done Done false 1020 Acked Read 2 Done Done Done false 1021 Acked Read 4 Done Done Done false 1022 Acked Read 10 Done Done Done false 1023 Acked Write 1 Done Done Done false 1024 Acked Read 2 Done Done Done false 1025 Acked Read 7 Done Done Done false 1026 Acked Read 8 Done Done Done false 1027 Acked Read 6 Done Done Done false 1028 Acked Write 3 Done Done Done false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 29 29 29 87 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 0 0 0 Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:11:41.466247882Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1262} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 17 17 17 27 27 27 F F F 1 020-039 17 17 17 27 27 27 F F F 2 040-059 19 19 19 31 31 31 T T T 3 060-079 16 16 16 24 24 24 F F F 4 080-099 12 12 12 17 17 17 F F F 5 100-119 18 18 18 28 28 28 T T T 6 120-139 17 17 17 27 27 27 T T T 7 140-159 16 16 16 26 26 26 T T T 8 160-179 14 14 14 22 22 22 F F F 9 180-199 17 17 17 27 27 27 T T T 10 200-219 16 16 16 24 24 24 F F F 11 220-239 18 18 18 30 30 30 F F F 12 240-259 13 13 13 18 18 18 T T T 13 260-279 13 13 13 18 18 18 F F F 14 280-299 12 12 12 17 17 17 T T T 15 300-319 16 16 16 26 26 26 T T T 16 320-339 19 19 19 31 31 31 F F F 17 340-359 19 19 19 31 31 31 F F F 18 360-379 18 18 18 30 30 30 F F F 19 380-399 7 7 7 8 8 8 F F F 20 400-419 10 10 10 15 15 15 T T T 21 420-439 18 18 18 28 28 28 F F F 22 440-459 6 6 6 7 7 7 T T T 23 460-479 18 18 18 30 30 30 T T T 24 480-499 18 18 18 28 28 28 F F F 25 500-519 18 18 18 29 29 29 T T T 26 520-539 18 18 18 29 29 29 F F F 27 540-559 13 13 13 18 18 18 T T T 28 560-579 19 19 19 31 31 31 F F F 29 580-599 19 19 19 31 31 31 F F F Max gen: 19, Max flush: 31 A Difference in extent metadata is expected here On loop 10 Verifying data now No change after verify Loop: 10 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 17 17 17 27 27 27 F F F 1 020-039 17 17 17 27 27 27 F F F 2 040-059 20 20 20 32 32 32 F F F 3 060-079 16 16 16 24 24 24 F F F 4 080-099 12 12 12 17 17 17 F F F 5 100-119 20 20 20 32 32 32 F F F 6 120-139 20 20 20 32 32 32 F F F 7 140-159 20 20 20 32 32 32 F F F 8 160-179 14 14 14 22 22 22 F F F 9 180-199 20 20 20 32 32 32 F F F 10 200-219 16 16 16 24 24 24 F F F 11 220-239 18 18 18 30 30 30 F F F 12 240-259 20 20 20 32 32 32 F F F 13 260-279 13 13 13 18 18 18 F F F 14 280-299 20 20 20 32 32 32 F F F 15 300-319 20 20 20 32 32 32 F F F 16 320-339 19 19 19 31 31 31 F F F 17 340-359 19 19 19 31 31 31 F F F 18 360-379 18 18 18 30 30 30 F F F 19 380-399 7 7 7 8 8 8 F F F 20 400-419 20 20 20 32 32 32 F F F 21 420-439 18 18 18 28 28 28 F F F 22 440-459 20 20 20 32 32 32 F F F 23 460-479 20 20 20 32 32 32 F F F 24 480-499 18 18 18 28 28 28 F F F 25 500-519 20 20 20 32 32 32 F F F 26 520-539 18 18 18 29 29 29 F F F 27 540-559 20 20 20 32 32 32 F F F 28 560-579 19 19 19 31 31 31 F F F 29 580-599 19 19 19 31 31 31 F F F Max gen: 20, Max flush: 32 Begin loop 11 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:45.753323684Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1270,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:11:45.796324787Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1270,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:45.796738609Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1270,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:45.828472116Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1270,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:45.829055087Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1270} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"abd5430f1508af0bc008143ea2566277ef714ec2\",\n git_commit_timestamp: \"2025-01-17T20:55:03.000000000Z\",\n git_branch: \"alan/tokio-to-140\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:45.829092519Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1270} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:45.829105607Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1270} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:45.829119635Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1270} {"msg":"Crucible 842ee847-f60a-4f21-bbae-41176f03b198 has session id: 65c84613-bed8-4413-8771-ce7d7e9b4e50","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:45.829617196Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1270,"session_id":"65c84613-bed8-4413-8771-ce7d7e9b4e50"} {"msg":"Upstairs opts: Upstairs UUID: 842ee847-f60a-4f21-bbae-41176f03b198, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:45.829657829Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1270,"session_id":"65c84613-bed8-4413-8771-ce7d7e9b4e50"} {"msg":"Crucible stats registered with UUID: 842ee847-f60a-4f21-bbae-41176f03b198","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:45.829681773Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1270,"session_id":"65c84613-bed8-4413-8771-ce7d7e9b4e50"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:11:45.829695972Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1270,"session_id":"65c84613-bed8-4413-8771-ce7d7e9b4e50"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:45.830183136Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1270,"task":"crutest"} {"msg":"The guest has requested activation with gen:22","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:45.830209101Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1270} {"msg":"842ee847-f60a-4f21-bbae-41176f03b198 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:45.830280694Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1270,"session_id":"65c84613-bed8-4413-8771-ce7d7e9b4e50"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:45.830302306Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1270,"client":"0","":"downstairs","session_id":"65c84613-bed8-4413-8771-ce7d7e9b4e50"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:45.830327871Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1270,"client":"0","":"downstairs","session_id":"65c84613-bed8-4413-8771-ce7d7e9b4e50"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:45.830341829Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1270,"client":"1","":"downstairs","session_id":"65c84613-bed8-4413-8771-ce7d7e9b4e50"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:45.830355437Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1270,"client":"1","":"downstairs","session_id":"65c84613-bed8-4413-8771-ce7d7e9b4e50"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:45.830367994Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1270,"client":"2","":"downstairs","session_id":"65c84613-bed8-4413-8771-ce7d7e9b4e50"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:45.830389517Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1270,"client":"2","":"downstairs","session_id":"65c84613-bed8-4413-8771-ce7d7e9b4e50"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:45.830413651Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1270,"":"io task","client":"2","":"downstairs","session_id":"65c84613-bed8-4413-8771-ce7d7e9b4e50"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:45.830438065Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1270,"":"io task","client":"0","":"downstairs","session_id":"65c84613-bed8-4413-8771-ce7d7e9b4e50"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:45.830452974Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1270,"":"io task","client":"1","":"downstairs","session_id":"65c84613-bed8-4413-8771-ce7d7e9b4e50"} {"msg":"ds_connection connected from Ok(127.0.0.1:37949)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:45.830466552Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1270,"":"io task","client":"2","":"downstairs","session_id":"65c84613-bed8-4413-8771-ce7d7e9b4e50"} {"msg":"ds_connection connected from Ok(127.0.0.1:55476)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:45.83048008Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1270,"":"io task","client":"0","":"downstairs","session_id":"65c84613-bed8-4413-8771-ce7d7e9b4e50"} {"msg":"ds_connection connected from Ok(127.0.0.1:56037)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:45.830523736Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1270,"":"io task","client":"1","":"downstairs","session_id":"65c84613-bed8-4413-8771-ce7d7e9b4e50"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:45.831173565Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1270,"client":"2","":"downstairs","session_id":"65c84613-bed8-4413-8771-ce7d7e9b4e50"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:45.831202742Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1270,"client":"1","":"downstairs","session_id":"65c84613-bed8-4413-8771-ce7d7e9b4e50"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:45.83135315Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1270,"client":"0","":"downstairs","session_id":"65c84613-bed8-4413-8771-ce7d7e9b4e50"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:45.831753694Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1270,"client":"1","":"downstairs","session_id":"65c84613-bed8-4413-8771-ce7d7e9b4e50"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:45.831919391Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1270,"client":"2","":"downstairs","session_id":"65c84613-bed8-4413-8771-ce7d7e9b4e50"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:45.832197884Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1270,"client":"0","":"downstairs","session_id":"65c84613-bed8-4413-8771-ce7d7e9b4e50"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:45.832352794Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1270,"session_id":"65c84613-bed8-4413-8771-ce7d7e9b4e50"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:45.832426938Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1270,"session_id":"65c84613-bed8-4413-8771-ce7d7e9b4e50"} {"msg":"[0]R flush_numbers[0..12]: [27, 27, 32, 24, 17, 32, 32, 32, 22, 32, 24, 30]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:45.83270475Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1270,"":"downstairs","":"downstairs","session_id":"65c84613-bed8-4413-8771-ce7d7e9b4e50"} {"msg":"[0]R generation[0..12]: [17, 17, 20, 16, 12, 20, 20, 20, 14, 20, 16, 18]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:45.832724352Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1270,"":"downstairs","":"downstairs","session_id":"65c84613-bed8-4413-8771-ce7d7e9b4e50"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:45.832739Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1270,"":"downstairs","":"downstairs","session_id":"65c84613-bed8-4413-8771-ce7d7e9b4e50"} {"msg":"[1]R flush_numbers[0..12]: [27, 27, 32, 24, 17, 32, 32, 32, 22, 32, 24, 30]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:45.832752588Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1270,"":"downstairs","":"downstairs","session_id":"65c84613-bed8-4413-8771-ce7d7e9b4e50"} {"msg":"[1]R generation[0..12]: [17, 17, 20, 16, 12, 20, 20, 20, 14, 20, 16, 18]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:45.832765696Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1270,"":"downstairs","":"downstairs","session_id":"65c84613-bed8-4413-8771-ce7d7e9b4e50"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:45.832779224Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1270,"":"downstairs","":"downstairs","session_id":"65c84613-bed8-4413-8771-ce7d7e9b4e50"} {"msg":"[2]R flush_numbers[0..12]: [27, 27, 32, 24, 17, 32, 32, 32, 22, 32, 24, 30]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:45.832792402Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1270,"":"downstairs","":"downstairs","session_id":"65c84613-bed8-4413-8771-ce7d7e9b4e50"} {"msg":"[2]R generation[0..12]: [17, 17, 20, 16, 12, 20, 20, 20, 14, 20, 16, 18]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:45.832805529Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1270,"":"downstairs","":"downstairs","session_id":"65c84613-bed8-4413-8771-ce7d7e9b4e50"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:45.832822999Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1270,"":"downstairs","":"downstairs","session_id":"65c84613-bed8-4413-8771-ce7d7e9b4e50"} {"msg":"Max found gen is 21","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:45.832836857Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1270,"":"downstairs","":"downstairs","session_id":"65c84613-bed8-4413-8771-ce7d7e9b4e50"} {"msg":"Generation requested: 22 >= found:21","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:45.832849645Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1270,"":"downstairs","":"downstairs","session_id":"65c84613-bed8-4413-8771-ce7d7e9b4e50"} {"msg":"Next flush: 33","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-17T21:11:45.832862032Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1270,"":"downstairs","":"downstairs","session_id":"65c84613-bed8-4413-8771-ce7d7e9b4e50"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:45.832877771Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1270,"":"downstairs","":"downstairs","session_id":"65c84613-bed8-4413-8771-ce7d7e9b4e50"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:45.832890429Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1270,"session_id":"65c84613-bed8-4413-8771-ce7d7e9b4e50"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:45.832902015Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1270,"client":"0","":"downstairs","session_id":"65c84613-bed8-4413-8771-ce7d7e9b4e50"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:45.832914613Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1270,"client":"1","":"downstairs","session_id":"65c84613-bed8-4413-8771-ce7d7e9b4e50"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:45.832928121Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1270,"client":"2","":"downstairs","session_id":"65c84613-bed8-4413-8771-ce7d7e9b4e50"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:45.832941969Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1270,"session_id":"65c84613-bed8-4413-8771-ce7d7e9b4e50"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:45.832954746Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1270,"session_id":"65c84613-bed8-4413-8771-ce7d7e9b4e50"} {"msg":"842ee847-f60a-4f21-bbae-41176f03b198 is now active with session: 65c84613-bed8-4413-8771-ce7d7e9b4e50","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:45.832967013Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1270,"session_id":"65c84613-bed8-4413-8771-ce7d7e9b4e50"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:45.83297857Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1270,"session_id":"65c84613-bed8-4413-8771-ce7d7e9b4e50"} {"msg":"The guest has finished waiting for activation with:22","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:45.832990737Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1270} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 51 len 32768 data: 5 6 6 6 6 6 6 5 02/30 Read block 273 len 40960 03/30 Read block 362 len 20480 04/30 Flush 05/30 Write block 26 len 40960 data: 3 3 3 3 3 3 4 3 3 3 06/30 Write block 98 len 32768 data: 2 2 2 2 2 2 2 2 07/30 Read block 257 len 16384 08/30 Write block 340 len 4096 data: 3 09/30 Write block 10 len 28672 data: 3 4 5 4 4 3 4 10/30 Write block 235 len 12288 data: 3 3 3 11/30 Read block 226 len 36864 12/30 Read block 447 len 12288 13/30 Read block 495 len 20480 14/30 Read block 223 len 12288 15/30 Read block 243 len 28672 16/30 Read block 411 len 24576 17/30 Write block 325 len 12288 data: 2 2 2 18/30 Read block 504 len 16384 19/30 Write block 396 len 24576 data: 3 3 3 4 4 3 20/30 Read block 217 len 12288 21/30 Write block 372 len 40960 data: 4 4 5 5 5 7 7 6 5 4 22/30 Write block 245 len 32768 data: 5 5 5 5 4 4 3 3 23/30 Read block 275 len 4096 24/30 Read block 123 len 16384 25/30 Write block 89 len 12288 data: 2 3 3 26/30 Read block 238 len 20480 27/30 Read block 546 len 12288 28/30 Read block 449 len 12288 29/30 Write block 168 len 40960 data: 3 3 3 2 3 3 3 4 4 4 30/30 Read block 566 len 12288 ---------------------------------------------------------------- Crucible gen:22 GIO:true work queues: Upstairs:0 downstairs:26 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1004 Acked Write 10 Done Done Done false 1005 Acked Write 8 Done Done Done false 1006 Acked Read 4 Done Done Done false 1007 Acked Write 1 Done Done Done false 1008 Acked Write 7 Done Done Done false 1009 Acked Write 3 Done Done Done false 1010 Acked Read 9 Done Done Done false 1011 Acked Read 3 Done Done Done false 1012 Acked Read 5 Done Done Done false 1013 Acked Read 3 Done Done Done false 1014 Acked Read 7 Done Done Done false 1015 Acked Read 6 Done Done Done false 1016 Acked Write 3 Done Done Done false 1017 Acked Read 4 Done Done Done false 1018 Acked Write 6 Done Done Done false 1019 Acked Read 3 Done Done Done false 1020 Acked Write 10 Done Done Done false 1021 Acked Write 8 Done Done Done false 1022 Acked Read 1 Done Done Done false 1023 Acked Read 4 Done Done Done false 1024 Acked Write 3 Done Done Done false 1025 Acked Read 5 Done Done Done false 1026 Acked Read 3 Done Done Done false 1027 Acked Read 3 Done Done Done false 1028 Acked Write 10 Done Done Done false 1029 Acked Read 3 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 26 26 26 78 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1003 1003 1003 Downstairs last five completed: 1003 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:11:45.853264717Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1270} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 17 17 17 27 27 27 T T T 1 020-039 17 17 17 27 27 27 T T T 2 040-059 22 22 22 33 33 33 F F F 3 060-079 16 16 16 24 24 24 F F F 4 080-099 12 12 12 17 17 17 T T T 5 100-119 20 20 20 32 32 32 T T T 6 120-139 20 20 20 32 32 32 F F F 7 140-159 20 20 20 32 32 32 F F F 8 160-179 14 14 14 22 22 22 T T T 9 180-199 20 20 20 32 32 32 F F F 10 200-219 16 16 16 24 24 24 F F F 11 220-239 18 18 18 30 30 30 T T T 12 240-259 20 20 20 32 32 32 T T T 13 260-279 13 13 13 18 18 18 F F F 14 280-299 20 20 20 32 32 32 F F F 15 300-319 20 20 20 32 32 32 F F F 16 320-339 19 19 19 31 31 31 T T T 17 340-359 19 19 19 31 31 31 T T T 18 360-379 18 18 18 30 30 30 T T T 19 380-399 7 7 7 8 8 8 T T T 20 400-419 20 20 20 32 32 32 T T T 21 420-439 18 18 18 28 28 28 F F F 22 440-459 20 20 20 32 32 32 F F F 23 460-479 20 20 20 32 32 32 F F F 24 480-499 18 18 18 28 28 28 F F F 25 500-519 20 20 20 32 32 32 F F F 26 520-539 18 18 18 29 29 29 F F F 27 540-559 20 20 20 32 32 32 F F F 28 560-579 19 19 19 31 31 31 F F F 29 580-599 19 19 19 31 31 31 F F F Max gen: 22, Max flush: 33 A Difference in extent metadata is expected here On loop 11 Verifying data now No change after verify Loop: 11 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 23 23 23 34 34 34 F F F 1 020-039 23 23 23 34 34 34 F F F 2 040-059 22 22 22 33 33 33 F F F 3 060-079 16 16 16 24 24 24 F F F 4 080-099 23 23 23 34 34 34 F F F 5 100-119 23 23 23 34 34 34 F F F 6 120-139 20 20 20 32 32 32 F F F 7 140-159 20 20 20 32 32 32 F F F 8 160-179 23 23 23 34 34 34 F F F 9 180-199 20 20 20 32 32 32 F F F 10 200-219 16 16 16 24 24 24 F F F 11 220-239 23 23 23 34 34 34 F F F 12 240-259 23 23 23 34 34 34 F F F 13 260-279 13 13 13 18 18 18 F F F 14 280-299 20 20 20 32 32 32 F F F 15 300-319 20 20 20 32 32 32 F F F 16 320-339 23 23 23 34 34 34 F F F 17 340-359 23 23 23 34 34 34 F F F 18 360-379 23 23 23 34 34 34 F F F 19 380-399 23 23 23 34 34 34 F F F 20 400-419 23 23 23 34 34 34 F F F 21 420-439 18 18 18 28 28 28 F F F 22 440-459 20 20 20 32 32 32 F F F 23 460-479 20 20 20 32 32 32 F F F 24 480-499 18 18 18 28 28 28 F F F 25 500-519 20 20 20 32 32 32 F F F 26 520-539 18 18 18 29 29 29 F F F 27 540-559 20 20 20 32 32 32 F F F 28 560-579 19 19 19 31 31 31 F F F 29 580-599 19 19 19 31 31 31 F F F Max gen: 23, Max flush: 34 Begin loop 12 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:50.160022274Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1278,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:11:50.202768417Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1278,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:50.203224064Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1278,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:50.234809384Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1278,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:50.235404572Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1278} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"abd5430f1508af0bc008143ea2566277ef714ec2\",\n git_commit_timestamp: \"2025-01-17T20:55:03.000000000Z\",\n git_branch: \"alan/tokio-to-140\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:50.235438983Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1278} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:50.235456033Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1278} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:50.235470691Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1278} {"msg":"Crucible 736eb367-488a-44d5-adda-a2a3d3f47482 has session id: 2999dff5-85b3-443a-8027-415d24917db8","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:50.236132078Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1278,"session_id":"2999dff5-85b3-443a-8027-415d24917db8"} {"msg":"Upstairs opts: Upstairs UUID: 736eb367-488a-44d5-adda-a2a3d3f47482, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:50.236165417Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1278,"session_id":"2999dff5-85b3-443a-8027-415d24917db8"} {"msg":"Crucible stats registered with UUID: 736eb367-488a-44d5-adda-a2a3d3f47482","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:50.236182727Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1278,"session_id":"2999dff5-85b3-443a-8027-415d24917db8"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:11:50.236198076Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1278,"session_id":"2999dff5-85b3-443a-8027-415d24917db8"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:50.236758394Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1278,"task":"crutest"} {"msg":"The guest has requested activation with gen:24","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:50.236780287Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1278} {"msg":"736eb367-488a-44d5-adda-a2a3d3f47482 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:50.236853179Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1278,"session_id":"2999dff5-85b3-443a-8027-415d24917db8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:50.236869919Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1278,"client":"0","":"downstairs","session_id":"2999dff5-85b3-443a-8027-415d24917db8"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:50.236900837Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1278,"client":"0","":"downstairs","session_id":"2999dff5-85b3-443a-8027-415d24917db8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:50.236926952Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1278,"client":"1","":"downstairs","session_id":"2999dff5-85b3-443a-8027-415d24917db8"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:50.236952697Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1278,"client":"1","":"downstairs","session_id":"2999dff5-85b3-443a-8027-415d24917db8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:50.236976311Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1278,"client":"2","":"downstairs","session_id":"2999dff5-85b3-443a-8027-415d24917db8"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:50.23700811Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1278,"client":"2","":"downstairs","session_id":"2999dff5-85b3-443a-8027-415d24917db8"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:50.237035726Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1278,"":"io task","client":"2","":"downstairs","session_id":"2999dff5-85b3-443a-8027-415d24917db8"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:50.237062111Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1278,"":"io task","client":"0","":"downstairs","session_id":"2999dff5-85b3-443a-8027-415d24917db8"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:50.237089617Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1278,"":"io task","client":"1","":"downstairs","session_id":"2999dff5-85b3-443a-8027-415d24917db8"} {"msg":"ds_connection connected from Ok(127.0.0.1:36592)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:50.237114862Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1278,"":"io task","client":"0","":"downstairs","session_id":"2999dff5-85b3-443a-8027-415d24917db8"} {"msg":"ds_connection connected from Ok(127.0.0.1:45465)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:50.237140507Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1278,"":"io task","client":"1","":"downstairs","session_id":"2999dff5-85b3-443a-8027-415d24917db8"} {"msg":"ds_connection connected from Ok(127.0.0.1:37747)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:50.237166182Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1278,"":"io task","client":"2","":"downstairs","session_id":"2999dff5-85b3-443a-8027-415d24917db8"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:50.237825217Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1278,"client":"2","":"downstairs","session_id":"2999dff5-85b3-443a-8027-415d24917db8"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:50.237858757Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1278,"client":"1","":"downstairs","session_id":"2999dff5-85b3-443a-8027-415d24917db8"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:50.237874156Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1278,"client":"0","":"downstairs","session_id":"2999dff5-85b3-443a-8027-415d24917db8"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:50.238537173Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1278,"client":"1","":"downstairs","session_id":"2999dff5-85b3-443a-8027-415d24917db8"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:50.23856628Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1278,"client":"0","":"downstairs","session_id":"2999dff5-85b3-443a-8027-415d24917db8"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:50.23858284Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1278,"client":"2","":"downstairs","session_id":"2999dff5-85b3-443a-8027-415d24917db8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:50.23913054Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1278,"session_id":"2999dff5-85b3-443a-8027-415d24917db8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:50.239158566Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1278,"session_id":"2999dff5-85b3-443a-8027-415d24917db8"} {"msg":"[0]R flush_numbers[0..12]: [34, 34, 33, 24, 34, 34, 32, 32, 34, 32, 24, 34]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:50.239188324Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1278,"":"downstairs","":"downstairs","session_id":"2999dff5-85b3-443a-8027-415d24917db8"} {"msg":"[0]R generation[0..12]: [23, 23, 22, 16, 23, 23, 20, 20, 23, 20, 16, 23]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:50.239203633Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1278,"":"downstairs","":"downstairs","session_id":"2999dff5-85b3-443a-8027-415d24917db8"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:50.239217851Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1278,"":"downstairs","":"downstairs","session_id":"2999dff5-85b3-443a-8027-415d24917db8"} {"msg":"[1]R flush_numbers[0..12]: [34, 34, 33, 24, 34, 34, 32, 32, 34, 32, 24, 34]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:50.239231009Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1278,"":"downstairs","":"downstairs","session_id":"2999dff5-85b3-443a-8027-415d24917db8"} {"msg":"[1]R generation[0..12]: [23, 23, 22, 16, 23, 23, 20, 20, 23, 20, 16, 23]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:50.239244116Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1278,"":"downstairs","":"downstairs","session_id":"2999dff5-85b3-443a-8027-415d24917db8"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:50.239257635Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1278,"":"downstairs","":"downstairs","session_id":"2999dff5-85b3-443a-8027-415d24917db8"} {"msg":"[2]R flush_numbers[0..12]: [34, 34, 33, 24, 34, 34, 32, 32, 34, 32, 24, 34]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:50.239270782Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1278,"":"downstairs","":"downstairs","session_id":"2999dff5-85b3-443a-8027-415d24917db8"} {"msg":"[2]R generation[0..12]: [23, 23, 22, 16, 23, 23, 20, 20, 23, 20, 16, 23]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:50.23928379Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1278,"":"downstairs","":"downstairs","session_id":"2999dff5-85b3-443a-8027-415d24917db8"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:50.239297158Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1278,"":"downstairs","":"downstairs","session_id":"2999dff5-85b3-443a-8027-415d24917db8"} {"msg":"Max found gen is 24","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:50.239309565Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1278,"":"downstairs","":"downstairs","session_id":"2999dff5-85b3-443a-8027-415d24917db8"} {"msg":"Generation requested: 24 >= found:24","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:50.239322452Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1278,"":"downstairs","":"downstairs","session_id":"2999dff5-85b3-443a-8027-415d24917db8"} {"msg":"Next flush: 35","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:50.23933499Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1278,"":"downstairs","":"downstairs","session_id":"2999dff5-85b3-443a-8027-415d24917db8"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:50.239347597Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1278,"":"downstairs","":"downstairs","session_id":"2999dff5-85b3-443a-8027-415d24917db8"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:50.239360384Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1278,"session_id":"2999dff5-85b3-443a-8027-415d24917db8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:50.239373812Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1278,"client":"0","":"downstairs","session_id":"2999dff5-85b3-443a-8027-415d24917db8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:50.23938686Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1278,"client":"1","":"downstairs","session_id":"2999dff5-85b3-443a-8027-415d24917db8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"Wait for a query_work_queue command to finish before sending IO time":"2025-01-17T21:11:50.239399527Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1278,"client":"2","":"downstairs","session_id":"2999dff5-85b3-443a-8027-415d24917db8"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:50.23942093Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1278,"session_id":"2999dff5-85b3-443a-8027-415d24917db8"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:50.239432817Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1278,"session_id":"2999dff5-85b3-443a-8027-415d24917db8"} {"msg":"736eb367-488a-44d5-adda-a2a3d3f47482 is now active with session: 2999dff5-85b3-443a-8027-415d24917db8","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:50.239445184Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1278,"session_id":"2999dff5-85b3-443a-8027-415d24917db8"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:50.239456801Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1278,"session_id":"2999dff5-85b3-443a-8027-415d24917db8"} {"msg":"The guest has finished waiting for activation with:24","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:50.239468558Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1278} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 350 len 40960 data: 3 3 2 2 2 2 3 3 3 3 02/30 Write block 523 len 16384 data: 3 2 2 2 03/30 Flush 04/30 Flush 05/30 Write block 375 len 20480 data: 6 6 8 8 7 06/30 Write block 443 len 12288 data: 2 3 3 07/30 Read block 161 len 24576 08/30 Read block 562 len 12288 09/30 Read block 538 len 4096 10/30 Read block 534 len 20480 11/30 Read block 102 len 28672 12/30 Read block 384 len 40960 13/30 Read block 577 len 8192 14/30 Flush 15/30 Write block 215 len 24576 data: 2 2 2 2 2 2 16/30 Read block 213 len 40960 17/30 Read block 127 len 12288 18/30 Read block 92 len 28672 19/30 Write block 121 len 12288 data: 4 4 4 20/30 Write block 147 len 28672 data: 3 3 3 3 4 3 2 21/30 Write block 499 len 20480 data: 2 3 3 3 4 22/30 Write block 274 len 8192 data: 2 2 23/30 Read block 5 len 16384 24/30 Flush 25/30 Write block 299 len 4096 data: 3 26/30 Read block 53 len 24576 27/30 Write block 111 len 40960 data: 4 5 4 5 5 3 3 4 4 5 28/30 Read block 306 len 24576 29/30 Write block 31 len 12288 data: 4 5 4 30/30 Write block 256 len 32768 data: 3 3 3 3 3 3 3 3 ---------------------------------------------------------------- Crucible gen:24 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 1 Done Done Done false 1025 Acked Read 6 Done Done Done false 1026 Acked Write 10 Done Done Done false 1027 Acked Read 6 Done Done Done false 1028 Acked Write 3 Done Done Done false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:11:50.258763545Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1278} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 23 23 23 34 34 34 F F F 1 020-039 23 23 23 34 34 34 T T T 2 040-059 22 22 22 33 33 33 F F F 3 060-079 16 16 16 24 24 24 F F F 4 080-099 23 23 23 34 34 34 F F F 5 100-119 23 23 23 34 34 34 T T T 6 120-139 24 24 24 38 38 38 T T T 7 140-159 24 24 24 38 38 38 F F F 8 160-179 23 23 23 34 34 34 F F F 9 180-199 20 20 20 32 32 32 F F F 10 200-219 24 24 24 38 38 38 F F F 11 220-239 24 24 24 38 38 38 F F F 12 240-259 23 23 23 34 34 34 T T T 13 260-279 24 24 24 38 38 38 T T T 14 280-299 20 20 20 32 32 32 T T T 15 300-319 20 20 20 32 32 32 F F F 16 320-339 23 23 23 34 34 34 F F F 17 340-359 24 24 24 35 35 35 F F F 18 360-379 24 24 24 37 37 37 F F F 19 380-399 23 23 23 34 34 34 F F F 20 400-419 23 23 23 34 34 34 F F F 21 420-439 18 18 18 28 28 28 F F F 22 440-459 24 24 24 37 37 37 F F F 23 460-479 20 20 20 32 32 32 F F F 24 480-499 24 24 24 38 38 38 F F F 25 500-519 24 24 24 38 38 38 F F F 26 520-539 24 24 24 35 35 35 F F F 27 540-559 20 20 20 32 32 32 F F F 28 560-579 19 19 19 31 31 31 F F F 29 580-599 19 19 19 31 31 31 F F F Max gen: 24, Max flush: 38 A Difference in extent metadata is expected here On loop 12 Verifying data now No change after verify Loop: 12 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 23 23 23 34 34 34 F F F 1 020-039 25 25 25 39 39 39 F F F 2 040-059 22 22 22 33 33 33 F F F 3 060-079 16 16 16 24 24 24 F F F 4 080-099 23 23 23 34 34 34 F F F 5 100-119 25 25 25 39 39 39 F F F 6 120-139 25 25 25 39 39 39 F F F 7 140-159 24 24 24 38 38 38 F F F 8 160-179 23 23 23 34 34 34 F F F 9 180-199 20 20 20 32 32 32 F F F 10 200-219 24 24 24 38 38 38 F F F 11 220-239 24 24 24 38 38 38 F F F 12 240-259 25 25 25 39 39 39 F F F 13 260-279 25 25 25 39 39 39 F F F 14 280-299 25 25 25 39 39 39 F F F 15 300-319 20 20 20 32 32 32 F F F 16 320-339 23 23 23 34 34 34 F F F 17 340-359 24 24 24 35 35 35 F F F 18 360-379 24 24 24 37 37 37 F F F 19 380-399 23 23 23 34 34 34 F F F 20 400-419 23 23 23 34 34 34 F F F 21 420-439 18 18 18 28 28 28 F F F 22 440-459 24 24 24 37 37 37 F F F 23 460-479 20 20 20 32 32 32 F F F 24 480-499 24 24 24 38 38 38 F F F 25 500-519 24 24 24 38 38 38 F F F 26 520-539 24 24 24 35 35 35 F F F 27 540-559 20 20 20 32 32 32 F F F 28 560-579 19 19 19 31 31 31 F F F 29 580-599 19 19 19 31 31 31 F F F Max gen: 25, Max flush: 39 Begin loop 13 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:54.53715198Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1286,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:54.582109127Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1286,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:54.582742747Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1286} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"abd5430f1508af0bc008143ea2566277ef714ec2\",\n git_commit_timestamp: \"2025-01-17T20:55:03.000000000Z\",\n git_branch: \"alan/tokio-to-140\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:54.58278312Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1286} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:54.582798319Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1286} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:54.582812668Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1286} {"msg":"Crucible fa9ad0f9-a263-4b70-b0c3-1322e89a07c4 has session id: 5722b7ed-3331-4316-a7dc-91547b3415d2","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:54.583303234Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1286,"session_id":"5722b7ed-3331-4316-a7dc-91547b3415d2"} {"msg":"Upstairs opts: Upstairs UUID: fa9ad0f9-a263-4b70-b0c3-1322e89a07c4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:54.583343748Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1286,"session_id":"5722b7ed-3331-4316-a7dc-91547b3415d2"} {"msg":"Crucible stats registered with UUID: fa9ad0f9-a263-4b70-b0c3-1322e89a07c4","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:54.583367882Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1286,"session_id":"5722b7ed-3331-4316-a7dc-91547b3415d2"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:11:54.583388574Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1286,"session_id":"5722b7ed-3331-4316-a7dc-91547b3415d2"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:54.58387703Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1286,"task":"crutest"} {"msg":"The guest has requested activation with gen:26","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:54.583900974Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1286} {"msg":"fa9ad0f9-a263-4b70-b0c3-1322e89a07c4 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:54.583965721Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1286,"session_id":"5722b7ed-3331-4316-a7dc-91547b3415d2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:54.583983432Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1286,"client":"0","":"downstairs","session_id":"5722b7ed-3331-4316-a7dc-91547b3415d2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:54.584006846Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1286,"client":"0","":"downstairs","session_id":"5722b7ed-3331-4316-a7dc-91547b3415d2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:54.584025066Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1286,"client":"1","":"downstairs","session_id":"5722b7ed-3331-4316-a7dc-91547b3415d2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:54.584039625Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1286,"client":"1","":"downstairs","session_id":"5722b7ed-3331-4316-a7dc-91547b3415d2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:54.584053173Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1286,"client":"2","":"downstairs","session_id":"5722b7ed-3331-4316-a7dc-91547b3415d2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:54.584067981Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1286,"client":"2","":"downstairs","session_id":"5722b7ed-3331-4316-a7dc-91547b3415d2"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:54.58408275Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1286,"":"io task","client":"2","":"downstairs","session_id":"5722b7ed-3331-4316-a7dc-91547b3415d2"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:54.584103972Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1286,"":"io task","client":"0","":"downstairs","session_id":"5722b7ed-3331-4316-a7dc-91547b3415d2"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:54.584125375Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1286,"":"io task","client":"1","":"downstairs","session_id":"5722b7ed-3331-4316-a7dc-91547b3415d2"} {"msg":"ds_connection connected from Ok(127.0.0.1:47148)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:54.584213076Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1286,"":"io task","client":"2","":"downstairs","session_id":"5722b7ed-3331-4316-a7dc-91547b3415d2"} {"msg":"ds_connection connected from Ok(127.0.0.1:64415)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:54.584249257Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1286,"":"io task","client":"1","":"downstairs","session_id":"5722b7ed-3331-4316-a7dc-91547b3415d2"} {"msg":"ds_connection connected from Ok(127.0.0.1:55216)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:54.584275293Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1286,"":"io task","client":"0","":"downstairs","session_id":"5722b7ed-3331-4316-a7dc-91547b3415d2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:54.585073899Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1286,"client":"2","":"downstairs","session_id":"5722b7ed-3331-4316-a7dc-91547b3415d2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:54.585101235Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1286,"client":"0","":"downstairs","session_id":"5722b7ed-3331-4316-a7dc-91547b3415d2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:54.58519455Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1286,"client":"1","":"downstairs","session_id":"5722b7ed-3331-4316-a7dc-91547b3415d2"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:54.585737527Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1286,"client":"1","":"downstairs","session_id":"5722b7ed-3331-4316-a7dc-91547b3415d2"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:54.585768425Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1286,"client":"2","":"downstairs","session_id":"5722b7ed-3331-4316-a7dc-91547b3415d2"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:54.585784295Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1286,"client":"0","":"downstairs","session_id":"5722b7ed-3331-4316-a7dc-91547b3415d2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:54.586285808Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1286,"session_id":"5722b7ed-3331-4316-a7dc-91547b3415d2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:54.586388598Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1286,"session_id":"5722b7ed-3331-4316-a7dc-91547b3415d2"} {"msg":"[0]R flush_numbers[0..12]: [34, 39, 33, 24, 34, 39, 39, 38, 34, 32, 38, 38]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:54.586457008Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1286,"":"downstairs","":"downstairs","session_id":"5722b7ed-3331-4316-a7dc-91547b3415d2"} {"msg":"[0]R generation[0..12]: [23, 25, 22, 16, 23, 25, 25, 24, 23, 20, 24, 24]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:54.586475789Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1286,"":"downstairs","":"downstairs","session_id":"5722b7ed-3331-4316-a7dc-91547b3415d2"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:54.586490407Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1286,"":"downstairs","":"downstairs","session_id":"5722b7ed-3331-4316-a7dc-91547b3415d2"} {"msg":"[1]R flush_numbers[0..12]: [34, 39, 33, 24, 34, 39, 39, 38, 34, 32, 38, 38]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:54.586503885Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1286,"":"downstairs","":"downstairs","session_id":"5722b7ed-3331-4316-a7dc-91547b3415d2"} {"msg":"[1]R generation[0..12]: [23, 25, 22, 16, 23, 25, 25, 24, 23, 20, 24, 24]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:54.586517193Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1286,"":"downstairs","":"downstairs","session_id":"5722b7ed-3331-4316-a7dc-91547b3415d2"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:54.586530761Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1286,"":"downstairs","":"downstairs","session_id":"5722b7ed-3331-4316-a7dc-91547b3415d2"} {"msg":"[2]R flush_numbers[0..12]: [34, 39, 33, 24, 34, 39, 39, 38, 34, 32, 38, 38]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:54.58654709Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1286,"":"downstairs","":"downstairs","session_id":"5722b7ed-3331-4316-a7dc-91547b3415d2"} {"msg":"[2]R generation[0..12]: [23, 25, 22, 16, 23, 25, 25, 24, 23, 20, 24, 24]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:54.586571495Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1286,"":"downstairs","":"downstairs","session_id":"5722b7ed-3331-4316-a7dc-91547b3415d2"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:54.586599591Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1286,"":"downstairs","":"downstairs","session_id":"5722b7ed-3331-4316-a7dc-91547b3415d2"} {"msg":"Max found gen is 26","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:54.586616431Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1286,"":"downstairs","":"downstairs","session_id":"5722b7ed-3331-4316-a7dc-91547b3415d2"} {"msg":"Generation requested: 26 >= found:26","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-17T21:11:54.586643066Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1286,"":"downstairs","":"downstairs","session_id":"5722b7ed-3331-4316-a7dc-91547b3415d2"} {"msg":"Next flush: 40","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:54.586668901Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1286,"":"downstairs","":"downstairs","session_id":"5722b7ed-3331-4316-a7dc-91547b3415d2"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:54.586689694Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1286,"":"downstairs","":"downstairs","session_id":"5722b7ed-3331-4316-a7dc-91547b3415d2"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:54.586709285Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1286,"session_id":"5722b7ed-3331-4316-a7dc-91547b3415d2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:54.586730848Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1286,"client":"0","":"downstairs","session_id":"5722b7ed-3331-4316-a7dc-91547b3415d2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:54.586755942Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1286,"client":"1","":"downstairs","session_id":"5722b7ed-3331-4316-a7dc-91547b3415d2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:54.586774733Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1286,"client":"2","":"downstairs","session_id":"5722b7ed-3331-4316-a7dc-91547b3415d2"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:54.586788191Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1286,"session_id":"5722b7ed-3331-4316-a7dc-91547b3415d2"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:54.586807322Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1286,"session_id":"5722b7ed-3331-4316-a7dc-91547b3415d2"} {"msg":"fa9ad0f9-a263-4b70-b0c3-1322e89a07c4 is now active with session: 5722b7ed-3331-4316-a7dc-91547b3415d2","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:54.586841702Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1286,"session_id":"5722b7ed-3331-4316-a7dc-91547b3415d2"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:54.586865426Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1286,"session_id":"5722b7ed-3331-4316-a7dc-91547b3415d2"} {"msg":"The guest has finished waiting for activation with:26","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:54.586886219Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1286} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 52 len 8192 data: 7 7 02/30 Write block 352 len 36864 data: 3 3 3 3 4 4 4 4 3 03/30 Write block 543 len 36864 data: 4 4 4 5 5 5 4 4 4 04/30 Write block 561 len 20480 data: 4 4 4 3 3 05/30 Read block 226 len 12288 06/30 Read block 98 len 16384 07/30 Write block 388 len 8192 data: 3 3 08/30 Read block 18 len 4096 09/30 Flush 10/30 Write block 337 len 20480 data: 3 3 3 4 3 11/30 Read block 203 len 24576 12/30 Write block 32 len 24576 data: 6 5 4 4 3 4 13/30 Read block 282 len 12288 14/30 Read block 109 len 40960 15/30 Read block 218 len 40960 16/30 Read block 1 len 16384 17/30 Write block 443 len 16384 data: 3 4 4 3 18/30 Write block 428 len 8192 data: 3 3 19/30 Write block 519 len 36864 data: 4 4 4 2 4 3 3 3 2 20/30 Write block 156 len 20480 data: 2 2 2 2 2 21/30 Read block 279 len 28672 22/30 Write block 496 len 36864 data: 2 2 2 3 4 4 4 5 6 23/30 Read block 197 len 32768 24/30 Flush 25/30 Write block 462 len 24576 data: 3 3 4 4 3 4 26/30 Read block 489 len 40960 27/30 Write block 21 len 24576 data: 3 2 2 2 2 4 28/30 Read block 137 len 24576 29/30 Read block 272 len 32768 30/30 Read block 388 len 16384 ---------------------------------------------------------------- Crucible gen:26 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 6 Done Done Done false 1025 Acked Read 10 Done Done Done false 1026 Acked Write 6 Done Done Done false 1027 Acked Read 6 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Read 4 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 6 5 5 16 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:11:54.606467973Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1286} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 23 23 23 34 34 34 F F F 1 020-039 26 26 26 41 41 41 T T T 2 040-059 26 26 26 40 40 40 F F F 3 060-079 16 16 16 24 24 24 F F F 4 080-099 23 23 23 34 34 34 F F F 5 100-119 25 25 25 39 39 39 F F F 6 120-139 25 25 25 39 39 39 F F F 7 140-159 26 26 26 41 41 41 F F F 8 160-179 26 26 26 41 41 41 F F F 9 180-199 20 20 20 32 32 32 F F F 10 200-219 24 24 24 38 38 38 F F F 11 220-239 24 24 24 38 38 38 F F F 12 240-259 25 25 25 39 39 39 F F F 13 260-279 25 25 25 39 39 39 F F F 14 280-299 25 25 25 39 39 39 F F F 15 300-319 20 20 20 32 32 32 F F F 16 320-339 26 26 26 41 41 41 F F F 17 340-359 26 26 26 41 41 41 F F F 18 360-379 26 26 26 40 40 40 F F F 19 380-399 26 26 26 40 40 40 F F F 20 400-419 23 23 23 34 34 34 F F F 21 420-439 26 26 26 41 41 41 F F F 22 440-459 26 26 26 41 41 41 F F F 23 460-479 20 20 20 32 32 32 T T T 24 480-499 26 26 26 41 41 41 F F F 25 500-519 26 26 26 41 41 41 F F F 26 520-539 26 26 26 41 41 41 F F F 27 540-559 26 26 26 40 40 40 F F F 28 560-579 26 26 26 40 40 40 F F F 29 580-599 19 19 19 31 31 31 F F F Max gen: 26, Max flush: 41 A Difference in extent metadata is expected here On loop 13 Verifying data now No change after verify Loop: 13 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 23 23 23 34 34 34 F F F 1 020-039 27 27 27 42 42 42 F F F 2 040-059 26 26 26 40 40 40 F F F 3 060-079 16 16 16 24 24 24 F F F 4 080-099 23 23 23 34 34 34 F F F 5 100-119 25 25 25 39 39 39 F F F 6 120-139 25 25 25 39 39 39 F F F 7 140-159 26 26 26 41 41 41 F F F 8 160-179 26 26 26 41 41 41 F F F 9 180-199 20 20 20 32 32 32 F F F 10 200-219 24 24 24 38 38 38 F F F 11 220-239 24 24 24 38 38 38 F F F 12 240-259 25 25 25 39 39 39 F F F 13 260-279 25 25 25 39 39 39 F F F 14 280-299 25 25 25 39 39 39 F F F 15 300-319 20 20 20 32 32 32 F F F 16 320-339 26 26 26 41 41 41 F F F 17 340-359 26 26 26 41 41 41 F F F 18 360-379 26 26 26 40 40 40 F F F 19 380-399 26 26 26 40 40 40 F F F 20 400-419 23 23 23 34 34 34 F F F 21 420-439 26 26 26 41 41 41 F F F 22 440-459 26 26 26 41 41 41 F F F 23 460-479 27 27 27 42 42 42 F F F 24 480-499 26 26 26 41 41 41 F F F 25 500-519 26 26 26 41 41 41 F F F 26 520-539 26 26 26 41 41 41 F F F 27 540-559 26 26 26 40 40 40 F F F 28 560-579 26 26 26 40 40 40 F F F 29 580-599 19 19 19 31 31 31 F F F Max gen: 27, Max flush: 42 Begin loop 14 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:58.828993628Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1294,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:58.873781786Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1294,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:58.874494012Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1294} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"abd5430f1508af0bc008143ea2566277ef714ec2\",\n git_commit_timestamp: \"2025-01-17T20:55:03.000000000Z\",\n git_branch: \"alan/tokio-to-140\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:58.874534886Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1294} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:58.874549234Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1294} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:58.874563383Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1294} {"msg":"Crucible af71ed00-925e-4367-ba01-e6bd5d3e58b7 has session id: 1ca33778-d2f5-4994-997d-bf082384c7a6","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:58.875110212Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1294,"session_id":"1ca33778-d2f5-4994-997d-bf082384c7a6"} {"msg":"Upstairs opts: Upstairs UUID: af71ed00-925e-4367-ba01-e6bd5d3e58b7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:58.875142981Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1294,"session_id":"1ca33778-d2f5-4994-997d-bf082384c7a6"} {"msg":"Crucible stats registered with UUID: af71ed00-925e-4367-ba01-e6bd5d3e58b7","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:58.875160021Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1294,"session_id":"1ca33778-d2f5-4994-997d-bf082384c7a6"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:11:58.87517524Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1294,"session_id":"1ca33778-d2f5-4994-997d-bf082384c7a6"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:58.875601759Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1294,"task":"crutest"} {"msg":"The guest has requested activation with gen:28","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:58.875628465Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1294} {"msg":"af71ed00-925e-4367-ba01-e6bd5d3e58b7 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:58.875701678Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1294,"session_id":"1ca33778-d2f5-4994-997d-bf082384c7a6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:58.87572333Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1294,"client":"0","":"downstairs","session_id":"1ca33778-d2f5-4994-997d-bf082384c7a6"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:58.87575588Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1294,"client":"0","":"downstairs","session_id":"1ca33778-d2f5-4994-997d-bf082384c7a6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:58.875786097Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1294,"client":"1","":"downstairs","session_id":"1ca33778-d2f5-4994-997d-bf082384c7a6"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:58.875821178Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1294,"client":"1","":"downstairs","session_id":"1ca33778-d2f5-4994-997d-bf082384c7a6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:58.875850315Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1294,"client":"2","":"downstairs","session_id":"1ca33778-d2f5-4994-997d-bf082384c7a6"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:58.875875399Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1294,"client":"2","":"downstairs","session_id":"1ca33778-d2f5-4994-997d-bf082384c7a6"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:58.875898203Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1294,"":"io task","client":"2","":"downstairs","session_id":"1ca33778-d2f5-4994-997d-bf082384c7a6"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:58.875926119Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1294,"":"io task","client":"0","":"downstairs","session_id":"1ca33778-d2f5-4994-997d-bf082384c7a6"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:58.875953445Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1294,"":"io task","client":"1","":"downstairs","session_id":"1ca33778-d2f5-4994-997d-bf082384c7a6"} {"msg":"ds_connection connected from Ok(127.0.0.1:51114)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:58.87597893Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1294,"":"io task","client":"2","":"downstairs","session_id":"1ca33778-d2f5-4994-997d-bf082384c7a6"} {"msg":"ds_connection connected from Ok(127.0.0.1:61451)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:58.876003044Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1294,"":"io task","client":"0","":"downstairs","session_id":"1ca33778-d2f5-4994-997d-bf082384c7a6"} {"msg":"ds_connection connected from Ok(127.0.0.1:63145)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:58.87603004Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1294,"":"io task","client":"1","":"downstairs","session_id":"1ca33778-d2f5-4994-997d-bf082384c7a6"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:58.876710517Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1294,"client":"0","":"downstairs","session_id":"1ca33778-d2f5-4994-997d-bf082384c7a6"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:58.876747099Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1294,"client":"2","":"downstairs","session_id":"1ca33778-d2f5-4994-997d-bf082384c7a6"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:58.876892304Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1294,"client":"1","":"downstairs","session_id":"1ca33778-d2f5-4994-997d-bf082384c7a6"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:58.877363589Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1294,"client":"2","":"downstairs","session_id":"1ca33778-d2f5-4994-997d-bf082384c7a6"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:58.877403392Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1294,"client":"0","":"downstairs","session_id":"1ca33778-d2f5-4994-997d-bf082384c7a6"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:58.877573182Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1294,"client":"1","":"downstairs","session_id":"1ca33778-d2f5-4994-997d-bf082384c7a6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:58.877978419Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1294,"session_id":"1ca33778-d2f5-4994-997d-bf082384c7a6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:58.878123993Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1294,"session_id":"1ca33778-d2f5-4994-997d-bf082384c7a6"} {"msg":"[0]R flush_numbers[0..12]: [34, 42, 40, 24, 34, 39, 39, 41, 41, 32, 38, 38]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:58.878190643Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1294,"":"downstairs","":"downstairs","session_id":"1ca33778-d2f5-4994-997d-bf082384c7a6"} {"msg":"[0]R generation[0..12]: [23, 27, 26, 16, 23, 25, 25, 26, 26, 20, 24, 24]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:58.878209543Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1294,"":"downstairs","":"downstairs","session_id":"1ca33778-d2f5-4994-997d-bf082384c7a6"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:58.878223862Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1294,"":"downstairs","":"downstairs","session_id":"1ca33778-d2f5-4994-997d-bf082384c7a6"} {"msg":"[1]R flush_numbers[0..12]: [34, 42, 40, 24, 34, 39, 39, 41, 41, 32, 38, 38]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:58.87823713Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1294,"":"downstairs","":"downstairs","session_id":"1ca33778-d2f5-4994-997d-bf082384c7a6"} {"msg":"[1]R generation[0..12]: [23, 27, 26, 16, 23, 25, 25, 26, 26, 20, 24, 24]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:58.878250377Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1294,"":"downstairs","":"downstairs","session_id":"1ca33778-d2f5-4994-997d-bf082384c7a6"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:58.878264175Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1294,"":"downstairs","":"downstairs","session_id":"1ca33778-d2f5-4994-997d-bf082384c7a6"} {"msg":"[2]R flush_numbers[0..12]: [34, 42, 40, 24, 34, 39, 39, 41, 41, 32, 38, 38]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:58.878281075Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1294,"":"downstairs","":"downstairs","session_id":"1ca33778-d2f5-4994-997d-bf082384c7a6"} {"msg":"[2]R generation[0..12]: [23, 27, 26, 16, 23, 25, 25, 26, 26, 20, 24, 24]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:58.878294513Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1294,"":"downstairs","":"downstairs","session_id":"1ca33778-d2f5-4994-997d-bf082384c7a6"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:58.878308351Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1294,"":"downstairs","":"downstairs","session_id":"1ca33778-d2f5-4994-997d-bf082384c7a6"} {"msg":"Max found gen is 28","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:58.878320909Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1294,"":"downstairs","":"downstairs","session_id":"1ca33778-d2f5-4994-997d-bf082384c7a6"} {"msg":"Generation requested: 28 >= found:28","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:58.878333786Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1294,"":"downstairs","":"downstairs","session_id":"1ca33778-d2f5-4994-997d-bf082384c7a6"} {"msg":"Next flush: 43","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:58.878346263Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1294,"":"downstairs","":"downstairs","session_id":"1ca33778-d2f5-4994-997d-bf082384c7a6"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:58.878366205Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1294,"":"downstairs","":"downstairs","session_id":"1ca33778-d2f5-4994-997d-bf082384c7a6"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-17T21:11:58.878389559Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1294,"session_id":"1ca33778-d2f5-4994-997d-bf082384c7a6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:58.878417555Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1294,"client":"0","":"downstairs","session_id":"1ca33778-d2f5-4994-997d-bf082384c7a6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:58.878435355Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1294,"client":"1","":"downstairs","session_id":"1ca33778-d2f5-4994-997d-bf082384c7a6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:58.878448503Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1294,"client":"2","":"downstairs","session_id":"1ca33778-d2f5-4994-997d-bf082384c7a6"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:58.878461271Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1294,"session_id":"1ca33778-d2f5-4994-997d-bf082384c7a6"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:58.878473037Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1294,"session_id":"1ca33778-d2f5-4994-997d-bf082384c7a6"} {"msg":"af71ed00-925e-4367-ba01-e6bd5d3e58b7 is now active with session: 1ca33778-d2f5-4994-997d-bf082384c7a6","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:58.878485535Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1294,"session_id":"1ca33778-d2f5-4994-997d-bf082384c7a6"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:58.878497051Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1294,"session_id":"1ca33778-d2f5-4994-997d-bf082384c7a6"} {"msg":"The guest has finished waiting for activation with:28","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:11:58.878508938Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1294} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 467 len 36864 data: 5 4 4 4 4 5 3 2 2 02/30 Read block 380 len 32768 03/30 Write block 425 len 28672 data: 2 3 3 4 4 3 4 04/30 Write block 543 len 32768 data: 5 5 5 6 6 6 5 5 05/30 Write block 515 len 40960 data: 5 5 4 4 5 5 5 3 5 4 06/30 Write block 448 len 8192 data: 3 4 07/30 Write block 94 len 32768 data: 2 2 2 2 3 3 3 3 08/30 Write block 375 len 40960 data: 7 7 9 9 8 6 5 4 4 3 09/30 Read block 414 len 8192 10/30 Write block 223 len 20480 data: 4 4 4 4 4 11/30 Write block 357 len 32768 data: 5 5 5 4 3 3 3 3 12/30 Write block 484 len 4096 data: 3 13/30 Read block 471 len 16384 14/30 Write block 131 len 8192 data: 2 2 15/30 Read block 353 len 16384 16/30 Write block 461 len 40960 data: 3 4 4 5 5 4 6 5 5 5 17/30 Write block 166 len 40960 data: 3 3 4 4 4 3 4 4 4 5 18/30 Write block 193 len 32768 data: 7 7 5 4 3 4 4 4 19/30 Read block 270 len 40960 20/30 Write block 414 len 16384 data: 2 2 2 2 21/30 Write block 35 len 4096 data: 5 22/30 Flush 23/30 Write block 501 len 40960 data: 5 5 6 7 6 6 6 5 5 5 24/30 Read block 488 len 40960 25/30 Flush 26/30 Write block 276 len 28672 data: 2 2 2 2 3 3 3 27/30 Write block 75 len 28672 data: 3 4 4 3 3 2 2 28/30 Write block 219 len 4096 data: 3 29/30 Write block 462 len 16384 data: 5 5 6 6 30/30 Read block 468 len 40960 ---------------------------------------------------------------- Crucible gen:28 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 7 Done Done Done false 1026 Acked Write 7 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Write 4 Done Done Sent false 1029 Acked Read 10 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 2 3 Done 5 4 3 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:11:58.897824628Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1294} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 23 23 23 34 34 34 F F F 1 020-039 28 28 28 43 43 43 F F F 2 040-059 26 26 26 40 40 40 F F F 3 060-079 16 16 16 24 24 24 T T T 4 080-099 28 28 28 43 43 43 T T T 5 100-119 28 28 28 43 43 43 F F F 6 120-139 28 28 28 43 43 43 F F F 7 140-159 26 26 26 41 41 41 F F F 8 160-179 28 28 28 43 43 43 F F F 9 180-199 28 28 28 43 43 43 F F F 10 200-219 28 28 28 43 43 43 T T T 11 220-239 28 28 28 43 43 43 F F F 12 240-259 25 25 25 39 39 39 F F F 13 260-279 25 25 25 39 39 39 T T T 14 280-299 25 25 25 39 39 39 T T T 15 300-319 20 20 20 32 32 32 F F F 16 320-339 26 26 26 41 41 41 F F F 17 340-359 28 28 28 43 43 43 F F F 18 360-379 28 28 28 43 43 43 F F F 19 380-399 28 28 28 43 43 43 F F F 20 400-419 28 28 28 43 43 43 F F F 21 420-439 28 28 28 43 43 43 F F F 22 440-459 28 28 28 43 43 43 F F F 23 460-479 28 28 28 43 43 43 T T T 24 480-499 28 28 28 43 43 43 F F F 25 500-519 28 28 28 44 44 44 F F F 26 520-539 28 28 28 43 43 43 F F F 27 540-559 28 28 28 43 43 43 F F F 28 560-579 26 26 26 40 40 40 F F F 29 580-599 19 19 19 31 31 31 F F F Max gen: 28, Max flush: 44 A Difference in extent metadata is expected here On loop 14 Verifying data now No change after verify Loop: 14 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 23 23 23 34 34 34 F F F 1 020-039 28 28 28 43 43 43 F F F 2 040-059 26 26 26 40 40 40 F F F 3 060-079 29 29 29 45 45 45 F F F 4 080-099 29 29 29 45 45 45 F F F 5 100-119 28 28 28 43 43 43 F F F 6 120-139 28 28 28 43 43 43 F F F 7 140-159 26 26 26 41 41 41 F F F 8 160-179 28 28 28 43 43 43 F F F 9 180-199 28 28 28 43 43 43 F F F 10 200-219 29 29 29 45 45 45 F F F 11 220-239 28 28 28 43 43 43 F F F 12 240-259 25 25 25 39 39 39 F F F 13 260-279 29 29 29 45 45 45 F F F 14 280-299 29 29 29 45 45 45 F F F 15 300-319 20 20 20 32 32 32 F F F 16 320-339 26 26 26 41 41 41 F F F 17 340-359 28 28 28 43 43 43 F F F 18 360-379 28 28 28 43 43 43 F F F 19 380-399 28 28 28 43 43 43 F F F 20 400-419 28 28 28 43 43 43 F F F 21 420-439 28 28 28 43 43 43 F F F 22 440-459 28 28 28 43 43 43 F F F 23 460-479 29 29 29 45 45 45 F F F 24 480-499 28 28 28 43 43 43 F F F 25 500-519 28 28 28 44 44 44 F F F 26 520-539 28 28 28 43 43 43 F F F 27 540-559 28 28 28 43 43 43 F F F 28 560-579 26 26 26 40 40 40 F F F 29 580-599 19 19 19 31 31 31 F F F Max gen: 29, Max flush: 45 Begin loop 15 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:03.147460949Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1302,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:03.191437232Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1302,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:03.192144946Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1302} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"abd5430f1508af0bc008143ea2566277ef714ec2\",\n git_commit_timestamp: \"2025-01-17T20:55:03.000000000Z\",\n git_branch: \"alan/tokio-to-140\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:03.192183538Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1302} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:03.192198477Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1302} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:03.192212835Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1302} {"msg":"Crucible 80ef5eca-e7b0-4512-bead-6d431fc9b998 has session id: 08be4cae-a995-45d6-acd1-48866aa658d6","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:03.192794605Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1302,"session_id":"08be4cae-a995-45d6-acd1-48866aa658d6"} {"msg":"Upstairs opts: Upstairs UUID: 80ef5eca-e7b0-4512-bead-6d431fc9b998, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:03.192822992Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1302,"session_id":"08be4cae-a995-45d6-acd1-48866aa658d6"} {"msg":"Crucible stats registered with UUID: 80ef5eca-e7b0-4512-bead-6d431fc9b998","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:03.19283687Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1302,"session_id":"08be4cae-a995-45d6-acd1-48866aa658d6"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:12:03.192849257Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1302,"session_id":"08be4cae-a995-45d6-acd1-48866aa658d6"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:03.193369101Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1302,"task":"crutest"} {"msg":"The guest has requested activation with gen:30","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:03.193394016Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1302} {"msg":"80ef5eca-e7b0-4512-bead-6d431fc9b998 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:03.193467208Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1302,"session_id":"08be4cae-a995-45d6-acd1-48866aa658d6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:03.193484969Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1302,"client":"0","":"downstairs","session_id":"08be4cae-a995-45d6-acd1-48866aa658d6"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:03.193509533Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1302,"client":"0","":"downstairs","session_id":"08be4cae-a995-45d6-acd1-48866aa658d6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:03.193533267Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1302,"client":"1","":"downstairs","session_id":"08be4cae-a995-45d6-acd1-48866aa658d6"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:03.193560493Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1302,"client":"1","":"downstairs","session_id":"08be4cae-a995-45d6-acd1-48866aa658d6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:03.193586378Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1302,"client":"2","":"downstairs","session_id":"08be4cae-a995-45d6-acd1-48866aa658d6"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:03.193613694Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1302,"client":"2","":"downstairs","session_id":"08be4cae-a995-45d6-acd1-48866aa658d6"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:03.19364044Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1302,"":"io task","client":"2","":"downstairs","session_id":"08be4cae-a995-45d6-acd1-48866aa658d6"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:03.193673219Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1302,"":"io task","client":"0","":"downstairs","session_id":"08be4cae-a995-45d6-acd1-48866aa658d6"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:03.193701325Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1302,"":"io task","client":"1","":"downstairs","session_id":"08be4cae-a995-45d6-acd1-48866aa658d6"} {"msg":"ds_connection connected from Ok(127.0.0.1:33568)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:03.193728521Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1302,"":"io task","client":"2","":"downstairs","session_id":"08be4cae-a995-45d6-acd1-48866aa658d6"} {"msg":"ds_connection connected from Ok(127.0.0.1:56299)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:03.193754586Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1302,"":"io task","client":"1","":"downstairs","session_id":"08be4cae-a995-45d6-acd1-48866aa658d6"} {"msg":"ds_connection connected from Ok(127.0.0.1:37615)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:03.193804656Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1302,"":"io task","client":"0","":"downstairs","session_id":"08be4cae-a995-45d6-acd1-48866aa658d6"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:03.194338778Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1302,"client":"1","":"downstairs","session_id":"08be4cae-a995-45d6-acd1-48866aa658d6"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:03.194375219Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1302,"client":"0","":"downstairs","session_id":"08be4cae-a995-45d6-acd1-48866aa658d6"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:03.194857811Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1302,"client":"2","":"downstairs","session_id":"08be4cae-a995-45d6-acd1-48866aa658d6"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:03.19499239Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1302,"client":"0","":"downstairs","session_id":"08be4cae-a995-45d6-acd1-48866aa658d6"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:03.195021837Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1302,"client":"1","":"downstairs","session_id":"08be4cae-a995-45d6-acd1-48866aa658d6"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:03.195396056Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1302,"client":"2","":"downstairs","session_id":"08be4cae-a995-45d6-acd1-48866aa658d6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:03.195555619Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1302,"session_id":"08be4cae-a995-45d6-acd1-48866aa658d6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:03.195641249Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1302,"session_id":"08be4cae-a995-45d6-acd1-48866aa658d6"} {"msg":"[0]R flush_numbers[0..12]: [34, 43, 40, 45, 45, 43, 43, 41, 43, 43, 45, 43]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:03.19589976Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1302,"":"downstairs","":"downstairs","session_id":"08be4cae-a995-45d6-acd1-48866aa658d6"} {"msg":"[0]R generation[0..12]: [23, 28, 26, 29, 29, 28, 28, 26, 28, 28, 29, 28]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:03.195918901Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1302,"":"downstairs","":"downstairs","session_id":"08be4cae-a995-45d6-acd1-48866aa658d6"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:03.195936491Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1302,"":"downstairs","":"downstairs","session_id":"08be4cae-a995-45d6-acd1-48866aa658d6"} {"msg":"[1]R flush_numbers[0..12]: [34, 43, 40, 45, 45, 43, 43, 41, 43, 43, 45, 43]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:03.19595028Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1302,"":"downstairs","":"downstairs","session_id":"08be4cae-a995-45d6-acd1-48866aa658d6"} {"msg":"[1]R generation[0..12]: [23, 28, 26, 29, 29, 28, 28, 26, 28, 28, 29, 28]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:03.195963607Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1302,"":"downstairs","":"downstairs","session_id":"08be4cae-a995-45d6-acd1-48866aa658d6"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:03.195977366Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1302,"":"downstairs","":"downstairs","session_id":"08be4cae-a995-45d6-acd1-48866aa658d6"} {"msg":"[2]R flush_numbers[0..12]: [34, 43, 40, 45, 45, 43, 43, 41, 43, 43, 45, 43]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:03.195990983Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1302,"":"downstairs","":"downstairs","session_id":"08be4cae-a995-45d6-acd1-48866aa658d6"} {"msg":"[2]R generation[0..12]: [23, 28, 26, 29, 29, 28, 28, 26, 28, 28, 29, 28]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:03.196013186Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1302,"":"downstairs","":"downstairs","session_id":"08be4cae-a995-45d6-acd1-48866aa658d6"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:03.196031607Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1302,"":"downstairs","":"downstairs","session_id":"08be4cae-a995-45d6-acd1-48866aa658d6"} {"msg":"Max found gen is 30","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:03.196044445Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1302,"":"downstairs","":"downstairs","session_id":"08be4cae-a995-45d6-acd1-48866aa658d6"} {"msg":"Generation requested: 30 >= found:30","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:03.196057372Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1302,"":"downstairs","":"downstairs","session_id":"08be4cae-a995-45d6-acd1-48866aa658d6"} {"msg":"Next flush: 46","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-17T21:12:03.196069929Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1302,"":"downstairs","":"downstairs","session_id":"08be4cae-a995-45d6-acd1-48866aa658d6"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:03.196085178Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1302,"":"downstairs","":"downstairs","session_id":"08be4cae-a995-45d6-acd1-48866aa658d6"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:03.196098146Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1302,"session_id":"08be4cae-a995-45d6-acd1-48866aa658d6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:03.196109583Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1302,"client":"0","":"downstairs","session_id":"08be4cae-a995-45d6-acd1-48866aa658d6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:03.19612274Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1302,"client":"1","":"downstairs","session_id":"08be4cae-a995-45d6-acd1-48866aa658d6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:03.196137669Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1302,"client":"2","":"downstairs","session_id":"08be4cae-a995-45d6-acd1-48866aa658d6"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:03.196151697Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1302,"session_id":"08be4cae-a995-45d6-acd1-48866aa658d6"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:03.196163734Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1302,"session_id":"08be4cae-a995-45d6-acd1-48866aa658d6"} {"msg":"80ef5eca-e7b0-4512-bead-6d431fc9b998 is now active with session: 08be4cae-a995-45d6-acd1-48866aa658d6","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:03.196176192Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1302,"session_id":"08be4cae-a995-45d6-acd1-48866aa658d6"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:03.196187929Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1302,"session_id":"08be4cae-a995-45d6-acd1-48866aa658d6"} {"msg":"The guest has finished waiting for activation with:30","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:03.196199756Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1302} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 130 len 20480 data: 3 3 3 2 3 02/30 Read block 454 len 12288 03/30 Read block 309 len 24576 04/30 Read block 188 len 20480 05/30 Write block 290 len 12288 data: 2 2 2 06/30 Flush 07/30 Write block 392 len 12288 data: 3 3 3 08/30 Read block 86 len 28672 09/30 Read block 476 len 20480 10/30 Write block 396 len 40960 data: 4 4 4 5 5 4 3 5 5 5 11/30 Read block 348 len 28672 12/30 Read block 350 len 40960 13/30 Write block 275 len 32768 data: 3 3 3 3 3 4 4 4 14/30 Write block 342 len 8192 data: 3 6 15/30 Flush 16/30 Write block 151 len 8192 data: 5 4 17/30 Write block 95 len 32768 data: 3 3 3 4 4 4 4 3 18/30 Read block 404 len 32768 19/30 Read block 419 len 8192 20/30 Write block 325 len 8192 data: 3 3 21/30 Write block 102 len 16384 data: 4 3 3 3 22/30 Write block 467 len 36864 data: 7 6 6 6 5 6 4 3 3 23/30 Write block 301 len 28672 data: 3 2 3 3 3 3 4 24/30 Read block 22 len 36864 25/30 Write block 265 len 4096 data: 3 26/30 Read block 423 len 16384 27/30 Write block 516 len 4096 data: 6 28/30 Read block 194 len 8192 29/30 Read block 327 len 32768 30/30 Read block 105 len 32768 ---------------------------------------------------------------- Crucible gen:30 GIO:true work queues: Upstairs:0 downstairs:15 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1015 Acked Write 2 Done Done Done false 1016 Acked Write 8 Done Done Done false 1017 Acked Read 8 Done Done Done false 1018 Acked Read 2 Done Done Done false 1019 Acked Write 2 Done Done Done false 1020 Acked Write 4 Done Done Done false 1021 Acked Write 9 Done Done Done false 1022 Acked Write 7 Done Done Done false 1023 Acked Read 9 Done Done Done false 1024 Acked Write 1 Done Done Done false 1025 Acked Read 4 Done Done Done false 1026 Acked Write 1 Done Done Done false 1027 Acked Read 2 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Read 8 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 15 15 15 45 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1014 1014 1014 Downstairs last five completed: 1014 1013 1012 1011 1010 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:12:03.215684884Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1302} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 23 23 23 34 34 34 F F F 1 020-039 28 28 28 43 43 43 F F F 2 040-059 26 26 26 40 40 40 F F F 3 060-079 29 29 29 45 45 45 F F F 4 080-099 29 29 29 45 45 45 T T T 5 100-119 28 28 28 43 43 43 T T T 6 120-139 30 30 30 46 46 46 F F F 7 140-159 26 26 26 41 41 41 T T T 8 160-179 28 28 28 43 43 43 F F F 9 180-199 28 28 28 43 43 43 F F F 10 200-219 29 29 29 45 45 45 F F F 11 220-239 28 28 28 43 43 43 F F F 12 240-259 25 25 25 39 39 39 F F F 13 260-279 30 30 30 47 47 47 T T T 14 280-299 30 30 30 47 47 47 F F F 15 300-319 20 20 20 32 32 32 T T T 16 320-339 26 26 26 41 41 41 T T T 17 340-359 30 30 30 47 47 47 F F F 18 360-379 28 28 28 43 43 43 F F F 19 380-399 30 30 30 47 47 47 F F F 20 400-419 30 30 30 47 47 47 F F F 21 420-439 28 28 28 43 43 43 F F F 22 440-459 28 28 28 43 43 43 F F F 23 460-479 29 29 29 45 45 45 T T T 24 480-499 28 28 28 43 43 43 F F F 25 500-519 28 28 28 44 44 44 T T T 26 520-539 28 28 28 43 43 43 F F F 27 540-559 28 28 28 43 43 43 F F F 28 560-579 26 26 26 40 40 40 F F F 29 580-599 19 19 19 31 31 31 F F F Max gen: 30, Max flush: 47 A Difference in extent metadata is expected here On loop 15 Verifying data now No change after verify Loop: 15 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 23 23 23 34 34 34 F F F 1 020-039 28 28 28 43 43 43 F F F 2 040-059 26 26 26 40 40 40 F F F 3 060-079 29 29 29 45 45 45 F F F 4 080-099 31 31 31 48 48 48 F F F 5 100-119 31 31 31 48 48 48 F F F 6 120-139 30 30 30 46 46 46 F F F 7 140-159 31 31 31 48 48 48 F F F 8 160-179 28 28 28 43 43 43 F F F 9 180-199 28 28 28 43 43 43 F F F 10 200-219 29 29 29 45 45 45 F F F 11 220-239 28 28 28 43 43 43 F F F 12 240-259 25 25 25 39 39 39 F F F 13 260-279 31 31 31 48 48 48 F F F 14 280-299 30 30 30 47 47 47 F F F 15 300-319 31 31 31 48 48 48 F F F 16 320-339 31 31 31 48 48 48 F F F 17 340-359 30 30 30 47 47 47 F F F 18 360-379 28 28 28 43 43 43 F F F 19 380-399 30 30 30 47 47 47 F F F 20 400-419 30 30 30 47 47 47 F F F 21 420-439 28 28 28 43 43 43 F F F 22 440-459 28 28 28 43 43 43 F F F 23 460-479 31 31 31 48 48 48 F F F 24 480-499 28 28 28 43 43 43 F F F 25 500-519 31 31 31 48 48 48 F F F 26 520-539 28 28 28 43 43 43 F F F 27 540-559 28 28 28 43 43 43 F F F 28 560-579 26 26 26 40 40 40 F F F 29 580-599 19 19 19 31 31 31 F F F Max gen: 31, Max flush: 48 Begin loop 16 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:07.474394478Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1310,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:07.52156861Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1310,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:07.522290341Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1310} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"abd5430f1508af0bc008143ea2566277ef714ec2\",\n git_commit_timestamp: \"2025-01-17T20:55:03.000000000Z\",\n git_branch: \"alan/tokio-to-140\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:07.522329364Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1310} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:07.522343732Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1310} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:07.522357981Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1310} {"msg":"Crucible aa76d2aa-84d3-4e0b-9b7c-278e8fe554ee has session id: e4bd1088-db7d-460e-b74c-98119714c387","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:07.522953679Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1310,"session_id":"e4bd1088-db7d-460e-b74c-98119714c387"} {"msg":"Upstairs opts: Upstairs UUID: aa76d2aa-84d3-4e0b-9b7c-278e8fe554ee, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:07.522994963Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1310,"session_id":"e4bd1088-db7d-460e-b74c-98119714c387"} {"msg":"Crucible stats registered with UUID: aa76d2aa-84d3-4e0b-9b7c-278e8fe554ee","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:07.523021999Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1310,"session_id":"e4bd1088-db7d-460e-b74c-98119714c387"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:12:07.523044062Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1310,"session_id":"e4bd1088-db7d-460e-b74c-98119714c387"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:07.523500048Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1310,"task":"crutest"} {"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:07.523526084Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1310} {"msg":"aa76d2aa-84d3-4e0b-9b7c-278e8fe554ee active request set","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:07.523580546Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1310,"session_id":"e4bd1088-db7d-460e-b74c-98119714c387"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:07.523606491Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1310,"client":"0","":"downstairs","session_id":"e4bd1088-db7d-460e-b74c-98119714c387"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:07.523632586Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1310,"client":"0","":"downstairs","session_id":"e4bd1088-db7d-460e-b74c-98119714c387"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:07.523649776Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1310,"client":"1","":"downstairs","session_id":"e4bd1088-db7d-460e-b74c-98119714c387"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:07.523664315Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1310,"client":"1","":"downstairs","session_id":"e4bd1088-db7d-460e-b74c-98119714c387"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:07.523677192Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1310,"client":"2","":"downstairs","session_id":"e4bd1088-db7d-460e-b74c-98119714c387"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:07.52369091Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1310,"client":"2","":"downstairs","session_id":"e4bd1088-db7d-460e-b74c-98119714c387"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:07.523703838Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1310,"":"io task","client":"2","":"downstairs","session_id":"e4bd1088-db7d-460e-b74c-98119714c387"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:07.523718296Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1310,"":"io task","client":"0","":"downstairs","session_id":"e4bd1088-db7d-460e-b74c-98119714c387"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:07.52374148Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1310,"":"io task","client":"1","":"downstairs","session_id":"e4bd1088-db7d-460e-b74c-98119714c387"} {"msg":"ds_connection connected from Ok(127.0.0.1:64155)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:07.523764853Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1310,"":"io task","client":"2","":"downstairs","session_id":"e4bd1088-db7d-460e-b74c-98119714c387"} {"msg":"ds_connection connected from Ok(127.0.0.1:47040)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:07.523882272Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1310,"":"io task","client":"1","":"downstairs","session_id":"e4bd1088-db7d-460e-b74c-98119714c387"} {"msg":"ds_connection connected from Ok(127.0.0.1:51457)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:07.523920834Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1310,"":"io task","client":"0","":"downstairs","session_id":"e4bd1088-db7d-460e-b74c-98119714c387"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:07.524481082Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1310,"client":"0","":"downstairs","session_id":"e4bd1088-db7d-460e-b74c-98119714c387"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:07.524643927Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1310,"client":"1","":"downstairs","session_id":"e4bd1088-db7d-460e-b74c-98119714c387"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:07.524675526Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1310,"client":"2","":"downstairs","session_id":"e4bd1088-db7d-460e-b74c-98119714c387"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:07.525155096Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1310,"client":"0","":"downstairs","session_id":"e4bd1088-db7d-460e-b74c-98119714c387"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:07.525187105Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1310,"client":"1","":"downstairs","session_id":"e4bd1088-db7d-460e-b74c-98119714c387"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:07.525460464Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1310,"client":"2","":"downstairs","session_id":"e4bd1088-db7d-460e-b74c-98119714c387"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:07.52569365Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1310,"session_id":"e4bd1088-db7d-460e-b74c-98119714c387"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:07.525767834Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1310,"session_id":"e4bd1088-db7d-460e-b74c-98119714c387"} {"msg":"[0]R flush_numbers[0..12]: [34, 43, 40, 45, 48, 48, 46, 48, 43, 43, 45, 43]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:07.525941385Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1310,"":"downstairs","":"downstairs","session_id":"e4bd1088-db7d-460e-b74c-98119714c387"} {"msg":"[0]R generation[0..12]: [23, 28, 26, 29, 31, 31, 30, 31, 28, 28, 29, 28]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:07.525969031Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1310,"":"downstairs","":"downstairs","session_id":"e4bd1088-db7d-460e-b74c-98119714c387"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:07.52598458Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1310,"":"downstairs","":"downstairs","session_id":"e4bd1088-db7d-460e-b74c-98119714c387"} {"msg":"[1]R flush_numbers[0..12]: [34, 43, 40, 45, 48, 48, 46, 48, 43, 43, 45, 43]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:07.526013568Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1310,"":"downstairs","":"downstairs","session_id":"e4bd1088-db7d-460e-b74c-98119714c387"} {"msg":"[1]R generation[0..12]: [23, 28, 26, 29, 31, 31, 30, 31, 28, 28, 29, 28]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:07.526028506Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1310,"":"downstairs","":"downstairs","session_id":"e4bd1088-db7d-460e-b74c-98119714c387"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:07.526042605Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1310,"":"downstairs","":"downstairs","session_id":"e4bd1088-db7d-460e-b74c-98119714c387"} {"msg":"[2]R flush_numbers[0..12]: [34, 43, 40, 45, 48, 48, 46, 48, 43, 43, 45, 43]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:07.526056102Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1310,"":"downstairs","":"downstairs","session_id":"e4bd1088-db7d-460e-b74c-98119714c387"} {"msg":"[2]R generation[0..12]: [23, 28, 26, 29, 31, 31, 30, 31, 28, 28, 29, 28]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:07.52606913Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1310,"":"downstairs","":"downstairs","session_id":"e4bd1088-db7d-460e-b74c-98119714c387"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:07.526082658Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1310,"":"downstairs","":"downstairs","session_id":"e4bd1088-db7d-460e-b74c-98119714c387"} {"msg":"Max found gen is 32","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-17T21:12:07.526095385Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1310,"":"downstairs","":"downstairs","session_id":"e4bd1088-db7d-460e-b74c-98119714c387"} {"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:07.526114947Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1310,"":"downstairs","":"downstairs","session_id":"e4bd1088-db7d-460e-b74c-98119714c387"} {"msg":"Next flush: 49","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:07.526128144Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1310,"":"downstairs","":"downstairs","session_id":"e4bd1088-db7d-460e-b74c-98119714c387"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:07.52615428Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1310,"":"downstairs","":"downstairs","session_id":"e4bd1088-db7d-460e-b74c-98119714c387"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:07.526167978Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1310,"session_id":"e4bd1088-db7d-460e-b74c-98119714c387"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:07.526179685Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1310,"client":"0","":"downstairs","session_id":"e4bd1088-db7d-460e-b74c-98119714c387"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:07.526192962Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1310,"client":"1","":"downstairs","session_id":"e4bd1088-db7d-460e-b74c-98119714c387"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:07.52620609Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1310,"client":"2","":"downstairs","session_id":"e4bd1088-db7d-460e-b74c-98119714c387"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:07.526219098Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1310,"session_id":"e4bd1088-db7d-460e-b74c-98119714c387"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:07.526230915Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1310,"session_id":"e4bd1088-db7d-460e-b74c-98119714c387"} {"msg":"aa76d2aa-84d3-4e0b-9b7c-278e8fe554ee is now active with session: e4bd1088-db7d-460e-b74c-98119714c387","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:07.526243462Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1310,"session_id":"e4bd1088-db7d-460e-b74c-98119714c387"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:07.526255029Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1310,"session_id":"e4bd1088-db7d-460e-b74c-98119714c387"} {"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:07.526266965Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1310} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 505 len 4096 data: 7 02/30 Write block 11 len 32768 data: 5 6 5 5 4 5 3 3 03/30 Read block 152 len 24576 04/30 Read block 143 len 20480 05/30 Write block 457 len 24576 data: 3 4 3 3 4 6 06/30 Write block 4 len 32768 data: 4 3 2 2 2 2 4 6 07/30 Read block 235 len 28672 08/30 Write block 9 len 20480 data: 3 5 7 7 6 09/30 Read block 324 len 40960 10/30 Read block 135 len 8192 11/30 Write block 92 len 28672 data: 2 2 3 4 4 4 5 12/30 Flush 13/30 Write block 428 len 28672 data: 5 5 4 5 4 3 3 14/30 Read block 565 len 4096 15/30 Read block 443 len 16384 16/30 Read block 55 len 12288 17/30 Read block 0 len 24576 18/30 Read block 559 len 4096 19/30 Read block 165 len 16384 20/30 Read block 209 len 8192 21/30 Write block 72 len 8192 data: 2 2 22/30 Read block 491 len 36864 23/30 Write block 354 len 16384 data: 4 4 5 6 24/30 Read block 158 len 36864 25/30 Read block 573 len 4096 26/30 Read block 254 len 28672 27/30 Read block 538 len 28672 28/30 Read block 383 len 40960 29/30 Write block 113 len 20480 data: 5 6 6 4 4 30/30 Write block 63 len 24576 data: 2 2 3 3 3 3 ---------------------------------------------------------------- Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:18 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1012 Acked Write 7 Done Done Done false 1013 Acked Read 1 Done Done Done false 1014 Acked Read 4 Done Done Done false 1015 Acked Read 3 Done Done Done false 1016 Acked Read 6 Done Done Done false 1017 Acked Read 1 Done Done Done false 1018 Acked Read 4 Done Done Done false 1019 Acked Read 2 Done Done Done false 1020 Acked Write 2 Done Done Done false 1021 Acked Read 9 Done Done Done false 1022 Acked Write 4 Done Done Done false 1023 Acked Read 9 Done Done Done false 1024 Acked Read 1 Done Done Done false 1025 Acked Read 7 Done Done Done false 1026 Acked Read 7 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Write 5 Sent Sent Sent false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 16 16 16 48 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1011 1011 1011 Downstairs last five completed: 1011 1010 1009 1008 1007 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:12:07.545905504Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1310} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 32 32 32 49 49 49 F F F 1 020-039 28 28 28 43 43 43 F F F 2 040-059 26 26 26 40 40 40 F F F 3 060-079 29 29 29 45 45 45 T T T 4 080-099 32 32 32 49 49 49 F F F 5 100-119 31 31 31 48 48 48 T T T 6 120-139 30 30 30 46 46 46 F F F 7 140-159 31 31 31 48 48 48 F F F 8 160-179 28 28 28 43 43 43 F F F 9 180-199 28 28 28 43 43 43 F F F 10 200-219 29 29 29 45 45 45 F F F 11 220-239 28 28 28 43 43 43 F F F 12 240-259 25 25 25 39 39 39 F F F 13 260-279 31 31 31 48 48 48 F F F 14 280-299 30 30 30 47 47 47 F F F 15 300-319 31 31 31 48 48 48 F F F 16 320-339 31 31 31 48 48 48 F F F 17 340-359 30 30 30 47 47 47 T T T 18 360-379 28 28 28 43 43 43 F F F 19 380-399 30 30 30 47 47 47 F F F 20 400-419 30 30 30 47 47 47 F F F 21 420-439 28 28 28 43 43 43 T T T 22 440-459 32 32 32 49 49 49 F F F 23 460-479 32 32 32 49 49 49 F F F 24 480-499 28 28 28 43 43 43 F F F 25 500-519 32 32 32 49 49 49 F F F 26 520-539 28 28 28 43 43 43 F F F 27 540-559 28 28 28 43 43 43 F F F 28 560-579 26 26 26 40 40 40 F F F 29 580-599 19 19 19 31 31 31 F F F Max gen: 32, Max flush: 49 A Difference in extent metadata is expected here On loop 16 Verifying data now No change after verify Loop: 16 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 32 32 32 49 49 49 F F F 1 020-039 28 28 28 43 43 43 F F F 2 040-059 26 26 26 40 40 40 F F F 3 060-079 33 33 33 50 50 50 F F F 4 080-099 32 32 32 49 49 49 F F F 5 100-119 33 33 33 50 50 50 F F F 6 120-139 30 30 30 46 46 46 F F F 7 140-159 31 31 31 48 48 48 F F F 8 160-179 28 28 28 43 43 43 F F F 9 180-199 28 28 28 43 43 43 F F F 10 200-219 29 29 29 45 45 45 F F F 11 220-239 28 28 28 43 43 43 F F F 12 240-259 25 25 25 39 39 39 F F F 13 260-279 31 31 31 48 48 48 F F F 14 280-299 30 30 30 47 47 47 F F F 15 300-319 31 31 31 48 48 48 F F F 16 320-339 31 31 31 48 48 48 F F F 17 340-359 33 33 33 50 50 50 F F F 18 360-379 28 28 28 43 43 43 F F F 19 380-399 30 30 30 47 47 47 F F F 20 400-419 30 30 30 47 47 47 F F F 21 420-439 33 33 33 50 50 50 F F F 22 440-459 32 32 32 49 49 49 F F F 23 460-479 32 32 32 49 49 49 F F F 24 480-499 28 28 28 43 43 43 F F F 25 500-519 32 32 32 49 49 49 F F F 26 520-539 28 28 28 43 43 43 F F F 27 540-559 28 28 28 43 43 43 F F F 28 560-579 26 26 26 40 40 40 F F F 29 580-599 19 19 19 31 31 31 F F F Max gen: 33, Max flush: 50 Begin loop 17 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:11.777036657Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1318,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:11.823157183Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1318,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:11.823983576Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1318} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"abd5430f1508af0bc008143ea2566277ef714ec2\",\n git_commit_timestamp: \"2025-01-17T20:55:03.000000000Z\",\n git_branch: \"alan/tokio-to-140\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:11.824026401Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1318} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:11.824040849Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1318} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:11.824108879Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1318} {"msg":"Crucible 436f9d58-d1b4-4708-b3f8-172b95e59f6a has session id: 6aafb33f-626d-4497-86a9-bde3af64e273","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:11.824642221Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1318,"session_id":"6aafb33f-626d-4497-86a9-bde3af64e273"} {"msg":"Upstairs opts: Upstairs UUID: 436f9d58-d1b4-4708-b3f8-172b95e59f6a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:11.824683135Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1318,"session_id":"6aafb33f-626d-4497-86a9-bde3af64e273"} {"msg":"Crucible stats registered with UUID: 436f9d58-d1b4-4708-b3f8-172b95e59f6a","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:11.824710461Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1318,"session_id":"6aafb33f-626d-4497-86a9-bde3af64e273"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:12:11.824730692Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1318,"session_id":"6aafb33f-626d-4497-86a9-bde3af64e273"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:11.825263864Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1318,"task":"crutest"} {"msg":"The guest has requested activation with gen:34","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:11.825289719Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1318} {"msg":"436f9d58-d1b4-4708-b3f8-172b95e59f6a active request set","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:11.82535958Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1318,"session_id":"6aafb33f-626d-4497-86a9-bde3af64e273"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:11.825378401Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1318,"client":"0","":"downstairs","session_id":"6aafb33f-626d-4497-86a9-bde3af64e273"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:11.82541037Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1318,"client":"0","":"downstairs","session_id":"6aafb33f-626d-4497-86a9-bde3af64e273"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:11.825436725Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1318,"client":"1","":"downstairs","session_id":"6aafb33f-626d-4497-86a9-bde3af64e273"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:11.825465042Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1318,"client":"1","":"downstairs","session_id":"6aafb33f-626d-4497-86a9-bde3af64e273"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:11.825490977Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1318,"client":"2","":"downstairs","session_id":"6aafb33f-626d-4497-86a9-bde3af64e273"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:11.825517953Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1318,"client":"2","":"downstairs","session_id":"6aafb33f-626d-4497-86a9-bde3af64e273"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:11.825543588Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1318,"":"io task","client":"2","":"downstairs","session_id":"6aafb33f-626d-4497-86a9-bde3af64e273"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:11.825569293Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1318,"":"io task","client":"0","":"downstairs","session_id":"6aafb33f-626d-4497-86a9-bde3af64e273"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:11.825591626Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1318,"":"io task","client":"1","":"downstairs","session_id":"6aafb33f-626d-4497-86a9-bde3af64e273"} {"msg":"ds_connection connected from Ok(127.0.0.1:32921)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:11.82561561Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1318,"":"io task","client":"0","":"downstairs","session_id":"6aafb33f-626d-4497-86a9-bde3af64e273"} {"msg":"ds_connection connected from Ok(127.0.0.1:49377)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:11.825639694Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1318,"":"io task","client":"2","":"downstairs","session_id":"6aafb33f-626d-4497-86a9-bde3af64e273"} {"msg":"ds_connection connected from Ok(127.0.0.1:64395)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:11.825682419Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1318,"":"io task","client":"1","":"downstairs","session_id":"6aafb33f-626d-4497-86a9-bde3af64e273"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:11.826329277Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1318,"client":"0","":"downstairs","session_id":"6aafb33f-626d-4497-86a9-bde3af64e273"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:11.826366128Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1318,"client":"1","":"downstairs","session_id":"6aafb33f-626d-4497-86a9-bde3af64e273"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:11.826530885Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1318,"client":"2","":"downstairs","session_id":"6aafb33f-626d-4497-86a9-bde3af64e273"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:11.826954912Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1318,"client":"1","":"downstairs","session_id":"6aafb33f-626d-4497-86a9-bde3af64e273"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:11.826998098Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1318,"client":"0","":"downstairs","session_id":"6aafb33f-626d-4497-86a9-bde3af64e273"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:11.827267955Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1318,"client":"2","":"downstairs","session_id":"6aafb33f-626d-4497-86a9-bde3af64e273"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:11.827496939Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1318,"session_id":"6aafb33f-626d-4497-86a9-bde3af64e273"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:11.827579888Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1318,"session_id":"6aafb33f-626d-4497-86a9-bde3af64e273"} {"msg":"[0]R flush_numbers[0..12]: [49, 43, 40, 50, 49, 50, 46, 48, 43, 43, 45, 43]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:11.827757572Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1318,"":"downstairs","":"downstairs","session_id":"6aafb33f-626d-4497-86a9-bde3af64e273"} {"msg":"[0]R generation[0..12]: [32, 28, 26, 33, 32, 33, 30, 31, 28, 28, 29, 28]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:11.827776212Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1318,"":"downstairs","":"downstairs","session_id":"6aafb33f-626d-4497-86a9-bde3af64e273"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:11.827790791Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1318,"":"downstairs","":"downstairs","session_id":"6aafb33f-626d-4497-86a9-bde3af64e273"} {"msg":"[1]R flush_numbers[0..12]: [49, 43, 40, 50, 49, 50, 46, 48, 43, 43, 45, 43]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:11.827804489Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1318,"":"downstairs","":"downstairs","session_id":"6aafb33f-626d-4497-86a9-bde3af64e273"} {"msg":"[1]R generation[0..12]: [32, 28, 26, 33, 32, 33, 30, 31, 28, 28, 29, 28]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:11.827817717Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1318,"":"downstairs","":"downstairs","session_id":"6aafb33f-626d-4497-86a9-bde3af64e273"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:11.827834196Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1318,"":"downstairs","":"downstairs","session_id":"6aafb33f-626d-4497-86a9-bde3af64e273"} {"msg":"[2]R flush_numbers[0..12]: [49, 43, 40, 50, 49, 50, 46, 48, 43, 43, 45, 43]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:11.827847654Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1318,"":"downstairs","":"downstairs","session_id":"6aafb33f-626d-4497-86a9-bde3af64e273"} {"msg":"[2]R generation[0..12]: [32, 28, 26, 33, 32, 33, 30, 31, 28, 28, 29, 28]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:11.827860822Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1318,"":"downstairs","":"downstairs","session_id":"6aafb33f-626d-4497-86a9-bde3af64e273"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:11.82787445Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1318,"":"downstairs","":"downstairs","session_id":"6aafb33f-626d-4497-86a9-bde3af64e273"} {"msg":"Max found gen is 34","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:11.827887067Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1318,"":"downstairs","":"downstairs","session_id":"6aafb33f-626d-4497-86a9-bde3af64e273"} {"msg":"Generation requested: 34 >= found:34","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:11.827899745Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1318,"":"downstairs","":"downstairs","session_id":"6aafb33f-626d-4497-86a9-bde3af64e273"} {"msg":"Next flush: 51","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:11.827912252Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1318,"":"downstairs","":"downstairs","session_id":"6aafb33f-626d-4497-86a9-bde3af64e273"} Wait for a query_work_queue command to finish before sending IO {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:11.827931663Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1318,"":"downstairs","":"downstairs","session_id":"6aafb33f-626d-4497-86a9-bde3af64e273"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:11.827944641Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1318,"session_id":"6aafb33f-626d-4497-86a9-bde3af64e273"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:11.827956128Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1318,"client":"0","":"downstairs","session_id":"6aafb33f-626d-4497-86a9-bde3af64e273"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:11.827968855Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1318,"client":"1","":"downstairs","session_id":"6aafb33f-626d-4497-86a9-bde3af64e273"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:11.827981383Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1318,"client":"2","":"downstairs","session_id":"6aafb33f-626d-4497-86a9-bde3af64e273"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:11.82799417Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1318,"session_id":"6aafb33f-626d-4497-86a9-bde3af64e273"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:11.828006157Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1318,"session_id":"6aafb33f-626d-4497-86a9-bde3af64e273"} {"msg":"436f9d58-d1b4-4708-b3f8-172b95e59f6a is now active with session: 6aafb33f-626d-4497-86a9-bde3af64e273","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:11.828018534Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1318,"session_id":"6aafb33f-626d-4497-86a9-bde3af64e273"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:11.828031582Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1318,"session_id":"6aafb33f-626d-4497-86a9-bde3af64e273"} {"msg":"The guest has finished waiting for activation with:34","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:11.828043499Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1318} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 558 len 24576 data: 3 3 4 5 5 5 02/30 Read block 322 len 40960 03/30 Read block 84 len 32768 04/30 Read block 90 len 28672 05/30 Write block 507 len 36864 data: 7 6 6 6 3 3 4 4 6 06/30 Read block 188 len 36864 07/30 Write block 531 len 28672 data: 2 2 2 2 2 3 3 08/30 Read block 452 len 12288 09/30 Read block 362 len 12288 10/30 Read block 367 len 40960 11/30 Read block 429 len 12288 12/30 Read block 206 len 8192 13/30 Write block 82 len 20480 data: 2 2 2 2 2 14/30 Flush 15/30 Write block 209 len 4096 data: 3 16/30 Read block 209 len 36864 17/30 Write block 350 len 4096 data: 4 18/30 Write block 132 len 24576 data: 4 3 4 3 4 4 19/30 Write block 475 len 12288 data: 4 3 3 20/30 Write block 452 len 24576 data: 3 2 2 2 3 4 21/30 Write block 557 len 32768 data: 3 4 4 5 6 6 6 4 22/30 Flush 23/30 Write block 258 len 40960 data: 4 4 4 4 4 4 3 4 4 4 24/30 Read block 91 len 28672 25/30 Read block 372 len 4096 26/30 Read block 160 len 16384 27/30 Write block 465 len 12288 data: 7 5 8 28/30 Read block 292 len 24576 29/30 Read block 249 len 24576 30/30 Read block 283 len 8192 ---------------------------------------------------------------- Crucible gen:34 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 10 Done Done Done false 1023 Acked Read 7 Done Done Done false 1024 Acked Read 1 Done Done Done false 1025 Acked Read 4 Done Done Done false 1026 Acked Write 3 Done Done Done false 1027 Acked Read 6 Done Done Done false 1028 Acked Read 6 Done Done Done false 1029 Acked Read 2 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 8 8 8 24 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1021 1021 1021 Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:12:11.848258083Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1318} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 32 32 32 49 49 49 F F F 1 020-039 28 28 28 43 43 43 F F F 2 040-059 26 26 26 40 40 40 F F F 3 060-079 33 33 33 50 50 50 F F F 4 080-099 34 34 34 51 51 51 F F F 5 100-119 33 33 33 50 50 50 F F F 6 120-139 34 34 34 52 52 52 F F F 7 140-159 31 31 31 48 48 48 F F F 8 160-179 28 28 28 43 43 43 F F F 9 180-199 28 28 28 43 43 43 F F F 10 200-219 34 34 34 52 52 52 F F F 11 220-239 28 28 28 43 43 43 F F F 12 240-259 25 25 25 39 39 39 T T T 13 260-279 31 31 31 48 48 48 T T T 14 280-299 30 30 30 47 47 47 F F F 15 300-319 31 31 31 48 48 48 F F F 16 320-339 31 31 31 48 48 48 F F F 17 340-359 34 34 34 52 52 52 F F F 18 360-379 28 28 28 43 43 43 F F F 19 380-399 30 30 30 47 47 47 F F F 20 400-419 30 30 30 47 47 47 F F F 21 420-439 33 33 33 50 50 50 F F F 22 440-459 34 34 34 52 52 52 F F F 23 460-479 34 34 34 52 52 52 T T T 24 480-499 28 28 28 43 43 43 F F F 25 500-519 34 34 34 51 51 51 F F F 26 520-539 34 34 34 51 51 51 F F F 27 540-559 34 34 34 52 52 52 F F F 28 560-579 34 34 34 52 52 52 F F F 29 580-599 19 19 19 31 31 31 F F F Max gen: 34, Max flush: 52 A Difference in extent metadata is expected here On loop 17 Verifying data now No change after verify Loop: 17 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 32 32 32 49 49 49 F F F 1 020-039 28 28 28 43 43 43 F F F 2 040-059 26 26 26 40 40 40 F F F 3 060-079 33 33 33 50 50 50 F F F 4 080-099 34 34 34 51 51 51 F F F 5 100-119 33 33 33 50 50 50 F F F 6 120-139 34 34 34 52 52 52 F F F 7 140-159 31 31 31 48 48 48 F F F 8 160-179 28 28 28 43 43 43 F F F 9 180-199 28 28 28 43 43 43 F F F 10 200-219 34 34 34 52 52 52 F F F 11 220-239 28 28 28 43 43 43 F F F 12 240-259 35 35 35 53 53 53 F F F 13 260-279 35 35 35 53 53 53 F F F 14 280-299 30 30 30 47 47 47 F F F 15 300-319 31 31 31 48 48 48 F F F 16 320-339 31 31 31 48 48 48 F F F 17 340-359 34 34 34 52 52 52 F F F 18 360-379 28 28 28 43 43 43 F F F 19 380-399 30 30 30 47 47 47 F F F 20 400-419 30 30 30 47 47 47 F F F 21 420-439 33 33 33 50 50 50 F F F 22 440-459 34 34 34 52 52 52 F F F 23 460-479 35 35 35 53 53 53 F F F 24 480-499 28 28 28 43 43 43 F F F 25 500-519 34 34 34 51 51 51 F F F 26 520-539 34 34 34 51 51 51 F F F 27 540-559 34 34 34 52 52 52 F F F 28 560-579 34 34 34 52 52 52 F F F 29 580-599 19 19 19 31 31 31 F F F Max gen: 35, Max flush: 53 Begin loop 18 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:16.073944666Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1326,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:16.117598671Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1326,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:16.118237384Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1326} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"abd5430f1508af0bc008143ea2566277ef714ec2\",\n git_commit_timestamp: \"2025-01-17T20:55:03.000000000Z\",\n git_branch: \"alan/tokio-to-140\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:16.118278418Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1326} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:16.118292696Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1326} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:16.118310547Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1326} {"msg":"Crucible e327605d-0375-4527-9a3a-3d9f03696d98 has session id: 8a73efbf-3f99-445a-a0b1-54f6ec585b32","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:16.118826919Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1326,"session_id":"8a73efbf-3f99-445a-a0b1-54f6ec585b32"} {"msg":"Upstairs opts: Upstairs UUID: e327605d-0375-4527-9a3a-3d9f03696d98, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:16.118875787Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1326,"session_id":"8a73efbf-3f99-445a-a0b1-54f6ec585b32"} {"msg":"Crucible stats registered with UUID: e327605d-0375-4527-9a3a-3d9f03696d98","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:16.118895289Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1326,"session_id":"8a73efbf-3f99-445a-a0b1-54f6ec585b32"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:12:16.118908447Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1326,"session_id":"8a73efbf-3f99-445a-a0b1-54f6ec585b32"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:16.119433633Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1326,"task":"crutest"} {"msg":"The guest has requested activation with gen:36","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:16.119466092Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1326} {"msg":"e327605d-0375-4527-9a3a-3d9f03696d98 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:16.119526928Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1326,"session_id":"8a73efbf-3f99-445a-a0b1-54f6ec585b32"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:16.119553273Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1326,"client":"0","":"downstairs","session_id":"8a73efbf-3f99-445a-a0b1-54f6ec585b32"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:16.119575646Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1326,"client":"0","":"downstairs","session_id":"8a73efbf-3f99-445a-a0b1-54f6ec585b32"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:16.119589485Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1326,"client":"1","":"downstairs","session_id":"8a73efbf-3f99-445a-a0b1-54f6ec585b32"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:16.119603403Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1326,"client":"1","":"downstairs","session_id":"8a73efbf-3f99-445a-a0b1-54f6ec585b32"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:16.11961625Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1326,"client":"2","":"downstairs","session_id":"8a73efbf-3f99-445a-a0b1-54f6ec585b32"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:16.119634031Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1326,"client":"2","":"downstairs","session_id":"8a73efbf-3f99-445a-a0b1-54f6ec585b32"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:16.119662647Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1326,"":"io task","client":"2","":"downstairs","session_id":"8a73efbf-3f99-445a-a0b1-54f6ec585b32"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:16.119688132Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1326,"":"io task","client":"0","":"downstairs","session_id":"8a73efbf-3f99-445a-a0b1-54f6ec585b32"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:16.119709034Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1326,"":"io task","client":"1","":"downstairs","session_id":"8a73efbf-3f99-445a-a0b1-54f6ec585b32"} {"msg":"ds_connection connected from Ok(127.0.0.1:64100)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:16.119725564Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1326,"":"io task","client":"2","":"downstairs","session_id":"8a73efbf-3f99-445a-a0b1-54f6ec585b32"} {"msg":"ds_connection connected from Ok(127.0.0.1:45829)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:16.11995876Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1326,"":"io task","client":"0","":"downstairs","session_id":"8a73efbf-3f99-445a-a0b1-54f6ec585b32"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:12:16.119999934Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1326,"":"io task","client":"1","":"downstairs","session_id":"8a73efbf-3f99-445a-a0b1-54f6ec585b32"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:12:16.120018475Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1326,"":"io task","client":"1","":"downstairs","session_id":"8a73efbf-3f99-445a-a0b1-54f6ec585b32"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:12:16.120033854Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1326,"session_id":"8a73efbf-3f99-445a-a0b1-54f6ec585b32"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:16.120176157Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1326,"":"io task","client":"1","":"downstairs","session_id":"8a73efbf-3f99-445a-a0b1-54f6ec585b32"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:16.120404991Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1326,"client":"2","":"downstairs","session_id":"8a73efbf-3f99-445a-a0b1-54f6ec585b32"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:16.120536398Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1326,"client":"0","":"downstairs","session_id":"8a73efbf-3f99-445a-a0b1-54f6ec585b32"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:16.120924194Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1326,"client":"2","":"downstairs","session_id":"8a73efbf-3f99-445a-a0b1-54f6ec585b32"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:16.121031828Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1326,"client":"0","":"downstairs","session_id":"8a73efbf-3f99-445a-a0b1-54f6ec585b32"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:16.121478629Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1326,"session_id":"8a73efbf-3f99-445a-a0b1-54f6ec585b32"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:16.12155002Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1326,"session_id":"8a73efbf-3f99-445a-a0b1-54f6ec585b32"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:26.12129033Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1326,"":"io task","client":"1","":"downstairs","session_id":"8a73efbf-3f99-445a-a0b1-54f6ec585b32"} {"msg":"ds_connection connected from Ok(127.0.0.1:37632)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:26.121643177Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1326,"":"io task","client":"1","":"downstairs","session_id":"8a73efbf-3f99-445a-a0b1-54f6ec585b32"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:26.122698403Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1326,"client":"1","":"downstairs","session_id":"8a73efbf-3f99-445a-a0b1-54f6ec585b32"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:26.123290319Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1326,"client":"1","":"downstairs","session_id":"8a73efbf-3f99-445a-a0b1-54f6ec585b32"} {"msg":"[0]R flush_numbers[0..12]: [49, 43, 40, 50, 51, 50, 52, 48, 43, 43, 52, 43]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:26.123925351Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1326,"":"downstairs","":"downstairs","session_id":"8a73efbf-3f99-445a-a0b1-54f6ec585b32"} {"msg":"[0]R generation[0..12]: [32, 28, 26, 33, 34, 33, 34, 31, 28, 28, 34, 28]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:26.123949265Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1326,"":"downstairs","":"downstairs","session_id":"8a73efbf-3f99-445a-a0b1-54f6ec585b32"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:26.123964553Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1326,"":"downstairs","":"downstairs","session_id":"8a73efbf-3f99-445a-a0b1-54f6ec585b32"} {"msg":"[1]R flush_numbers[0..12]: [49, 43, 40, 50, 51, 50, 52, 48, 43, 43, 52, 43]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:26.123978442Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1326,"":"downstairs","":"downstairs","session_id":"8a73efbf-3f99-445a-a0b1-54f6ec585b32"} {"msg":"[1]R generation[0..12]: [32, 28, 26, 33, 34, 33, 34, 31, 28, 28, 34, 28]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:26.123991979Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1326,"":"downstairs","":"downstairs","session_id":"8a73efbf-3f99-445a-a0b1-54f6ec585b32"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:26.124015283Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1326,"":"downstairs","":"downstairs","session_id":"8a73efbf-3f99-445a-a0b1-54f6ec585b32"} {"msg":"[2]R flush_numbers[0..12]: [49, 43, 40, 50, 51, 50, 52, 48, 43, 43, 52, 43]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:26.124034864Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1326,"":"downstairs","":"downstairs","session_id":"8a73efbf-3f99-445a-a0b1-54f6ec585b32"} {"msg":"[2]R generation[0..12]: [32, 28, 26, 33, 34, 33, 34, 31, 28, 28, 34, 28]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:26.124048512Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1326,"":"downstairs","":"downstairs","session_id":"8a73efbf-3f99-445a-a0b1-54f6ec585b32"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:26.12406213Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1326,"":"downstairs","":"downstairs","session_id":"8a73efbf-3f99-445a-a0b1-54f6ec585b32"} {"msg":"Max found gen is 36","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:26.12407935Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1326,"":"downstairs","":"downstairs","session_id":"8a73efbf-3f99-445a-a0b1-54f6ec585b32"} {"msg":"Generation requested: 36 >= found:36","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:26.124092578Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1326,"":"downstairs","":"downstairs","session_id":"8a73efbf-3f99-445a-a0b1-54f6ec585b32"} {"msg":"Next flush: 54","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:26.124105286Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1326,"":"downstairs","":"downstairs","session_id":"8a73efbf-3f99-445a-a0b1-54f6ec585b32"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:26.124117783Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1326,"":"downstairs","":"downstairs","session_id":"8a73efbf-3f99-445a-a0b1-54f6ec585b32"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-17T21:12:26.124132672Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1326,"session_id":"8a73efbf-3f99-445a-a0b1-54f6ec585b32"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:26.124157726Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1326,"client":"0","":"downstairs","session_id":"8a73efbf-3f99-445a-a0b1-54f6ec585b32"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:26.124182801Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1326,"client":"1","":"downstairs","session_id":"8a73efbf-3f99-445a-a0b1-54f6ec585b32"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:26.124207856Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1326,"client":"2","":"downstairs","session_id":"8a73efbf-3f99-445a-a0b1-54f6ec585b32"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:26.124233311Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1326,"session_id":"8a73efbf-3f99-445a-a0b1-54f6ec585b32"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:26.124257915Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1326,"session_id":"8a73efbf-3f99-445a-a0b1-54f6ec585b32"} {"msg":"e327605d-0375-4527-9a3a-3d9f03696d98 is now active with session: 8a73efbf-3f99-445a-a0b1-54f6ec585b32","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:26.12428324Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1326,"session_id":"8a73efbf-3f99-445a-a0b1-54f6ec585b32"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:26.124307504Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1326,"session_id":"8a73efbf-3f99-445a-a0b1-54f6ec585b32"} {"msg":"The guest has finished waiting for activation with:36","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:26.124332538Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1326} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 334 len 28672 data: 4 5 5 4 4 4 5 02/30 Flush 03/30 Write block 490 len 8192 data: 2 2 04/30 Write block 415 len 4096 data: 3 05/30 Write block 75 len 4096 data: 4 06/30 Read block 442 len 36864 07/30 Read block 356 len 20480 08/30 Read block 500 len 4096 09/30 Write block 71 len 20480 data: 3 3 3 3 5 10/30 Read block 34 len 40960 11/30 Write block 576 len 20480 data: 3 3 2 3 3 12/30 Write block 281 len 24576 data: 5 5 3 3 3 2 13/30 Flush 14/30 Write block 136 len 36864 data: 5 5 4 5 5 4 4 4 4 15/30 Read block 264 len 40960 16/30 Read block 448 len 12288 17/30 Read block 70 len 32768 18/30 Read block 540 len 20480 19/30 Write block 465 len 20480 data: 8 6 9 7 7 20/30 Write block 60 len 12288 data: 3 3 2 21/30 Read block 557 len 4096 22/30 Flush 23/30 Write block 216 len 8192 data: 3 3 24/30 Read block 70 len 24576 25/30 Read block 510 len 32768 26/30 Flush 27/30 Write block 518 len 40960 data: 5 6 6 6 4 6 5 4 4 3 28/30 Write block 232 len 12288 data: 3 3 3 29/30 Write block 493 len 16384 data: 2 2 2 3 30/30 Read block 74 len 12288 ---------------------------------------------------------------- Crucible gen:36 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 10 Done Done Done false 1027 Acked Write 3 Done Done Done false 1028 Acked Write 4 Done Done Done false 1029 Acked Read 3 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 0 1 Done 4 3 4 11 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:12:26.143210362Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1326} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 32 32 32 49 49 49 F F F 1 020-039 28 28 28 43 43 43 F F F 2 040-059 26 26 26 40 40 40 F F F 3 060-079 36 36 36 56 56 56 F F F 4 080-099 34 34 34 51 51 51 F F F 5 100-119 33 33 33 50 50 50 F F F 6 120-139 36 36 36 56 56 56 F F F 7 140-159 36 36 36 56 56 56 F F F 8 160-179 28 28 28 43 43 43 F F F 9 180-199 28 28 28 43 43 43 F F F 10 200-219 36 36 36 57 57 57 F F F 11 220-239 28 28 28 43 43 43 T T T 12 240-259 35 35 35 53 53 53 F F F 13 260-279 35 35 35 53 53 53 F F F 14 280-299 36 36 36 55 55 55 F F F 15 300-319 31 31 31 48 48 48 F F F 16 320-339 36 36 36 54 54 54 F F F 17 340-359 36 36 36 54 54 54 F F F 18 360-379 28 28 28 43 43 43 F F F 19 380-399 30 30 30 47 47 47 F F F 20 400-419 36 36 36 55 55 55 F F F 21 420-439 33 33 33 50 50 50 F F F 22 440-459 34 34 34 52 52 52 F F F 23 460-479 36 36 36 56 56 56 F F F 24 480-499 36 36 36 55 55 55 T T T 25 500-519 34 34 34 51 51 51 T T T 26 520-539 34 34 34 51 51 51 T T T 27 540-559 34 34 34 52 52 52 F F F 28 560-579 36 36 36 55 55 55 F F F 29 580-599 36 36 36 55 55 55 F F F Max gen: 36, Max flush: 57 A Difference in extent metadata is expected here On loop 18 Verifying data now No change after verify Loop: 18 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 32 32 32 49 49 49 F F F 1 020-039 28 28 28 43 43 43 F F F 2 040-059 26 26 26 40 40 40 F F F 3 060-079 36 36 36 56 56 56 F F F 4 080-099 34 34 34 51 51 51 F F F 5 100-119 33 33 33 50 50 50 F F F 6 120-139 36 36 36 56 56 56 F F F 7 140-159 36 36 36 56 56 56 F F F 8 160-179 28 28 28 43 43 43 F F F 9 180-199 28 28 28 43 43 43 F F F 10 200-219 36 36 36 57 57 57 F F F 11 220-239 37 37 37 58 58 58 F F F 12 240-259 35 35 35 53 53 53 F F F 13 260-279 35 35 35 53 53 53 F F F 14 280-299 36 36 36 55 55 55 F F F 15 300-319 31 31 31 48 48 48 F F F 16 320-339 36 36 36 54 54 54 F F F 17 340-359 36 36 36 54 54 54 F F F 18 360-379 28 28 28 43 43 43 F F F 19 380-399 30 30 30 47 47 47 F F F 20 400-419 36 36 36 55 55 55 F F F 21 420-439 33 33 33 50 50 50 F F F 22 440-459 34 34 34 52 52 52 F F F 23 460-479 36 36 36 56 56 56 F F F 24 480-499 37 37 37 58 58 58 F F F 25 500-519 37 37 37 58 58 58 F F F 26 520-539 37 37 37 58 58 58 F F F 27 540-559 34 34 34 52 52 52 F F F 28 560-579 36 36 36 55 55 55 F F F 29 580-599 36 36 36 55 55 55 F F F Max gen: 37, Max flush: 58 Begin loop 19 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:30.376428387Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1334,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:12:30.419375978Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1334,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:30.419801727Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1334,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:30.451478431Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1334,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:30.452121627Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1334} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"abd5430f1508af0bc008143ea2566277ef714ec2\",\n git_commit_timestamp: \"2025-01-17T20:55:03.000000000Z\",\n git_branch: \"alan/tokio-to-140\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:30.45216087Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1334} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:30.452176779Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1334} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:30.4521944Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1334} {"msg":"Crucible ab6425ee-c5e7-41cb-bdda-9e35a6aac112 has session id: 1483a152-93ae-4e22-8fa6-4f17c5f00467","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:30.452709741Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1334,"session_id":"1483a152-93ae-4e22-8fa6-4f17c5f00467"} {"msg":"Upstairs opts: Upstairs UUID: ab6425ee-c5e7-41cb-bdda-9e35a6aac112, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:30.452746953Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1334,"session_id":"1483a152-93ae-4e22-8fa6-4f17c5f00467"} {"msg":"Crucible stats registered with UUID: ab6425ee-c5e7-41cb-bdda-9e35a6aac112","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:30.452773979Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1334,"session_id":"1483a152-93ae-4e22-8fa6-4f17c5f00467"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:12:30.452794921Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1334,"session_id":"1483a152-93ae-4e22-8fa6-4f17c5f00467"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:30.453186059Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1334,"task":"crutest"} {"msg":"The guest has requested activation with gen:38","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:30.453218128Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1334} {"msg":"ab6425ee-c5e7-41cb-bdda-9e35a6aac112 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:30.453273691Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1334,"session_id":"1483a152-93ae-4e22-8fa6-4f17c5f00467"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:30.453299606Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1334,"client":"0","":"downstairs","session_id":"1483a152-93ae-4e22-8fa6-4f17c5f00467"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:30.45332437Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1334,"client":"0","":"downstairs","session_id":"1483a152-93ae-4e22-8fa6-4f17c5f00467"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:30.453338999Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1334,"client":"1","":"downstairs","session_id":"1483a152-93ae-4e22-8fa6-4f17c5f00467"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:30.453364113Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1334,"client":"1","":"downstairs","session_id":"1483a152-93ae-4e22-8fa6-4f17c5f00467"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:30.453389478Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1334,"client":"2","":"downstairs","session_id":"1483a152-93ae-4e22-8fa6-4f17c5f00467"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:30.453417154Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1334,"client":"2","":"downstairs","session_id":"1483a152-93ae-4e22-8fa6-4f17c5f00467"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:30.453440888Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1334,"":"io task","client":"2","":"downstairs","session_id":"1483a152-93ae-4e22-8fa6-4f17c5f00467"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:30.453468094Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1334,"":"io task","client":"0","":"downstairs","session_id":"1483a152-93ae-4e22-8fa6-4f17c5f00467"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:30.453492519Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1334,"":"io task","client":"1","":"downstairs","session_id":"1483a152-93ae-4e22-8fa6-4f17c5f00467"} {"msg":"ds_connection connected from Ok(127.0.0.1:58378)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:30.453516763Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1334,"":"io task","client":"2","":"downstairs","session_id":"1483a152-93ae-4e22-8fa6-4f17c5f00467"} {"msg":"ds_connection connected from Ok(127.0.0.1:32995)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:30.453543718Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1334,"":"io task","client":"1","":"downstairs","session_id":"1483a152-93ae-4e22-8fa6-4f17c5f00467"} {"msg":"ds_connection connected from Ok(127.0.0.1:37570)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:30.453575777Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1334,"":"io task","client":"0","":"downstairs","session_id":"1483a152-93ae-4e22-8fa6-4f17c5f00467"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:30.454164811Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1334,"client":"1","":"downstairs","session_id":"1483a152-93ae-4e22-8fa6-4f17c5f00467"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:30.454195229Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1334,"client":"2","":"downstairs","session_id":"1483a152-93ae-4e22-8fa6-4f17c5f00467"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:30.454383989Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1334,"client":"0","":"downstairs","session_id":"1483a152-93ae-4e22-8fa6-4f17c5f00467"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:30.454890636Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1334,"client":"1","":"downstairs","session_id":"1483a152-93ae-4e22-8fa6-4f17c5f00467"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:30.454918522Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1334,"client":"2","":"downstairs","session_id":"1483a152-93ae-4e22-8fa6-4f17c5f00467"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:30.454933801Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1334,"client":"0","":"downstairs","session_id":"1483a152-93ae-4e22-8fa6-4f17c5f00467"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:30.455472426Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1334,"session_id":"1483a152-93ae-4e22-8fa6-4f17c5f00467"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:30.455572874Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1334,"session_id":"1483a152-93ae-4e22-8fa6-4f17c5f00467"} {"msg":"[0]R flush_numbers[0..12]: [49, 43, 40, 56, 51, 50, 56, 56, 43, 43, 57, 58]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:30.455641555Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1334,"":"downstairs","":"downstairs","session_id":"1483a152-93ae-4e22-8fa6-4f17c5f00467"} {"msg":"[0]R generation[0..12]: [32, 28, 26, 36, 34, 33, 36, 36, 28, 28, 36, 37]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:30.455661386Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1334,"":"downstairs","":"downstairs","session_id":"1483a152-93ae-4e22-8fa6-4f17c5f00467"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:30.455675764Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1334,"":"downstairs","":"downstairs","session_id":"1483a152-93ae-4e22-8fa6-4f17c5f00467"} {"msg":"[1]R flush_numbers[0..12]: [49, 43, 40, 56, 51, 50, 56, 56, 43, 43, 57, 58]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:30.455689343Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1334,"":"downstairs","":"downstairs","session_id":"1483a152-93ae-4e22-8fa6-4f17c5f00467"} {"msg":"[1]R generation[0..12]: [32, 28, 26, 36, 34, 33, 36, 36, 28, 28, 36, 37]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:30.45570235Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1334,"":"downstairs","":"downstairs","session_id":"1483a152-93ae-4e22-8fa6-4f17c5f00467"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:30.45571915Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1334,"":"downstairs","":"downstairs","session_id":"1483a152-93ae-4e22-8fa6-4f17c5f00467"} {"msg":"[2]R flush_numbers[0..12]: [49, 43, 40, 56, 51, 50, 56, 56, 43, 43, 57, 58]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:30.455732628Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1334,"":"downstairs","":"downstairs","session_id":"1483a152-93ae-4e22-8fa6-4f17c5f00467"} {"msg":"[2]R generation[0..12]: [32, 28, 26, 36, 34, 33, 36, 36, 28, 28, 36, 37]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:30.455746246Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1334,"":"downstairs","":"downstairs","session_id":"1483a152-93ae-4e22-8fa6-4f17c5f00467"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:30.455760114Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1334,"":"downstairs","":"downstairs","session_id":"1483a152-93ae-4e22-8fa6-4f17c5f00467"} {"msg":"Max found gen is 38","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:30.455772841Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1334,"":"downstairs","":"downstairs","session_id":"1483a152-93ae-4e22-8fa6-4f17c5f00467"} {"msg":"Generation requested: 38 >= found:38","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:30.455786119Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1334,"":"downstairs","":"downstairs","session_id":"1483a152-93ae-4e22-8fa6-4f17c5f00467"} {"msg":"Next flush: 59","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-17T21:12:30.455798726Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1334,"":"downstairs","":"downstairs","session_id":"1483a152-93ae-4e22-8fa6-4f17c5f00467"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:30.455814996Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1334,"":"downstairs","":"downstairs","session_id":"1483a152-93ae-4e22-8fa6-4f17c5f00467"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:30.455827483Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1334,"session_id":"1483a152-93ae-4e22-8fa6-4f17c5f00467"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:30.45583904Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1334,"client":"0","":"downstairs","session_id":"1483a152-93ae-4e22-8fa6-4f17c5f00467"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:30.455851958Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1334,"client":"1","":"downstairs","session_id":"1483a152-93ae-4e22-8fa6-4f17c5f00467"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:30.455864785Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1334,"client":"2","":"downstairs","session_id":"1483a152-93ae-4e22-8fa6-4f17c5f00467"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:30.455877673Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1334,"session_id":"1483a152-93ae-4e22-8fa6-4f17c5f00467"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:30.45588964Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1334,"session_id":"1483a152-93ae-4e22-8fa6-4f17c5f00467"} {"msg":"ab6425ee-c5e7-41cb-bdda-9e35a6aac112 is now active with session: 1483a152-93ae-4e22-8fa6-4f17c5f00467","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:30.455902087Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1334,"session_id":"1483a152-93ae-4e22-8fa6-4f17c5f00467"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:30.455915595Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1334,"session_id":"1483a152-93ae-4e22-8fa6-4f17c5f00467"} {"msg":"The guest has finished waiting for activation with:38","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:30.455927602Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1334} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 352 len 32768 data: 4 4 5 5 6 7 6 6 02/30 Flush 03/30 Write block 499 len 28672 data: 4 5 6 6 7 8 8 04/30 Write block 109 len 4096 data: 4 05/30 Write block 425 len 12288 data: 3 4 4 06/30 Write block 235 len 40960 data: 4 4 4 2 2 2 3 4 4 5 07/30 Read block 539 len 20480 08/30 Read block 252 len 36864 09/30 Read block 595 len 4096 10/30 Read block 65 len 40960 11/30 Read block 512 len 4096 12/30 Write block 358 len 20480 data: 7 7 5 4 4 13/30 Read block 38 len 20480 14/30 Read block 17 len 28672 15/30 Write block 154 len 36864 data: 2 2 3 3 3 3 3 2 2 16/30 Read block 197 len 4096 17/30 Write block 509 len 32768 data: 7 7 4 4 5 5 7 7 18/30 Read block 373 len 16384 19/30 Read block 207 len 40960 20/30 Write block 514 len 20480 data: 6 8 8 5 6 21/30 Read block 473 len 28672 22/30 Read block 24 len 36864 23/30 Write block 98 len 32768 data: 6 5 5 5 5 4 4 4 24/30 Write block 401 len 12288 data: 5 4 6 25/30 Read block 224 len 36864 26/30 Write block 366 len 36864 data: 3 2 2 2 2 2 5 5 6 27/30 Write block 201 len 32768 data: 4 3 3 3 3 2 3 3 28/30 Read block 531 len 4096 29/30 Write block 280 len 36864 data: 5 6 6 4 4 4 3 2 2 30/30 Read block 589 len 4096 ---------------------------------------------------------------- Crucible gen:38 GIO:true work queues: Upstairs:0 downstairs:28 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1002 Acked Write 7 Done Done Done false 1003 Acked Write 1 Done Done Done false 1004 Acked Write 3 Done Done Done false 1005 Acked Write 10 Done Done Done false 1006 Acked Read 5 Done Done Done false 1007 Acked Read 9 Done Done Done false 1008 Acked Read 1 Done Done Done false 1009 Acked Read 10 Done Done Done false 1010 Acked Read 1 Done Done Done false 1011 Acked Write 5 Done Done Done false 1012 Acked Read 5 Done Done Done false 1013 Acked Read 7 Done Done Done false 1014 Acked Write 9 Done Done Done false 1015 Acked Read 1 Done Done Done false 1016 Acked Write 8 Done Done Done false 1017 Acked Read 4 Done Done Done false 1018 Acked Read 10 Done Done Done false 1019 Acked Write 5 Done Done Done false 1020 Acked Read 7 Done Done Done false 1021 Acked Read 9 Done Done Done false 1022 Acked Write 8 Done Done Done false 1023 Acked Write 3 Done Done Done false 1024 Acked Read 9 Done Done Done false 1025 Acked Write 9 Done Done Done false 1026 Acked Write 8 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Write 9 Done Done Done false 1029 Acked Read 1 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 28 28 28 84 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1001 1001 1001 Downstairs last five completed: 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:12:30.476916349Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1334} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 32 32 32 49 49 49 F F F 1 020-039 28 28 28 43 43 43 F F F 2 040-059 26 26 26 40 40 40 F F F 3 060-079 36 36 36 56 56 56 F F F 4 080-099 34 34 34 51 51 51 T T T 5 100-119 33 33 33 50 50 50 T T T 6 120-139 36 36 36 56 56 56 F F F 7 140-159 36 36 36 56 56 56 T T T 8 160-179 28 28 28 43 43 43 T T T 9 180-199 28 28 28 43 43 43 F F F 10 200-219 36 36 36 57 57 57 T T T 11 220-239 37 37 37 58 58 58 T T T 12 240-259 35 35 35 53 53 53 T T T 13 260-279 35 35 35 53 53 53 F F F 14 280-299 36 36 36 55 55 55 T T T 15 300-319 31 31 31 48 48 48 F F F 16 320-339 36 36 36 54 54 54 F F F 17 340-359 38 38 38 59 59 59 T T T 18 360-379 28 28 28 43 43 43 T T T 19 380-399 30 30 30 47 47 47 F F F 20 400-419 36 36 36 55 55 55 T T T 21 420-439 33 33 33 50 50 50 T T T 22 440-459 34 34 34 52 52 52 F F F 23 460-479 36 36 36 56 56 56 F F F 24 480-499 37 37 37 58 58 58 T T T 25 500-519 37 37 37 58 58 58 T T T 26 520-539 37 37 37 58 58 58 F F F 27 540-559 34 34 34 52 52 52 F F F 28 560-579 36 36 36 55 55 55 F F F 29 580-599 36 36 36 55 55 55 F F F Max gen: 38, Max flush: 59 A Difference in extent metadata is expected here On loop 19 Verifying data now No change after verify Loop: 19 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 32 32 32 49 49 49 F F F 1 020-039 28 28 28 43 43 43 F F F 2 040-059 26 26 26 40 40 40 F F F 3 060-079 36 36 36 56 56 56 F F F 4 080-099 39 39 39 60 60 60 F F F 5 100-119 39 39 39 60 60 60 F F F 6 120-139 36 36 36 56 56 56 F F F 7 140-159 39 39 39 60 60 60 F F F 8 160-179 39 39 39 60 60 60 F F F 9 180-199 28 28 28 43 43 43 F F F 10 200-219 39 39 39 60 60 60 F F F 11 220-239 39 39 39 60 60 60 F F F 12 240-259 39 39 39 60 60 60 F F F 13 260-279 35 35 35 53 53 53 F F F 14 280-299 39 39 39 60 60 60 F F F 15 300-319 31 31 31 48 48 48 F F F 16 320-339 36 36 36 54 54 54 F F F 17 340-359 39 39 39 60 60 60 F F F 18 360-379 39 39 39 60 60 60 F F F 19 380-399 30 30 30 47 47 47 F F F 20 400-419 39 39 39 60 60 60 F F F 21 420-439 39 39 39 60 60 60 F F F 22 440-459 34 34 34 52 52 52 F F F 23 460-479 36 36 36 56 56 56 F F F 24 480-499 39 39 39 60 60 60 F F F 25 500-519 39 39 39 60 60 60 F F F 26 520-539 37 37 37 58 58 58 F F F 27 540-559 34 34 34 52 52 52 F F F 28 560-579 36 36 36 55 55 55 F F F 29 580-599 36 36 36 55 55 55 F F F Max gen: 39, Max flush: 60 Begin loop 20 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:34.794834817Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1342,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:34.839047228Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1342,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:34.839664189Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1342} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"abd5430f1508af0bc008143ea2566277ef714ec2\",\n git_commit_timestamp: \"2025-01-17T20:55:03.000000000Z\",\n git_branch: \"alan/tokio-to-140\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:34.839703872Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1342} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:34.839718431Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1342} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:34.839732599Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1342} {"msg":"Crucible 2095efb0-1d24-4ee7-96ec-dbeb6c709104 has session id: 06a1b2ef-be1c-4453-ab0b-6a33279acb1f","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:34.840301071Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1342,"session_id":"06a1b2ef-be1c-4453-ab0b-6a33279acb1f"} {"msg":"Upstairs opts: Upstairs UUID: 2095efb0-1d24-4ee7-96ec-dbeb6c709104, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:34.840336082Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1342,"session_id":"06a1b2ef-be1c-4453-ab0b-6a33279acb1f"} {"msg":"Crucible stats registered with UUID: 2095efb0-1d24-4ee7-96ec-dbeb6c709104","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:34.840361597Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1342,"session_id":"06a1b2ef-be1c-4453-ab0b-6a33279acb1f"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:12:34.840383159Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1342,"session_id":"06a1b2ef-be1c-4453-ab0b-6a33279acb1f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:34.840756958Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1342,"task":"crutest"} {"msg":"The guest has requested activation with gen:40","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:34.840780731Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1342} {"msg":"2095efb0-1d24-4ee7-96ec-dbeb6c709104 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:34.840849131Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1342,"session_id":"06a1b2ef-be1c-4453-ab0b-6a33279acb1f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:34.840865771Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1342,"client":"0","":"downstairs","session_id":"06a1b2ef-be1c-4453-ab0b-6a33279acb1f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:34.840889645Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1342,"client":"0","":"downstairs","session_id":"06a1b2ef-be1c-4453-ab0b-6a33279acb1f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:34.840906865Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1342,"client":"1","":"downstairs","session_id":"06a1b2ef-be1c-4453-ab0b-6a33279acb1f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:34.840921394Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1342,"client":"1","":"downstairs","session_id":"06a1b2ef-be1c-4453-ab0b-6a33279acb1f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:34.840934401Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1342,"client":"2","":"downstairs","session_id":"06a1b2ef-be1c-4453-ab0b-6a33279acb1f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:34.840948249Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1342,"client":"2","":"downstairs","session_id":"06a1b2ef-be1c-4453-ab0b-6a33279acb1f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:34.840964389Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1342,"":"io task","client":"2","":"downstairs","session_id":"06a1b2ef-be1c-4453-ab0b-6a33279acb1f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:34.840986702Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1342,"":"io task","client":"0","":"downstairs","session_id":"06a1b2ef-be1c-4453-ab0b-6a33279acb1f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:34.841030648Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1342,"":"io task","client":"1","":"downstairs","session_id":"06a1b2ef-be1c-4453-ab0b-6a33279acb1f"} {"msg":"ds_connection connected from Ok(127.0.0.1:39978)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:34.841064528Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1342,"":"io task","client":"1","":"downstairs","session_id":"06a1b2ef-be1c-4453-ab0b-6a33279acb1f"} {"msg":"ds_connection connected from Ok(127.0.0.1:35767)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:34.841090693Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1342,"":"io task","client":"0","":"downstairs","session_id":"06a1b2ef-be1c-4453-ab0b-6a33279acb1f"} {"msg":"ds_connection connected from Ok(127.0.0.1:56627)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:34.841116948Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1342,"":"io task","client":"2","":"downstairs","session_id":"06a1b2ef-be1c-4453-ab0b-6a33279acb1f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:34.841820469Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1342,"client":"0","":"downstairs","session_id":"06a1b2ef-be1c-4453-ab0b-6a33279acb1f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:34.841858952Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1342,"client":"2","":"downstairs","session_id":"06a1b2ef-be1c-4453-ab0b-6a33279acb1f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:34.842363296Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1342,"client":"0","":"downstairs","session_id":"06a1b2ef-be1c-4453-ab0b-6a33279acb1f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:34.842502428Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1342,"client":"2","":"downstairs","session_id":"06a1b2ef-be1c-4453-ab0b-6a33279acb1f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:34.842930238Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1342,"session_id":"06a1b2ef-be1c-4453-ab0b-6a33279acb1f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:34.843045766Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1342,"session_id":"06a1b2ef-be1c-4453-ab0b-6a33279acb1f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:34.843367824Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1342,"client":"1","":"downstairs","session_id":"06a1b2ef-be1c-4453-ab0b-6a33279acb1f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:34.843886477Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1342,"client":"1","":"downstairs","session_id":"06a1b2ef-be1c-4453-ab0b-6a33279acb1f"} {"msg":"[0]R flush_numbers[0..12]: [49, 43, 40, 56, 60, 60, 56, 60, 60, 43, 60, 60]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:34.844368278Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1342,"":"downstairs","":"downstairs","session_id":"06a1b2ef-be1c-4453-ab0b-6a33279acb1f"} {"msg":"[0]R generation[0..12]: [32, 28, 26, 36, 39, 39, 36, 39, 39, 28, 39, 39]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:34.844390281Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1342,"":"downstairs","":"downstairs","session_id":"06a1b2ef-be1c-4453-ab0b-6a33279acb1f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:34.84440499Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1342,"":"downstairs","":"downstairs","session_id":"06a1b2ef-be1c-4453-ab0b-6a33279acb1f"} {"msg":"[1]R flush_numbers[0..12]: [49, 43, 40, 56, 60, 60, 56, 60, 60, 43, 60, 60]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:34.844418788Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1342,"":"downstairs","":"downstairs","session_id":"06a1b2ef-be1c-4453-ab0b-6a33279acb1f"} {"msg":"[1]R generation[0..12]: [32, 28, 26, 36, 39, 39, 36, 39, 39, 28, 39, 39]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:34.844432236Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1342,"":"downstairs","":"downstairs","session_id":"06a1b2ef-be1c-4453-ab0b-6a33279acb1f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:34.844445734Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1342,"":"downstairs","":"downstairs","session_id":"06a1b2ef-be1c-4453-ab0b-6a33279acb1f"} {"msg":"[2]R flush_numbers[0..12]: [49, 43, 40, 56, 60, 60, 56, 60, 60, 43, 60, 60]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:34.844458791Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1342,"":"downstairs","":"downstairs","session_id":"06a1b2ef-be1c-4453-ab0b-6a33279acb1f"} {"msg":"[2]R generation[0..12]: [32, 28, 26, 36, 39, 39, 36, 39, 39, 28, 39, 39]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:34.844472109Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1342,"":"downstairs","":"downstairs","session_id":"06a1b2ef-be1c-4453-ab0b-6a33279acb1f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:34.844485437Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1342,"":"downstairs","":"downstairs","session_id":"06a1b2ef-be1c-4453-ab0b-6a33279acb1f"} {"msg":"Max found gen is 40","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:34.844497804Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1342,"":"downstairs","":"downstairs","session_id":"06a1b2ef-be1c-4453-ab0b-6a33279acb1f"} {"msg":"Generation requested: 40 >= found:40","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:34.844510502Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1342,"":"downstairs","":"downstairs","session_id":"06a1b2ef-be1c-4453-ab0b-6a33279acb1f"} {"msg":"Next flush: 61","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-17T21:12:34.844526361Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1342,"":"downstairs","":"downstairs","session_id":"06a1b2ef-be1c-4453-ab0b-6a33279acb1f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:34.844545442Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1342,"":"downstairs","":"downstairs","session_id":"06a1b2ef-be1c-4453-ab0b-6a33279acb1f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:34.844558239Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1342,"session_id":"06a1b2ef-be1c-4453-ab0b-6a33279acb1f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:34.844569816Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1342,"client":"0","":"downstairs","session_id":"06a1b2ef-be1c-4453-ab0b-6a33279acb1f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:34.844582644Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1342,"client":"1","":"downstairs","session_id":"06a1b2ef-be1c-4453-ab0b-6a33279acb1f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:34.844595301Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1342,"client":"2","":"downstairs","session_id":"06a1b2ef-be1c-4453-ab0b-6a33279acb1f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:34.844607959Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1342,"session_id":"06a1b2ef-be1c-4453-ab0b-6a33279acb1f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:34.844619785Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1342,"session_id":"06a1b2ef-be1c-4453-ab0b-6a33279acb1f"} {"msg":"2095efb0-1d24-4ee7-96ec-dbeb6c709104 is now active with session: 06a1b2ef-be1c-4453-ab0b-6a33279acb1f","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:34.844632163Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1342,"session_id":"06a1b2ef-be1c-4453-ab0b-6a33279acb1f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:34.844643659Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1342,"session_id":"06a1b2ef-be1c-4453-ab0b-6a33279acb1f"} {"msg":"The guest has finished waiting for activation with:40","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:34.844655356Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1342} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 59 len 12288 data: 4 4 4 03/30 Write block 9 len 12288 data: 4 6 8 04/30 Write block 346 len 12288 data: 3 3 3 05/30 Write block 336 len 12288 data: 6 5 5 06/30 Read block 365 len 20480 07/30 Read block 7 len 12288 08/30 Read block 430 len 32768 09/30 Write block 384 len 32768 data: 4 3 3 3 4 4 3 3 10/30 Read block 128 len 8192 11/30 Flush 12/30 Flush 13/30 Write block 273 len 20480 data: 2 3 4 4 4 14/30 Flush 15/30 Write block 350 len 16384 data: 5 4 5 5 16/30 Flush 17/30 Write block 343 len 40960 data: 7 5 4 4 4 4 3 6 5 6 18/30 Write block 400 len 8192 data: 6 6 19/30 Write block 447 len 36864 data: 3 4 5 4 4 4 3 3 3 20/30 Write block 515 len 36864 data: 9 9 6 7 7 7 7 5 7 21/30 Write block 51 len 16384 data: 6 8 8 7 22/30 Flush 23/30 Write block 74 len 20480 data: 4 6 5 5 4 24/30 Write block 371 len 36864 data: 3 6 6 7 8 8 10 10 9 25/30 Read block 149 len 16384 26/30 Read block 140 len 12288 27/30 Write block 56 len 20480 data: 7 7 6 5 5 28/30 Read block 318 len 4096 29/30 Write block 295 len 24576 data: 3 3 3 3 4 3 30/30 Read block 214 len 40960 ---------------------------------------------------------------- Crucible gen:40 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 5 Done Done Done false 1023 Acked Write 9 Done Done Done false 1024 Acked Read 4 Done Done Done false 1025 Acked Read 3 Done Done Done false 1026 Acked Write 5 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Write 6 Done Done Done false 1029 Acked Read 10 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 8 7 7 22 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1021 1021 1021 Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:12:34.863598568Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1342} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 40 40 40 62 62 62 F F F 1 020-039 28 28 28 43 43 43 F F F 2 040-059 40 40 40 66 66 66 T T T 3 060-079 40 40 40 62 62 62 T T T 4 080-099 39 39 39 60 60 60 F F F 5 100-119 39 39 39 60 60 60 F F F 6 120-139 36 36 36 56 56 56 F F F 7 140-159 39 39 39 60 60 60 F F F 8 160-179 39 39 39 60 60 60 F F F 9 180-199 28 28 28 43 43 43 F F F 10 200-219 39 39 39 60 60 60 F F F 11 220-239 39 39 39 60 60 60 F F F 12 240-259 39 39 39 60 60 60 F F F 13 260-279 40 40 40 64 64 64 F F F 14 280-299 39 39 39 60 60 60 T T T 15 300-319 31 31 31 48 48 48 T T T 16 320-339 40 40 40 62 62 62 F F F 17 340-359 40 40 40 66 66 66 F F F 18 360-379 39 39 39 60 60 60 T T T 19 380-399 40 40 40 62 62 62 F F F 20 400-419 40 40 40 66 66 66 F F F 21 420-439 39 39 39 60 60 60 F F F 22 440-459 40 40 40 66 66 66 F F F 23 460-479 36 36 36 56 56 56 F F F 24 480-499 39 39 39 60 60 60 F F F 25 500-519 40 40 40 66 66 66 F F F 26 520-539 40 40 40 66 66 66 F F F 27 540-559 34 34 34 52 52 52 F F F 28 560-579 36 36 36 55 55 55 F F F 29 580-599 36 36 36 55 55 55 F F F Max gen: 40, Max flush: 66 A Difference in extent metadata is expected here On loop 20 Verifying data now No change after verify Loop: 20 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 40 40 40 62 62 62 F F F 1 020-039 28 28 28 43 43 43 F F F 2 040-059 41 41 41 67 67 67 F F F 3 060-079 41 41 41 67 67 67 F F F 4 080-099 39 39 39 60 60 60 F F F 5 100-119 39 39 39 60 60 60 F F F 6 120-139 36 36 36 56 56 56 F F F 7 140-159 39 39 39 60 60 60 F F F 8 160-179 39 39 39 60 60 60 F F F 9 180-199 28 28 28 43 43 43 F F F 10 200-219 39 39 39 60 60 60 F F F 11 220-239 39 39 39 60 60 60 F F F 12 240-259 39 39 39 60 60 60 F F F 13 260-279 40 40 40 64 64 64 F F F 14 280-299 41 41 41 67 67 67 F F F 15 300-319 41 41 41 67 67 67 F F F 16 320-339 40 40 40 62 62 62 F F F 17 340-359 40 40 40 66 66 66 F F F 18 360-379 41 41 41 67 67 67 F F F 19 380-399 40 40 40 62 62 62 F F F 20 400-419 40 40 40 66 66 66 F F F 21 420-439 39 39 39 60 60 60 F F F 22 440-459 40 40 40 66 66 66 F F F 23 460-479 36 36 36 56 56 56 F F F 24 480-499 39 39 39 60 60 60 F F F 25 500-519 40 40 40 66 66 66 F F F 26 520-539 40 40 40 66 66 66 F F F 27 540-559 34 34 34 52 52 52 F F F 28 560-579 36 36 36 55 55 55 F F F 29 580-599 36 36 36 55 55 55 F F F Max gen: 41, Max flush: 67 Begin loop 21 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:39.104012208Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1350,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:39.148532029Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1350,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:39.149205482Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1350} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"abd5430f1508af0bc008143ea2566277ef714ec2\",\n git_commit_timestamp: \"2025-01-17T20:55:03.000000000Z\",\n git_branch: \"alan/tokio-to-140\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:39.149245936Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1350} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:39.149260365Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1350} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:39.149274763Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1350} {"msg":"Crucible edde016a-17fc-432d-9c9f-bb281048ac11 has session id: 1154d2ca-9f61-4ca7-8134-a7e15eb16550","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:39.14978351Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1350,"session_id":"1154d2ca-9f61-4ca7-8134-a7e15eb16550"} {"msg":"Upstairs opts: Upstairs UUID: edde016a-17fc-432d-9c9f-bb281048ac11, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:39.149953099Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1350,"session_id":"1154d2ca-9f61-4ca7-8134-a7e15eb16550"} {"msg":"Crucible stats registered with UUID: edde016a-17fc-432d-9c9f-bb281048ac11","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:39.149989211Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1350,"session_id":"1154d2ca-9f61-4ca7-8134-a7e15eb16550"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:12:39.150012944Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1350,"session_id":"1154d2ca-9f61-4ca7-8134-a7e15eb16550"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:39.150477806Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1350,"task":"crutest"} {"msg":"The guest has requested activation with gen:42","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:39.150509475Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1350} {"msg":"edde016a-17fc-432d-9c9f-bb281048ac11 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:39.150544905Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1350,"session_id":"1154d2ca-9f61-4ca7-8134-a7e15eb16550"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:39.150560945Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1350,"client":"0","":"downstairs","session_id":"1154d2ca-9f61-4ca7-8134-a7e15eb16550"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:39.15058825Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1350,"client":"0","":"downstairs","session_id":"1154d2ca-9f61-4ca7-8134-a7e15eb16550"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:39.150617158Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1350,"client":"1","":"downstairs","session_id":"1154d2ca-9f61-4ca7-8134-a7e15eb16550"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:39.150651067Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1350,"client":"1","":"downstairs","session_id":"1154d2ca-9f61-4ca7-8134-a7e15eb16550"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:39.150685547Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1350,"client":"2","":"downstairs","session_id":"1154d2ca-9f61-4ca7-8134-a7e15eb16550"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:39.150720108Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1350,"client":"2","":"downstairs","session_id":"1154d2ca-9f61-4ca7-8134-a7e15eb16550"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:39.150755428Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1350,"":"io task","client":"2","":"downstairs","session_id":"1154d2ca-9f61-4ca7-8134-a7e15eb16550"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:39.150786336Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1350,"":"io task","client":"0","":"downstairs","session_id":"1154d2ca-9f61-4ca7-8134-a7e15eb16550"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:39.150814383Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1350,"":"io task","client":"1","":"downstairs","session_id":"1154d2ca-9f61-4ca7-8134-a7e15eb16550"} {"msg":"ds_connection connected from Ok(127.0.0.1:33853)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:39.150838497Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1350,"":"io task","client":"0","":"downstairs","session_id":"1154d2ca-9f61-4ca7-8134-a7e15eb16550"} {"msg":"ds_connection connected from Ok(127.0.0.1:59989)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:39.150864072Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1350,"":"io task","client":"2","":"downstairs","session_id":"1154d2ca-9f61-4ca7-8134-a7e15eb16550"} {"msg":"ds_connection connected from Ok(127.0.0.1:52473)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:39.150889647Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1350,"":"io task","client":"1","":"downstairs","session_id":"1154d2ca-9f61-4ca7-8134-a7e15eb16550"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:39.151599221Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1350,"client":"0","":"downstairs","session_id":"1154d2ca-9f61-4ca7-8134-a7e15eb16550"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:39.15163001Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1350,"client":"1","":"downstairs","session_id":"1154d2ca-9f61-4ca7-8134-a7e15eb16550"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:39.151663279Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1350,"client":"2","":"downstairs","session_id":"1154d2ca-9f61-4ca7-8134-a7e15eb16550"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:39.152223516Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1350,"client":"1","":"downstairs","session_id":"1154d2ca-9f61-4ca7-8134-a7e15eb16550"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:39.152359065Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1350,"client":"0","":"downstairs","session_id":"1154d2ca-9f61-4ca7-8134-a7e15eb16550"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:39.152398199Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1350,"client":"2","":"downstairs","session_id":"1154d2ca-9f61-4ca7-8134-a7e15eb16550"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:39.152735596Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1350,"session_id":"1154d2ca-9f61-4ca7-8134-a7e15eb16550"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:39.15293202Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1350,"session_id":"1154d2ca-9f61-4ca7-8134-a7e15eb16550"} {"msg":"[0]R flush_numbers[0..12]: [62, 43, 67, 67, 60, 60, 56, 60, 60, 43, 60, 60]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:39.153064918Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1350,"":"downstairs","":"downstairs","session_id":"1154d2ca-9f61-4ca7-8134-a7e15eb16550"} {"msg":"[0]R generation[0..12]: [40, 28, 41, 41, 39, 39, 36, 39, 39, 28, 39, 39]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:39.153084469Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1350,"":"downstairs","":"downstairs","session_id":"1154d2ca-9f61-4ca7-8134-a7e15eb16550"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:39.153099288Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1350,"":"downstairs","":"downstairs","session_id":"1154d2ca-9f61-4ca7-8134-a7e15eb16550"} {"msg":"[1]R flush_numbers[0..12]: [62, 43, 67, 67, 60, 60, 56, 60, 60, 43, 60, 60]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:39.153113086Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1350,"":"downstairs","":"downstairs","session_id":"1154d2ca-9f61-4ca7-8134-a7e15eb16550"} {"msg":"[1]R generation[0..12]: [40, 28, 41, 41, 39, 39, 36, 39, 39, 28, 39, 39]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:39.153126684Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1350,"":"downstairs","":"downstairs","session_id":"1154d2ca-9f61-4ca7-8134-a7e15eb16550"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:39.153140562Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1350,"":"downstairs","":"downstairs","session_id":"1154d2ca-9f61-4ca7-8134-a7e15eb16550"} {"msg":"[2]R flush_numbers[0..12]: [62, 43, 67, 67, 60, 60, 56, 60, 60, 43, 60, 60]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:39.15315374Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1350,"":"downstairs","":"downstairs","session_id":"1154d2ca-9f61-4ca7-8134-a7e15eb16550"} {"msg":"[2]R generation[0..12]: [40, 28, 41, 41, 39, 39, 36, 39, 39, 28, 39, 39]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:39.153170019Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1350,"":"downstairs","":"downstairs","session_id":"1154d2ca-9f61-4ca7-8134-a7e15eb16550"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:39.153183998Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1350,"":"downstairs","":"downstairs","session_id":"1154d2ca-9f61-4ca7-8134-a7e15eb16550"} {"msg":"Max found gen is 42","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:39.153196935Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1350,"":"downstairs","":"downstairs","session_id":"1154d2ca-9f61-4ca7-8134-a7e15eb16550"} {"msg":"Generation requested: 42 >= found:42","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:39.153209893Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1350,"":"downstairs","":"downstairs","session_id":"1154d2ca-9f61-4ca7-8134-a7e15eb16550"} {"msg":"Next flush: 68","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-17T21:12:39.15322258Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1350,"":"downstairs","":"downstairs","session_id":"1154d2ca-9f61-4ca7-8134-a7e15eb16550"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:39.153238069Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1350,"":"downstairs","":"downstairs","session_id":"1154d2ca-9f61-4ca7-8134-a7e15eb16550"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:39.153251147Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1350,"session_id":"1154d2ca-9f61-4ca7-8134-a7e15eb16550"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:39.153262834Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1350,"client":"0","":"downstairs","session_id":"1154d2ca-9f61-4ca7-8134-a7e15eb16550"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:39.153276312Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1350,"client":"1","":"downstairs","session_id":"1154d2ca-9f61-4ca7-8134-a7e15eb16550"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:39.153289889Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1350,"client":"2","":"downstairs","session_id":"1154d2ca-9f61-4ca7-8134-a7e15eb16550"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:39.153303477Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1350,"session_id":"1154d2ca-9f61-4ca7-8134-a7e15eb16550"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:39.153315665Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1350,"session_id":"1154d2ca-9f61-4ca7-8134-a7e15eb16550"} {"msg":"edde016a-17fc-432d-9c9f-bb281048ac11 is now active with session: 1154d2ca-9f61-4ca7-8134-a7e15eb16550","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:39.153328142Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1350,"session_id":"1154d2ca-9f61-4ca7-8134-a7e15eb16550"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:39.153339809Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1350,"session_id":"1154d2ca-9f61-4ca7-8134-a7e15eb16550"} {"msg":"The guest has finished waiting for activation with:42","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:39.153351816Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1350} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Flush 03/30 Write block 489 len 24576 data: 3 3 3 2 3 3 04/30 Read block 534 len 36864 05/30 Read block 491 len 32768 06/30 Read block 195 len 12288 07/30 Read block 192 len 28672 08/30 Write block 164 len 32768 data: 2 3 4 4 5 5 5 4 09/30 Flush 10/30 Write block 12 len 24576 data: 8 7 6 5 6 4 11/30 Write block 69 len 16384 data: 3 3 4 4 12/30 Write block 186 len 20480 data: 2 2 3 3 5 13/30 Write block 289 len 36864 data: 2 3 3 3 2 2 4 4 4 14/30 Read block 308 len 12288 15/30 Read block 209 len 16384 16/30 Read block 340 len 8192 17/30 Read block 9 len 4096 18/30 Write block 171 len 20480 data: 5 5 5 5 6 19/30 Write block 76 len 32768 data: 6 6 5 4 3 3 3 3 20/30 Write block 498 len 16384 data: 3 5 6 7 21/30 Read block 123 len 36864 22/30 Read block 323 len 4096 23/30 Read block 100 len 20480 24/30 Flush 25/30 Write block 312 len 36864 data: 3 3 3 4 4 4 4 5 5 26/30 Flush 27/30 Write block 581 len 40960 data: 3 3 3 3 3 2 2 2 2 2 28/30 Read block 360 len 32768 29/30 Write block 586 len 20480 data: 3 3 3 3 3 30/30 Write block 579 len 8192 data: 4 4 ---------------------------------------------------------------- Crucible gen:42 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 10 Done Done Done false 1027 Acked Read 8 Done Done Done false 1028 Acked Write 5 Sent Sent Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 2 2 2 6 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:12:39.173305608Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1350} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 42 42 42 71 71 71 F F F 1 020-039 28 28 28 43 43 43 F F F 2 040-059 41 41 41 67 67 67 F F F 3 060-079 42 42 42 71 71 71 F F F 4 080-099 42 42 42 71 71 71 F F F 5 100-119 39 39 39 60 60 60 F F F 6 120-139 36 36 36 56 56 56 F F F 7 140-159 39 39 39 60 60 60 F F F 8 160-179 42 42 42 71 71 71 F F F 9 180-199 42 42 42 71 71 71 F F F 10 200-219 39 39 39 60 60 60 F F F 11 220-239 39 39 39 60 60 60 F F F 12 240-259 39 39 39 60 60 60 F F F 13 260-279 40 40 40 64 64 64 F F F 14 280-299 42 42 42 71 71 71 F F F 15 300-319 42 42 42 72 72 72 F F F 16 320-339 42 42 42 72 72 72 F F F 17 340-359 40 40 40 66 66 66 F F F 18 360-379 41 41 41 67 67 67 F F F 19 380-399 40 40 40 62 62 62 F F F 20 400-419 40 40 40 66 66 66 F F F 21 420-439 39 39 39 60 60 60 F F F 22 440-459 40 40 40 66 66 66 F F F 23 460-479 36 36 36 56 56 56 F F F 24 480-499 42 42 42 71 71 71 F F F 25 500-519 42 42 42 71 71 71 F F F 26 520-539 40 40 40 66 66 66 F F F 27 540-559 34 34 34 52 52 52 F F F 28 560-579 36 36 36 55 55 55 T T T 29 580-599 36 36 36 55 55 55 T T T Max gen: 42, Max flush: 72 A Difference in extent metadata is expected here On loop 21 Verifying data now No change after verify Loop: 21 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 42 42 42 71 71 71 F F F 1 020-039 28 28 28 43 43 43 F F F 2 040-059 41 41 41 67 67 67 F F F 3 060-079 42 42 42 71 71 71 F F F 4 080-099 42 42 42 71 71 71 F F F 5 100-119 39 39 39 60 60 60 F F F 6 120-139 36 36 36 56 56 56 F F F 7 140-159 39 39 39 60 60 60 F F F 8 160-179 42 42 42 71 71 71 F F F 9 180-199 42 42 42 71 71 71 F F F 10 200-219 39 39 39 60 60 60 F F F 11 220-239 39 39 39 60 60 60 F F F 12 240-259 39 39 39 60 60 60 F F F 13 260-279 40 40 40 64 64 64 F F F 14 280-299 42 42 42 71 71 71 F F F 15 300-319 42 42 42 72 72 72 F F F 16 320-339 42 42 42 72 72 72 F F F 17 340-359 40 40 40 66 66 66 F F F 18 360-379 41 41 41 67 67 67 F F F 19 380-399 40 40 40 62 62 62 F F F 20 400-419 40 40 40 66 66 66 F F F 21 420-439 39 39 39 60 60 60 F F F 22 440-459 40 40 40 66 66 66 F F F 23 460-479 36 36 36 56 56 56 F F F 24 480-499 42 42 42 71 71 71 F F F 25 500-519 42 42 42 71 71 71 F F F 26 520-539 40 40 40 66 66 66 F F F 27 540-559 34 34 34 52 52 52 F F F 28 560-579 43 43 43 73 73 73 F F F 29 580-599 43 43 43 73 73 73 F F F Max gen: 43, Max flush: 73 Begin loop 22 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:43.394372091Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1358,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:43.438865616Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1358,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:43.43950564Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1358} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"abd5430f1508af0bc008143ea2566277ef714ec2\",\n git_commit_timestamp: \"2025-01-17T20:55:03.000000000Z\",\n git_branch: \"alan/tokio-to-140\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:43.439555909Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1358} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:43.439576761Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1358} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:43.439599114Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1358} {"msg":"Crucible 848c37e2-21ce-41bf-9248-17cbf430e13c has session id: 6c54a0b7-2eb1-424b-bf95-8d9e1be48c66","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:43.440100798Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1358,"session_id":"6c54a0b7-2eb1-424b-bf95-8d9e1be48c66"} {"msg":"Upstairs opts: Upstairs UUID: 848c37e2-21ce-41bf-9248-17cbf430e13c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:43.440159602Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1358,"session_id":"6c54a0b7-2eb1-424b-bf95-8d9e1be48c66"} {"msg":"Crucible stats registered with UUID: 848c37e2-21ce-41bf-9248-17cbf430e13c","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:43.440191511Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1358,"session_id":"6c54a0b7-2eb1-424b-bf95-8d9e1be48c66"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:12:43.440213233Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1358,"session_id":"6c54a0b7-2eb1-424b-bf95-8d9e1be48c66"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:43.440602721Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1358,"task":"crutest"} {"msg":"The guest has requested activation with gen:44","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:43.440626855Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1358} {"msg":"848c37e2-21ce-41bf-9248-17cbf430e13c active request set","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:43.440692053Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1358,"session_id":"6c54a0b7-2eb1-424b-bf95-8d9e1be48c66"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:43.440709113Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1358,"client":"0","":"downstairs","session_id":"6c54a0b7-2eb1-424b-bf95-8d9e1be48c66"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:43.440734898Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1358,"client":"0","":"downstairs","session_id":"6c54a0b7-2eb1-424b-bf95-8d9e1be48c66"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:43.440758182Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1358,"client":"1","":"downstairs","session_id":"6c54a0b7-2eb1-424b-bf95-8d9e1be48c66"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:43.440784938Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1358,"client":"1","":"downstairs","session_id":"6c54a0b7-2eb1-424b-bf95-8d9e1be48c66"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:43.440811653Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1358,"client":"2","":"downstairs","session_id":"6c54a0b7-2eb1-424b-bf95-8d9e1be48c66"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:43.4408411Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1358,"client":"2","":"downstairs","session_id":"6c54a0b7-2eb1-424b-bf95-8d9e1be48c66"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:43.440869056Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1358,"":"io task","client":"2","":"downstairs","session_id":"6c54a0b7-2eb1-424b-bf95-8d9e1be48c66"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:43.440895862Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1358,"":"io task","client":"0","":"downstairs","session_id":"6c54a0b7-2eb1-424b-bf95-8d9e1be48c66"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:43.440919726Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1358,"":"io task","client":"1","":"downstairs","session_id":"6c54a0b7-2eb1-424b-bf95-8d9e1be48c66"} {"msg":"ds_connection connected from Ok(127.0.0.1:57898)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:43.440951545Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1358,"":"io task","client":"1","":"downstairs","session_id":"6c54a0b7-2eb1-424b-bf95-8d9e1be48c66"} {"msg":"ds_connection connected from Ok(127.0.0.1:62682)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:43.440985284Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1358,"":"io task","client":"2","":"downstairs","session_id":"6c54a0b7-2eb1-424b-bf95-8d9e1be48c66"} {"msg":"ds_connection connected from Ok(127.0.0.1:61884)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:43.441013831Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1358,"":"io task","client":"0","":"downstairs","session_id":"6c54a0b7-2eb1-424b-bf95-8d9e1be48c66"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:43.441795098Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1358,"client":"1","":"downstairs","session_id":"6c54a0b7-2eb1-424b-bf95-8d9e1be48c66"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:43.441834851Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1358,"client":"0","":"downstairs","session_id":"6c54a0b7-2eb1-424b-bf95-8d9e1be48c66"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:43.442505253Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1358,"client":"1","":"downstairs","session_id":"6c54a0b7-2eb1-424b-bf95-8d9e1be48c66"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:43.442531618Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1358,"client":"2","":"downstairs","session_id":"6c54a0b7-2eb1-424b-bf95-8d9e1be48c66"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:43.442546797Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1358,"client":"0","":"downstairs","session_id":"6c54a0b7-2eb1-424b-bf95-8d9e1be48c66"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:43.443078928Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1358,"session_id":"6c54a0b7-2eb1-424b-bf95-8d9e1be48c66"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:43.443101231Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1358,"session_id":"6c54a0b7-2eb1-424b-bf95-8d9e1be48c66"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:43.443114519Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1358,"client":"2","":"downstairs","session_id":"6c54a0b7-2eb1-424b-bf95-8d9e1be48c66"} {"msg":"[0]R flush_numbers[0..12]: [71, 43, 67, 71, 71, 60, 56, 60, 71, 71, 60, 60]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:43.443601173Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1358,"":"downstairs","":"downstairs","session_id":"6c54a0b7-2eb1-424b-bf95-8d9e1be48c66"} {"msg":"[0]R generation[0..12]: [42, 28, 41, 42, 42, 39, 36, 39, 42, 42, 39, 39]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:43.443623847Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1358,"":"downstairs","":"downstairs","session_id":"6c54a0b7-2eb1-424b-bf95-8d9e1be48c66"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:43.443638926Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1358,"":"downstairs","":"downstairs","session_id":"6c54a0b7-2eb1-424b-bf95-8d9e1be48c66"} {"msg":"[1]R flush_numbers[0..12]: [71, 43, 67, 71, 71, 60, 56, 60, 71, 71, 60, 60]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:43.443652674Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1358,"":"downstairs","":"downstairs","session_id":"6c54a0b7-2eb1-424b-bf95-8d9e1be48c66"} {"msg":"[1]R generation[0..12]: [42, 28, 41, 42, 42, 39, 36, 39, 42, 42, 39, 39]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:43.443670014Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1358,"":"downstairs","":"downstairs","session_id":"6c54a0b7-2eb1-424b-bf95-8d9e1be48c66"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:43.443684993Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1358,"":"downstairs","":"downstairs","session_id":"6c54a0b7-2eb1-424b-bf95-8d9e1be48c66"} {"msg":"[2]R flush_numbers[0..12]: [71, 43, 67, 71, 71, 60, 56, 60, 71, 71, 60, 60]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:43.44369809Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1358,"":"downstairs","":"downstairs","session_id":"6c54a0b7-2eb1-424b-bf95-8d9e1be48c66"} {"msg":"[2]R generation[0..12]: [42, 28, 41, 42, 42, 39, 36, 39, 42, 42, 39, 39]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:43.443711208Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1358,"":"downstairs","":"downstairs","session_id":"6c54a0b7-2eb1-424b-bf95-8d9e1be48c66"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:43.443724696Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1358,"":"downstairs","":"downstairs","session_id":"6c54a0b7-2eb1-424b-bf95-8d9e1be48c66"} {"msg":"Max found gen is 44","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:43.443737163Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1358,"":"downstairs","":"downstairs","session_id":"6c54a0b7-2eb1-424b-bf95-8d9e1be48c66"} {"msg":"Generation requested: 44 >= found:44","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:43.44374962Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1358,"":"downstairs","":"downstairs","session_id":"6c54a0b7-2eb1-424b-bf95-8d9e1be48c66"} {"msg":"Next flush: 74","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:43.443761998Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1358,"":"downstairs","":"downstairs","session_id":"6c54a0b7-2eb1-424b-bf95-8d9e1be48c66"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:43.443774475Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1358,"":"downstairs","":"downstairs","session_id":"6c54a0b7-2eb1-424b-bf95-8d9e1be48c66"} Wait for a query_work_queue command to finish before sending IO {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:43.443793516Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1358,"session_id":"6c54a0b7-2eb1-424b-bf95-8d9e1be48c66"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:43.443805243Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1358,"client":"0","":"downstairs","session_id":"6c54a0b7-2eb1-424b-bf95-8d9e1be48c66"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:43.44381819Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1358,"client":"1","":"downstairs","session_id":"6c54a0b7-2eb1-424b-bf95-8d9e1be48c66"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:43.443830898Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1358,"client":"2","":"downstairs","session_id":"6c54a0b7-2eb1-424b-bf95-8d9e1be48c66"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:43.443843625Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1358,"session_id":"6c54a0b7-2eb1-424b-bf95-8d9e1be48c66"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:43.443855382Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1358,"session_id":"6c54a0b7-2eb1-424b-bf95-8d9e1be48c66"} {"msg":"848c37e2-21ce-41bf-9248-17cbf430e13c is now active with session: 6c54a0b7-2eb1-424b-bf95-8d9e1be48c66","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:43.443870531Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1358,"session_id":"6c54a0b7-2eb1-424b-bf95-8d9e1be48c66"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:43.443882298Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1358,"session_id":"6c54a0b7-2eb1-424b-bf95-8d9e1be48c66"} {"msg":"The guest has finished waiting for activation with:44","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:43.443894275Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1358} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 27 len 8192 data: 4 4 02/30 Flush 03/30 Write block 240 len 20480 data: 3 4 5 5 6 04/30 Read block 136 len 36864 05/30 Read block 32 len 20480 06/30 Flush 07/30 Flush 08/30 Write block 255 len 40960 data: 2 4 4 5 5 5 5 5 5 4 09/30 Write block 323 len 36864 data: 4 3 4 4 3 2 2 3 3 10/30 Read block 428 len 16384 11/30 Write block 32 len 16384 data: 7 6 5 6 12/30 Write block 216 len 32768 data: 4 4 3 4 3 3 4 5 13/30 Read block 18 len 4096 14/30 Read block 214 len 20480 15/30 Read block 26 len 20480 16/30 Write block 222 len 12288 data: 5 6 5 17/30 Write block 292 len 40960 data: 4 3 3 5 5 5 4 5 4 4 18/30 Read block 215 len 40960 19/30 Write block 110 len 24576 data: 4 5 6 6 7 7 20/30 Write block 427 len 24576 data: 5 6 6 5 6 5 21/30 Read block 407 len 24576 22/30 Read block 550 len 4096 23/30 Write block 394 len 8192 data: 4 3 24/30 Read block 337 len 28672 25/30 Write block 154 len 8192 data: 3 3 26/30 Write block 454 len 32768 data: 4 4 4 5 5 4 4 5 27/30 Write block 370 len 4096 data: 3 28/30 Write block 75 len 36864 data: 7 7 7 6 5 4 4 4 4 29/30 Write block 409 len 36864 data: 3 3 3 3 3 3 4 3 3 30/30 Write block 269 len 20480 data: 4 4 3 3 3 ---------------------------------------------------------------- Crucible gen:44 GIO:true work queues: Upstairs:0 downstairs:23 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1007 Acked Write 10 Done Done Done false 1008 Acked Write 9 Done Done Done false 1009 Acked Read 4 Done Done Done false 1010 Acked Write 4 Done Done Done false 1011 Acked Write 8 Done Done Done false 1012 Acked Read 1 Done Done Done false 1013 Acked Read 5 Done Done Done false 1014 Acked Read 5 Done Done Done false 1015 Acked Write 3 Done Done Done false 1016 Acked Write 10 Done Done Done false 1017 Acked Read 10 Done Done Done false 1018 Acked Write 6 Done Done Done false 1019 Acked Write 6 Done Done Done false 1020 Acked Read 6 Done Done Done false 1021 Acked Read 1 Done Done Done false 1022 Acked Write 2 Done Done Done false 1023 Acked Read 7 Done Done Done false 1024 Acked Write 2 Done Done Done false 1025 Acked Write 8 Done Done Done false 1026 Acked Write 1 Done Done Done false 1027 Acked Write 9 Done Done Done false 1028 Acked Write 9 Sent Sent Sent false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 21 21 21 63 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1006 1006 1006 Downstairs last five completed: 1006 1005 1004 1003 1002 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:12:43.462389575Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1358} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 42 42 42 71 71 71 F F F 1 020-039 44 44 44 74 74 74 T T T 2 040-059 41 41 41 67 67 67 F F F 3 060-079 42 42 42 71 71 71 T T T 4 080-099 42 42 42 71 71 71 T T T 5 100-119 39 39 39 60 60 60 T T T 6 120-139 36 36 36 56 56 56 F F F 7 140-159 39 39 39 60 60 60 T T T 8 160-179 42 42 42 71 71 71 F F F 9 180-199 42 42 42 71 71 71 F F F 10 200-219 39 39 39 60 60 60 T T T 11 220-239 39 39 39 60 60 60 T T T 12 240-259 44 44 44 75 75 75 T T T 13 260-279 40 40 40 64 64 64 T T T 14 280-299 42 42 42 71 71 71 T T T 15 300-319 42 42 42 72 72 72 T T T 16 320-339 42 42 42 72 72 72 T T T 17 340-359 40 40 40 66 66 66 F F F 18 360-379 41 41 41 67 67 67 T T T 19 380-399 40 40 40 62 62 62 T T T 20 400-419 40 40 40 66 66 66 T T T 21 420-439 39 39 39 60 60 60 T T T 22 440-459 40 40 40 66 66 66 T T T 23 460-479 36 36 36 56 56 56 T T T 24 480-499 42 42 42 71 71 71 F F F 25 500-519 42 42 42 71 71 71 F F F 26 520-539 40 40 40 66 66 66 F F F 27 540-559 34 34 34 52 52 52 F F F 28 560-579 43 43 43 73 73 73 F F F 29 580-599 43 43 43 73 73 73 F F F Max gen: 44, Max flush: 75 A Difference in extent metadata is expected here On loop 22 Verifying data now No change after verify Loop: 22 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 42 42 42 71 71 71 F F F 1 020-039 45 45 45 76 76 76 F F F 2 040-059 41 41 41 67 67 67 F F F 3 060-079 45 45 45 76 76 76 F F F 4 080-099 45 45 45 76 76 76 F F F 5 100-119 45 45 45 76 76 76 F F F 6 120-139 36 36 36 56 56 56 F F F 7 140-159 45 45 45 76 76 76 F F F 8 160-179 42 42 42 71 71 71 F F F 9 180-199 42 42 42 71 71 71 F F F 10 200-219 45 45 45 76 76 76 F F F 11 220-239 45 45 45 76 76 76 F F F 12 240-259 45 45 45 76 76 76 F F F 13 260-279 45 45 45 76 76 76 F F F 14 280-299 45 45 45 76 76 76 F F F 15 300-319 45 45 45 76 76 76 F F F 16 320-339 45 45 45 76 76 76 F F F 17 340-359 40 40 40 66 66 66 F F F 18 360-379 45 45 45 76 76 76 F F F 19 380-399 45 45 45 76 76 76 F F F 20 400-419 45 45 45 76 76 76 F F F 21 420-439 45 45 45 76 76 76 F F F 22 440-459 45 45 45 76 76 76 F F F 23 460-479 45 45 45 76 76 76 F F F 24 480-499 42 42 42 71 71 71 F F F 25 500-519 42 42 42 71 71 71 F F F 26 520-539 40 40 40 66 66 66 F F F 27 540-559 34 34 34 52 52 52 F F F 28 560-579 43 43 43 73 73 73 F F F 29 580-599 43 43 43 73 73 73 F F F Max gen: 45, Max flush: 76 Begin loop 23 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:57.76739714Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1366,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:12:57.812009844Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1366,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:57.812454974Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1366,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:57.844745998Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1366,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:57.845372804Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1366} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"abd5430f1508af0bc008143ea2566277ef714ec2\",\n git_commit_timestamp: \"2025-01-17T20:55:03.000000000Z\",\n git_branch: \"alan/tokio-to-140\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:57.845406434Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1366} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:57.845419962Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1366} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:57.8454349Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1366} {"msg":"Crucible 1b91ac4a-e9e2-4c4c-b68a-670e9e9b47ea has session id: 6ff07c8e-024b-42b2-ab77-8091408fe3f6","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:57.846062797Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1366,"session_id":"6ff07c8e-024b-42b2-ab77-8091408fe3f6"} {"msg":"Upstairs opts: Upstairs UUID: 1b91ac4a-e9e2-4c4c-b68a-670e9e9b47ea, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:57.84610214Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1366,"session_id":"6ff07c8e-024b-42b2-ab77-8091408fe3f6"} {"msg":"Crucible stats registered with UUID: 1b91ac4a-e9e2-4c4c-b68a-670e9e9b47ea","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:57.846125274Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1366,"session_id":"6ff07c8e-024b-42b2-ab77-8091408fe3f6"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:12:57.846139722Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1366,"session_id":"6ff07c8e-024b-42b2-ab77-8091408fe3f6"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:57.846660907Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1366,"task":"crutest"} {"msg":"The guest has requested activation with gen:46","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:57.846693826Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1366} {"msg":"1b91ac4a-e9e2-4c4c-b68a-670e9e9b47ea active request set","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:57.846746827Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1366,"session_id":"6ff07c8e-024b-42b2-ab77-8091408fe3f6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:57.846776244Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1366,"client":"0","":"downstairs","session_id":"6ff07c8e-024b-42b2-ab77-8091408fe3f6"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:57.846813636Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1366,"client":"0","":"downstairs","session_id":"6ff07c8e-024b-42b2-ab77-8091408fe3f6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:57.846841032Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1366,"client":"1","":"downstairs","session_id":"6ff07c8e-024b-42b2-ab77-8091408fe3f6"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:57.846869609Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1366,"client":"1","":"downstairs","session_id":"6ff07c8e-024b-42b2-ab77-8091408fe3f6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:57.846896335Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1366,"client":"2","":"downstairs","session_id":"6ff07c8e-024b-42b2-ab77-8091408fe3f6"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:57.84692304Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1366,"client":"2","":"downstairs","session_id":"6ff07c8e-024b-42b2-ab77-8091408fe3f6"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:57.846950206Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1366,"":"io task","client":"2","":"downstairs","session_id":"6ff07c8e-024b-42b2-ab77-8091408fe3f6"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:57.846977212Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1366,"":"io task","client":"0","":"downstairs","session_id":"6ff07c8e-024b-42b2-ab77-8091408fe3f6"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:57.847000825Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1366,"":"io task","client":"1","":"downstairs","session_id":"6ff07c8e-024b-42b2-ab77-8091408fe3f6"} {"msg":"ds_connection connected from Ok(127.0.0.1:47221)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:57.84702501Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1366,"":"io task","client":"0","":"downstairs","session_id":"6ff07c8e-024b-42b2-ab77-8091408fe3f6"} {"msg":"ds_connection connected from Ok(127.0.0.1:64325)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:57.847047713Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1366,"":"io task","client":"2","":"downstairs","session_id":"6ff07c8e-024b-42b2-ab77-8091408fe3f6"} {"msg":"ds_connection connected from Ok(127.0.0.1:57672)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:57.847149032Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1366,"":"io task","client":"1","":"downstairs","session_id":"6ff07c8e-024b-42b2-ab77-8091408fe3f6"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:57.847689198Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1366,"client":"2","":"downstairs","session_id":"6ff07c8e-024b-42b2-ab77-8091408fe3f6"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:57.84772704Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1366,"client":"1","":"downstairs","session_id":"6ff07c8e-024b-42b2-ab77-8091408fe3f6"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:57.847837975Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1366,"client":"0","":"downstairs","session_id":"6ff07c8e-024b-42b2-ab77-8091408fe3f6"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:57.848386085Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1366,"client":"2","":"downstairs","session_id":"6ff07c8e-024b-42b2-ab77-8091408fe3f6"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:57.848518252Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1366,"client":"1","":"downstairs","session_id":"6ff07c8e-024b-42b2-ab77-8091408fe3f6"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:57.848549771Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1366,"client":"0","":"downstairs","session_id":"6ff07c8e-024b-42b2-ab77-8091408fe3f6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:57.848876742Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1366,"session_id":"6ff07c8e-024b-42b2-ab77-8091408fe3f6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:57.849019285Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1366,"session_id":"6ff07c8e-024b-42b2-ab77-8091408fe3f6"} {"msg":"[0]R flush_numbers[0..12]: [71, 76, 67, 76, 76, 76, 56, 76, 71, 71, 76, 76]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:57.849138285Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1366,"":"downstairs","":"downstairs","session_id":"6ff07c8e-024b-42b2-ab77-8091408fe3f6"} {"msg":"[0]R generation[0..12]: [42, 45, 41, 45, 45, 45, 36, 45, 42, 42, 45, 45]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:57.849157816Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1366,"":"downstairs","":"downstairs","session_id":"6ff07c8e-024b-42b2-ab77-8091408fe3f6"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:57.849172725Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1366,"":"downstairs","":"downstairs","session_id":"6ff07c8e-024b-42b2-ab77-8091408fe3f6"} {"msg":"[1]R flush_numbers[0..12]: [71, 76, 67, 76, 76, 76, 56, 76, 71, 71, 76, 76]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:57.849186513Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1366,"":"downstairs","":"downstairs","session_id":"6ff07c8e-024b-42b2-ab77-8091408fe3f6"} {"msg":"[1]R generation[0..12]: [42, 45, 41, 45, 45, 45, 36, 45, 42, 42, 45, 45]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:57.849200211Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1366,"":"downstairs","":"downstairs","session_id":"6ff07c8e-024b-42b2-ab77-8091408fe3f6"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:57.849213989Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1366,"":"downstairs","":"downstairs","session_id":"6ff07c8e-024b-42b2-ab77-8091408fe3f6"} {"msg":"[2]R flush_numbers[0..12]: [71, 76, 67, 76, 76, 76, 56, 76, 71, 71, 76, 76]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:57.849227177Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1366,"":"downstairs","":"downstairs","session_id":"6ff07c8e-024b-42b2-ab77-8091408fe3f6"} {"msg":"[2]R generation[0..12]: [42, 45, 41, 45, 45, 45, 36, 45, 42, 42, 45, 45]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:57.849240124Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1366,"":"downstairs","":"downstairs","session_id":"6ff07c8e-024b-42b2-ab77-8091408fe3f6"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:57.849253402Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1366,"":"downstairs","":"downstairs","session_id":"6ff07c8e-024b-42b2-ab77-8091408fe3f6"} {"msg":"Max found gen is 46","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:57.849269712Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1366,"":"downstairs","":"downstairs","session_id":"6ff07c8e-024b-42b2-ab77-8091408fe3f6"} {"msg":"Generation requested: 46 >= found:46","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:57.84928339Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1366,"":"downstairs","":"downstairs","session_id":"6ff07c8e-024b-42b2-ab77-8091408fe3f6"} {"msg":"Next flush: 77","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:57.849296117Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1366,"":"downstairs","":"downstairs","session_id":"6ff07c8e-024b-42b2-ab77-8091408fe3f6"} Wait for a query_work_queue command to finish before sending IO {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:57.849315959Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1366,"":"downstairs","":"downstairs","session_id":"6ff07c8e-024b-42b2-ab77-8091408fe3f6"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:57.849340173Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1366,"session_id":"6ff07c8e-024b-42b2-ab77-8091408fe3f6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:57.849363206Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1366,"client":"0","":"downstairs","session_id":"6ff07c8e-024b-42b2-ab77-8091408fe3f6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:57.84938732Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1366,"client":"1","":"downstairs","session_id":"6ff07c8e-024b-42b2-ab77-8091408fe3f6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:57.849408743Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1366,"client":"2","":"downstairs","session_id":"6ff07c8e-024b-42b2-ab77-8091408fe3f6"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:57.849422631Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1366,"session_id":"6ff07c8e-024b-42b2-ab77-8091408fe3f6"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:57.849434488Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1366,"session_id":"6ff07c8e-024b-42b2-ab77-8091408fe3f6"} {"msg":"1b91ac4a-e9e2-4c4c-b68a-670e9e9b47ea is now active with session: 6ff07c8e-024b-42b2-ab77-8091408fe3f6","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:57.849446895Z","hostname":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1366,"session_id":"6ff07c8e-024b-42b2-ab77-8091408fe3f6"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:57.849463255Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1366,"session_id":"6ff07c8e-024b-42b2-ab77-8091408fe3f6"} {"msg":"The guest has finished waiting for activation with:46","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:12:57.849476222Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1366} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 588 len 32768 data: 4 4 4 2 2 2 2 2 02/30 Read block 84 len 20480 03/30 Read block 294 len 36864 04/30 Read block 329 len 20480 05/30 Flush 06/30 Write block 398 len 4096 data: 5 07/30 Write block 68 len 24576 data: 4 4 4 5 5 4 08/30 Write block 159 len 40960 data: 4 4 3 3 2 3 4 5 5 6 09/30 Flush 10/30 Write block 389 len 40960 data: 5 4 4 4 4 5 4 5 5 6 11/30 Write block 334 len 24576 data: 5 6 7 6 6 5 12/30 Read block 292 len 24576 13/30 Flush 14/30 Write block 384 len 24576 data: 5 4 4 4 5 6 15/30 Read block 589 len 12288 16/30 Read block 301 len 8192 17/30 Read block 137 len 28672 18/30 Read block 298 len 36864 19/30 Read block 215 len 16384 20/30 Read block 273 len 16384 21/30 Write block 592 len 8192 data: 3 3 22/30 Read block 546 len 40960 23/30 Read block 17 len 36864 24/30 Flush 25/30 Write block 191 len 40960 data: 5 6 8 8 6 5 4 5 5 5 26/30 Read block 27 len 12288 27/30 Read block 452 len 4096 28/30 Read block 524 len 32768 29/30 Read block 491 len 32768 30/30 Write block 230 len 28672 data: 5 4 4 4 4 5 5 ---------------------------------------------------------------- Crucible gen:46 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 10 Done Done Done false 1025 Acked Read 3 Done Done Done false 1026 Acked Read 1 Done Done Done false 1027 Acked Read 8 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:12:57.870425607Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1366} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 42 42 42 71 71 71 F F F 1 020-039 45 45 45 76 76 76 F F F 2 040-059 41 41 41 67 67 67 F F F 3 060-079 46 46 46 78 78 78 F F F 4 080-099 45 45 45 76 76 76 F F F 5 100-119 45 45 45 76 76 76 F F F 6 120-139 36 36 36 56 56 56 F F F 7 140-159 46 46 46 78 78 78 F F F 8 160-179 46 46 46 78 78 78 F F F 9 180-199 42 42 42 71 71 71 T T T 10 200-219 45 45 45 76 76 76 T T T 11 220-239 45 45 45 76 76 76 T T T 12 240-259 45 45 45 76 76 76 F F F 13 260-279 45 45 45 76 76 76 F F F 14 280-299 45 45 45 76 76 76 F F F 15 300-319 45 45 45 76 76 76 F F F 16 320-339 46 46 46 79 79 79 F F F 17 340-359 40 40 40 66 66 66 F F F 18 360-379 45 45 45 76 76 76 F F F 19 380-399 46 46 46 80 80 80 F F F 20 400-419 45 45 45 76 76 76 F F F 21 420-439 45 45 45 76 76 76 F F F 22 440-459 45 45 45 76 76 76 F F F 23 460-479 45 45 45 76 76 76 F F F 24 480-499 42 42 42 71 71 71 F F F 25 500-519 42 42 42 71 71 71 F F F 26 520-539 40 40 40 66 66 66 F F F 27 540-559 34 34 34 52 52 52 F F F 28 560-579 43 43 43 73 73 73 F F F 29 580-599 46 46 46 80 80 80 F F F Max gen: 46, Max flush: 80 A Difference in extent metadata is expected here On loop 23 Verifying data now No change after verify Loop: 23 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 42 42 42 71 71 71 F F F 1 020-039 45 45 45 76 76 76 F F F 2 040-059 41 41 41 67 67 67 F F F 3 060-079 46 46 46 78 78 78 F F F 4 080-099 45 45 45 76 76 76 F F F 5 100-119 45 45 45 76 76 76 F F F 6 120-139 36 36 36 56 56 56 F F F 7 140-159 46 46 46 78 78 78 F F F 8 160-179 46 46 46 78 78 78 F F F 9 180-199 47 47 47 81 81 81 F F F 10 200-219 47 47 47 81 81 81 F F F 11 220-239 47 47 47 81 81 81 F F F 12 240-259 45 45 45 76 76 76 F F F 13 260-279 45 45 45 76 76 76 F F F 14 280-299 45 45 45 76 76 76 F F F 15 300-319 45 45 45 76 76 76 F F F 16 320-339 46 46 46 79 79 79 F F F 17 340-359 40 40 40 66 66 66 F F F 18 360-379 45 45 45 76 76 76 F F F 19 380-399 46 46 46 80 80 80 F F F 20 400-419 45 45 45 76 76 76 F F F 21 420-439 45 45 45 76 76 76 F F F 22 440-459 45 45 45 76 76 76 F F F 23 460-479 45 45 45 76 76 76 F F F 24 480-499 42 42 42 71 71 71 F F F 25 500-519 42 42 42 71 71 71 F F F 26 520-539 40 40 40 66 66 66 F F F 27 540-559 34 34 34 52 52 52 F F F 28 560-579 43 43 43 73 73 73 F F F 29 580-599 46 46 46 80 80 80 F F F Max gen: 47, Max flush: 81 Begin loop 24 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:02.1292884Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1374,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:13:02.172637046Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1374,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:02.173092802Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1374,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:02.205093025Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1374,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:02.205687613Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1374} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"abd5430f1508af0bc008143ea2566277ef714ec2\",\n git_commit_timestamp: \"2025-01-17T20:55:03.000000000Z\",\n git_branch: \"alan/tokio-to-140\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:02.205728156Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1374} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:02.205744576Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1374} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:02.205763017Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1374} {"msg":"Crucible 6f7968b1-8653-4f08-91a0-c09bfe5daf49 has session id: 6a23135a-c5e8-4828-a656-5ce66939f180","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:02.206408544Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1374,"session_id":"6a23135a-c5e8-4828-a656-5ce66939f180"} {"msg":"Upstairs opts: Upstairs UUID: 6f7968b1-8653-4f08-91a0-c09bfe5daf49, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:02.206442694Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1374,"session_id":"6a23135a-c5e8-4828-a656-5ce66939f180"} {"msg":"Crucible stats registered with UUID: 6f7968b1-8653-4f08-91a0-c09bfe5daf49","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:02.206464057Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1374,"session_id":"6a23135a-c5e8-4828-a656-5ce66939f180"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:13:02.206479876Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1374,"session_id":"6a23135a-c5e8-4828-a656-5ce66939f180"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:02.20703447Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1374,"task":"crutest"} {"msg":"The guest has requested activation with gen:48","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:02.207059345Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1374} {"msg":"6f7968b1-8653-4f08-91a0-c09bfe5daf49 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:02.207127865Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1374,"session_id":"6a23135a-c5e8-4828-a656-5ce66939f180"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:02.207144364Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1374,"client":"0","":"downstairs","session_id":"6a23135a-c5e8-4828-a656-5ce66939f180"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:02.207168678Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1374,"client":"0","":"downstairs","session_id":"6a23135a-c5e8-4828-a656-5ce66939f180"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:02.207183197Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1374,"client":"1","":"downstairs","session_id":"6a23135a-c5e8-4828-a656-5ce66939f180"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:02.207197125Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1374,"client":"1","":"downstairs","session_id":"6a23135a-c5e8-4828-a656-5ce66939f180"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:02.207209882Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1374,"client":"2","":"downstairs","session_id":"6a23135a-c5e8-4828-a656-5ce66939f180"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:02.207229534Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1374,"client":"2","":"downstairs","session_id":"6a23135a-c5e8-4828-a656-5ce66939f180"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:02.207300225Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1374,"":"io task","client":"2","":"downstairs","session_id":"6a23135a-c5e8-4828-a656-5ce66939f180"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:02.207336316Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1374,"":"io task","client":"0","":"downstairs","session_id":"6a23135a-c5e8-4828-a656-5ce66939f180"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:02.207361301Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1374,"":"io task","client":"1","":"downstairs","session_id":"6a23135a-c5e8-4828-a656-5ce66939f180"} {"msg":"ds_connection connected from Ok(127.0.0.1:55061)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:02.207388247Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1374,"":"io task","client":"2","":"downstairs","session_id":"6a23135a-c5e8-4828-a656-5ce66939f180"} {"msg":"ds_connection connected from Ok(127.0.0.1:40827)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:02.207421616Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1374,"":"io task","client":"0","":"downstairs","session_id":"6a23135a-c5e8-4828-a656-5ce66939f180"} {"msg":"ds_connection connected from Ok(127.0.0.1:47460)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:02.207447661Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1374,"":"io task","client":"1","":"downstairs","session_id":"6a23135a-c5e8-4828-a656-5ce66939f180"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:02.208114621Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1374,"client":"1","":"downstairs","session_id":"6a23135a-c5e8-4828-a656-5ce66939f180"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:02.208153064Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1374,"client":"2","":"downstairs","session_id":"6a23135a-c5e8-4828-a656-5ce66939f180"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:02.208257235Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1374,"client":"0","":"downstairs","session_id":"6a23135a-c5e8-4828-a656-5ce66939f180"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:02.208828038Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1374,"client":"2","":"downstairs","session_id":"6a23135a-c5e8-4828-a656-5ce66939f180"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:02.208855855Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1374,"client":"0","":"downstairs","session_id":"6a23135a-c5e8-4828-a656-5ce66939f180"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:02.208871444Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1374,"client":"1","":"downstairs","session_id":"6a23135a-c5e8-4828-a656-5ce66939f180"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:02.209393649Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1374,"session_id":"6a23135a-c5e8-4828-a656-5ce66939f180"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:02.209494358Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1374,"session_id":"6a23135a-c5e8-4828-a656-5ce66939f180"} {"msg":"[0]R flush_numbers[0..12]: [71, 76, 67, 78, 76, 76, 56, 78, 78, 81, 81, 81]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:02.209564178Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1374,"":"downstairs","":"downstairs","session_id":"6a23135a-c5e8-4828-a656-5ce66939f180"} {"msg":"[0]R generation[0..12]: [42, 45, 41, 46, 45, 45, 36, 46, 46, 47, 47, 47]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:02.20958292Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1374,"":"downstairs","":"downstairs","session_id":"6a23135a-c5e8-4828-a656-5ce66939f180"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:02.209597558Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1374,"":"downstairs","":"downstairs","session_id":"6a23135a-c5e8-4828-a656-5ce66939f180"} {"msg":"[1]R flush_numbers[0..12]: [71, 76, 67, 78, 76, 76, 56, 78, 78, 81, 81, 81]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:02.209611406Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1374,"":"downstairs","":"downstairs","session_id":"6a23135a-c5e8-4828-a656-5ce66939f180"} {"msg":"[1]R generation[0..12]: [42, 45, 41, 46, 45, 45, 36, 46, 46, 47, 47, 47]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:02.209630347Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1374,"":"downstairs","":"downstairs","session_id":"6a23135a-c5e8-4828-a656-5ce66939f180"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:02.209644476Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1374,"":"downstairs","":"downstairs","session_id":"6a23135a-c5e8-4828-a656-5ce66939f180"} {"msg":"[2]R flush_numbers[0..12]: [71, 76, 67, 78, 76, 76, 56, 78, 78, 81, 81, 81]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:02.209657863Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1374,"":"downstairs","":"downstairs","session_id":"6a23135a-c5e8-4828-a656-5ce66939f180"} {"msg":"[2]R generation[0..12]: [42, 45, 41, 46, 45, 45, 36, 46, 46, 47, 47, 47]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:02.209670941Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1374,"":"downstairs","":"downstairs","session_id":"6a23135a-c5e8-4828-a656-5ce66939f180"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:02.209684639Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1374,"":"downstairs","":"downstairs","session_id":"6a23135a-c5e8-4828-a656-5ce66939f180"} {"msg":"Max found gen is 48","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:02.209697216Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1374,"":"downstairs","":"downstairs","session_id":"6a23135a-c5e8-4828-a656-5ce66939f180"} {"msg":"Generation requested: 48 >= found:48","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:02.209709864Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1374,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO 6a23135a-c5e8-4828-a656-5ce66939f180"} {"msg":"Next flush: 82","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:02.209730426Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1374,"":"downstairs","":"downstairs","session_id":"6a23135a-c5e8-4828-a656-5ce66939f180"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:02.209748646Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1374,"":"downstairs","":"downstairs","session_id":"6a23135a-c5e8-4828-a656-5ce66939f180"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:02.209766747Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1374,"session_id":"6a23135a-c5e8-4828-a656-5ce66939f180"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:02.209778974Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1374,"client":"0","":"downstairs","session_id":"6a23135a-c5e8-4828-a656-5ce66939f180"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:02.209792002Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1374,"client":"1","":"downstairs","session_id":"6a23135a-c5e8-4828-a656-5ce66939f180"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:02.209804719Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1374,"client":"2","":"downstairs","session_id":"6a23135a-c5e8-4828-a656-5ce66939f180"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:02.209817607Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1374,"session_id":"6a23135a-c5e8-4828-a656-5ce66939f180"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:02.209829494Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1374,"session_id":"6a23135a-c5e8-4828-a656-5ce66939f180"} {"msg":"6f7968b1-8653-4f08-91a0-c09bfe5daf49 is now active with session: 6a23135a-c5e8-4828-a656-5ce66939f180","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:02.209844012Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1374,"session_id":"6a23135a-c5e8-4828-a656-5ce66939f180"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:02.209855699Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1374,"session_id":"6a23135a-c5e8-4828-a656-5ce66939f180"} {"msg":"The guest has finished waiting for activation with:48","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:02.209867506Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1374} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 221 len 40960 data: 4 6 7 6 5 5 5 4 4 6 02/30 Write block 418 len 8192 data: 2 2 03/30 Write block 326 len 28672 data: 5 4 3 3 4 4 4 04/30 Write block 125 len 24576 data: 2 2 3 3 3 4 05/30 Flush 06/30 Write block 303 len 12288 data: 4 4 4 07/30 Write block 92 len 20480 data: 3 3 4 5 5 08/30 Write block 523 len 24576 data: 8 6 5 5 4 2 09/30 Write block 481 len 24576 data: 4 4 3 4 3 3 10/30 Flush 11/30 Write block 153 len 8192 data: 3 4 12/30 Write block 575 len 28672 data: 3 4 4 3 5 5 4 13/30 Read block 99 len 20480 14/30 Read block 165 len 20480 15/30 Read block 32 len 28672 16/30 Write block 13 len 28672 data: 8 7 6 7 5 4 3 17/30 Read block 487 len 36864 18/30 Write block 347 len 8192 data: 5 5 19/30 Write block 495 len 36864 data: 3 4 3 4 6 7 8 7 8 20/30 Flush 21/30 Write block 67 len 40960 data: 4 5 5 5 6 6 5 5 8 8 22/30 Read block 292 len 4096 23/30 Read block 439 len 12288 24/30 Read block 436 len 24576 25/30 Read block 40 len 20480 26/30 Write block 311 len 12288 data: 3 4 4 27/30 Read block 555 len 36864 28/30 Read block 569 len 16384 29/30 Write block 209 len 4096 data: 4 30/30 Write block 430 len 32768 data: 6 7 6 4 4 3 2 2 ---------------------------------------------------------------- Crucible gen:48 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1020 Acked Write 10 Done Done Done false 1021 Acked Read 1 Done Done Done false 1022 Acked Read 3 Done Done Done false 1023 Acked Read 6 Done Done Done false 1024 Acked Read 5 Done Done Done false 1025 Acked Write 3 Done Done Done false 1026 Acked Read 9 Done Done Done false 1027 Acked Read 4 Done Done Done false 1028 Acked Write 1 Done Done Done false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 9 9 9 27 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1019 1019 1019 Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:13:02.228700543Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1374} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 48 48 48 84 84 84 F F F 1 020-039 45 45 45 76 76 76 F F F 2 040-059 41 41 41 67 67 67 F F F 3 060-079 46 46 46 78 78 78 T T T 4 080-099 48 48 48 83 83 83 F F F 5 100-119 45 45 45 76 76 76 F F F 6 120-139 48 48 48 82 82 82 F F F 7 140-159 48 48 48 84 84 84 F F F 8 160-179 46 46 46 78 78 78 F F F 9 180-199 47 47 47 81 81 81 F F F 10 200-219 47 47 47 81 81 81 T T T 11 220-239 48 48 48 82 82 82 F F F 12 240-259 45 45 45 76 76 76 F F F 13 260-279 45 45 45 76 76 76 F F F 14 280-299 45 45 45 76 76 76 F F F 15 300-319 48 48 48 83 83 83 T T T 16 320-339 48 48 48 82 82 82 F F F 17 340-359 48 48 48 84 84 84 F F F 18 360-379 45 45 45 76 76 76 F F F 19 380-399 46 46 46 80 80 80 F F F 20 400-419 48 48 48 82 82 82 F F F 21 420-439 45 45 45 76 76 76 T T T 22 440-459 45 45 45 76 76 76 F F F 23 460-479 45 45 45 76 76 76 F F F 24 480-499 48 48 48 84 84 84 F F F 25 500-519 48 48 48 84 84 84 F F F 26 520-539 48 48 48 83 83 83 F F F 27 540-559 34 34 34 52 52 52 F F F 28 560-579 48 48 48 84 84 84 F F F 29 580-599 48 48 48 84 84 84 F F F Max gen: 48, Max flush: 84 A Difference in extent metadata is expected here On loop 24 Verifying data now No change after verify Loop: 24 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 48 48 48 84 84 84 F F F 1 020-039 45 45 45 76 76 76 F F F 2 040-059 41 41 41 67 67 67 F F F 3 060-079 49 49 49 85 85 85 F F F 4 080-099 48 48 48 83 83 83 F F F 5 100-119 45 45 45 76 76 76 F F F 6 120-139 48 48 48 82 82 82 F F F 7 140-159 48 48 48 84 84 84 F F F 8 160-179 46 46 46 78 78 78 F F F 9 180-199 47 47 47 81 81 81 F F F 10 200-219 49 49 49 85 85 85 F F F 11 220-239 48 48 48 82 82 82 F F F 12 240-259 45 45 45 76 76 76 F F F 13 260-279 45 45 45 76 76 76 F F F 14 280-299 45 45 45 76 76 76 F F F 15 300-319 49 49 49 85 85 85 F F F 16 320-339 48 48 48 82 82 82 F F F 17 340-359 48 48 48 84 84 84 F F F 18 360-379 45 45 45 76 76 76 F F F 19 380-399 46 46 46 80 80 80 F F F 20 400-419 48 48 48 82 82 82 F F F 21 420-439 49 49 49 85 85 85 F F F 22 440-459 45 45 45 76 76 76 F F F 23 460-479 45 45 45 76 76 76 F F F 24 480-499 48 48 48 84 84 84 F F F 25 500-519 48 48 48 84 84 84 F F F 26 520-539 48 48 48 83 83 83 F F F 27 540-559 34 34 34 52 52 52 F F F 28 560-579 48 48 48 84 84 84 F F F 29 580-599 48 48 48 84 84 84 F F F Max gen: 49, Max flush: 85 Begin loop 25 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:06.493695629Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1382,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:06.537809202Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1382,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:06.538496644Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1382} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"abd5430f1508af0bc008143ea2566277ef714ec2\",\n git_commit_timestamp: \"2025-01-17T20:55:03.000000000Z\",\n git_branch: \"alan/tokio-to-140\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:06.538535747Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1382} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:06.538553698Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1382} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:06.538569597Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1382} {"msg":"Crucible 8e914f0f-3fe0-448a-a85c-e91ead8ee0bb has session id: 66c16499-df83-4ffe-aaa6-fe15ee27fb40","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:06.539015988Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1382,"session_id":"66c16499-df83-4ffe-aaa6-fe15ee27fb40"} {"msg":"Upstairs opts: Upstairs UUID: 8e914f0f-3fe0-448a-a85c-e91ead8ee0bb, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:06.539101037Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1382,"session_id":"66c16499-df83-4ffe-aaa6-fe15ee27fb40"} {"msg":"Crucible stats registered with UUID: 8e914f0f-3fe0-448a-a85c-e91ead8ee0bb","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:06.53912185Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1382,"session_id":"66c16499-df83-4ffe-aaa6-fe15ee27fb40"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:13:06.539134867Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1382,"session_id":"66c16499-df83-4ffe-aaa6-fe15ee27fb40"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:06.539571682Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1382,"task":"crutest"} {"msg":"The guest has requested activation with gen:50","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:06.539596757Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1382} {"msg":"8e914f0f-3fe0-448a-a85c-e91ead8ee0bb active request set","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:06.539661895Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1382,"session_id":"66c16499-df83-4ffe-aaa6-fe15ee27fb40"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:06.539679415Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1382,"client":"0","":"downstairs","session_id":"66c16499-df83-4ffe-aaa6-fe15ee27fb40"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:06.539706251Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1382,"client":"0","":"downstairs","session_id":"66c16499-df83-4ffe-aaa6-fe15ee27fb40"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:06.539730315Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1382,"client":"1","":"downstairs","session_id":"66c16499-df83-4ffe-aaa6-fe15ee27fb40"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:06.539757671Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1382,"client":"1","":"downstairs","session_id":"66c16499-df83-4ffe-aaa6-fe15ee27fb40"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:06.539782285Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1382,"client":"2","":"downstairs","session_id":"66c16499-df83-4ffe-aaa6-fe15ee27fb40"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:06.539815485Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1382,"client":"2","":"downstairs","session_id":"66c16499-df83-4ffe-aaa6-fe15ee27fb40"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:06.53984206Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1382,"":"io task","client":"2","":"downstairs","session_id":"66c16499-df83-4ffe-aaa6-fe15ee27fb40"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:06.539870407Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1382,"":"io task","client":"0","":"downstairs","session_id":"66c16499-df83-4ffe-aaa6-fe15ee27fb40"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:06.539897763Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1382,"":"io task","client":"1","":"downstairs","session_id":"66c16499-df83-4ffe-aaa6-fe15ee27fb40"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:13:06.539928181Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1382,"":"io task","client":"2","":"downstairs","session_id":"66c16499-df83-4ffe-aaa6-fe15ee27fb40"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:13:06.539958648Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1382,"":"io task","client":"2","":"downstairs","session_id":"66c16499-df83-4ffe-aaa6-fe15ee27fb40"} {"msg":"ds_connection connected from Ok(127.0.0.1:59782)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:06.539987245Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1382,"":"io task","client":"0","":"downstairs","session_id":"66c16499-df83-4ffe-aaa6-fe15ee27fb40"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:13:06.540084002Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1382,"session_id":"66c16499-df83-4ffe-aaa6-fe15ee27fb40"} {"msg":"ds_connection connected from Ok(127.0.0.1:64942)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:06.540116671Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1382,"":"io task","client":"1","":"downstairs","session_id":"66c16499-df83-4ffe-aaa6-fe15ee27fb40"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:06.540141956Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1382,"":"io task","client":"2","":"downstairs","session_id":"66c16499-df83-4ffe-aaa6-fe15ee27fb40"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:06.540607538Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1382,"client":"1","":"downstairs","session_id":"66c16499-df83-4ffe-aaa6-fe15ee27fb40"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:06.540635064Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1382,"client":"0","":"downstairs","session_id":"66c16499-df83-4ffe-aaa6-fe15ee27fb40"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:06.541318683Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1382,"client":"0","":"downstairs","session_id":"66c16499-df83-4ffe-aaa6-fe15ee27fb40"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:06.541345739Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1382,"client":"1","":"downstairs","session_id":"66c16499-df83-4ffe-aaa6-fe15ee27fb40"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:06.541855007Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1382,"session_id":"66c16499-df83-4ffe-aaa6-fe15ee27fb40"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:06.541878611Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1382,"session_id":"66c16499-df83-4ffe-aaa6-fe15ee27fb40"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:16.541723992Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1382,"":"io task","client":"2","":"downstairs","session_id":"66c16499-df83-4ffe-aaa6-fe15ee27fb40"} {"msg":"ds_connection connected from Ok(127.0.0.1:59041)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:16.541855359Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1382,"":"io task","client":"2","":"downstairs","session_id":"66c16499-df83-4ffe-aaa6-fe15ee27fb40"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:16.542851872Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1382,"client":"2","":"downstairs","session_id":"66c16499-df83-4ffe-aaa6-fe15ee27fb40"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:16.543460968Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1382,"client":"2","":"downstairs","session_id":"66c16499-df83-4ffe-aaa6-fe15ee27fb40"} {"msg":"[0]R flush_numbers[0..12]: [84, 76, 67, 85, 83, 76, 82, 84, 78, 81, 85, 82]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:16.544093757Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1382,"":"downstairs","":"downstairs","session_id":"66c16499-df83-4ffe-aaa6-fe15ee27fb40"} {"msg":"[0]R generation[0..12]: [48, 45, 41, 49, 48, 45, 48, 48, 46, 47, 49, 48]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:16.544119803Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1382,"":"downstairs","":"downstairs","session_id":"66c16499-df83-4ffe-aaa6-fe15ee27fb40"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:16.544138684Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1382,"":"downstairs","":"downstairs","session_id":"66c16499-df83-4ffe-aaa6-fe15ee27fb40"} {"msg":"[1]R flush_numbers[0..12]: [84, 76, 67, 85, 83, 76, 82, 84, 78, 81, 85, 82]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:16.544155504Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1382,"":"downstairs","":"downstairs","session_id":"66c16499-df83-4ffe-aaa6-fe15ee27fb40"} {"msg":"[1]R generation[0..12]: [48, 45, 41, 49, 48, 45, 48, 48, 46, 47, 49, 48]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:16.544172073Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1382,"":"downstairs","":"downstairs","session_id":"66c16499-df83-4ffe-aaa6-fe15ee27fb40"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:16.544188953Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1382,"":"downstairs","":"downstairs","session_id":"66c16499-df83-4ffe-aaa6-fe15ee27fb40"} {"msg":"[2]R flush_numbers[0..12]: [84, 76, 67, 85, 83, 76, 82, 84, 78, 81, 85, 82]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:16.544205373Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1382,"":"downstairs","":"downstairs","session_id":"66c16499-df83-4ffe-aaa6-fe15ee27fb40"} {"msg":"[2]R generation[0..12]: [48, 45, 41, 49, 48, 45, 48, 48, 46, 47, 49, 48]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:16.544221902Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1382,"":"downstairs","":"downstairs","session_id":"66c16499-df83-4ffe-aaa6-fe15ee27fb40"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:16.544244336Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1382,"":"downstairs","":"downstairs","session_id":"66c16499-df83-4ffe-aaa6-fe15ee27fb40"} {"msg":"Max found gen is 50","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-17T21:13:16.544260355Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1382,"":"downstairs","":"downstairs","session_id":"66c16499-df83-4ffe-aaa6-fe15ee27fb40"} {"msg":"Generation requested: 50 >= found:50","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:16.544282608Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1382,"":"downstairs","":"downstairs","session_id":"66c16499-df83-4ffe-aaa6-fe15ee27fb40"} {"msg":"Next flush: 86","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:16.544298317Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1382,"":"downstairs","":"downstairs","session_id":"66c16499-df83-4ffe-aaa6-fe15ee27fb40"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:16.544313856Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1382,"":"downstairs","":"downstairs","session_id":"66c16499-df83-4ffe-aaa6-fe15ee27fb40"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:16.544329705Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1382,"session_id":"66c16499-df83-4ffe-aaa6-fe15ee27fb40"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:16.544344124Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1382,"client":"0","":"downstairs","session_id":"66c16499-df83-4ffe-aaa6-fe15ee27fb40"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:16.544360293Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1382,"client":"1","":"downstairs","session_id":"66c16499-df83-4ffe-aaa6-fe15ee27fb40"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:16.544376353Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1382,"client":"2","":"downstairs","session_id":"66c16499-df83-4ffe-aaa6-fe15ee27fb40"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:16.544392442Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1382,"session_id":"66c16499-df83-4ffe-aaa6-fe15ee27fb40"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:16.544407301Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1382,"session_id":"66c16499-df83-4ffe-aaa6-fe15ee27fb40"} {"msg":"8e914f0f-3fe0-448a-a85c-e91ead8ee0bb is now active with session: 66c16499-df83-4ffe-aaa6-fe15ee27fb40","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:16.54442267Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1382,"session_id":"66c16499-df83-4ffe-aaa6-fe15ee27fb40"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:16.544437298Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1382,"session_id":"66c16499-df83-4ffe-aaa6-fe15ee27fb40"} {"msg":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 The guest has finished waiting for activation with:50","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:16.544455709Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1382} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 146 len 20480 data: 3 4 4 4 4 02/30 Flush 03/30 Write block 339 len 36864 data: 6 6 4 4 8 6 5 5 6 04/30 Read block 423 len 20480 05/30 Write block 49 len 40960 data: 3 4 7 9 9 8 7 8 8 7 06/30 Read block 191 len 32768 07/30 Read block 513 len 28672 08/30 Read block 358 len 36864 09/30 Write block 273 len 16384 data: 4 4 5 5 10/30 Read block 119 len 12288 11/30 Write block 509 len 4096 data: 8 12/30 Write block 180 len 12288 data: 5 6 5 13/30 Read block 519 len 20480 14/30 Flush 15/30 Write block 25 len 24576 data: 3 5 5 5 4 4 16/30 Write block 161 len 24576 data: 4 4 3 4 5 6 17/30 Write block 241 len 32768 data: 5 6 6 7 6 6 6 6 18/30 Read block 314 len 28672 19/30 Write block 353 len 16384 data: 6 6 6 7 20/30 Read block 122 len 12288 21/30 Write block 412 len 12288 data: 4 4 4 22/30 Read block 368 len 40960 23/30 Read block 296 len 12288 24/30 Flush 25/30 Write block 389 len 40960 data: 7 5 5 5 5 6 5 6 6 7 26/30 Read block 222 len 40960 27/30 Write block 79 len 28672 data: 6 5 5 5 5 3 3 28/30 Write block 25 len 16384 data: 4 6 6 6 29/30 Read block 390 len 16384 30/30 Read block 242 len 28672 ---------------------------------------------------------------- Crucible gen:50 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 10 Done Done Done false 1025 Acked Read 10 Done Done Done false 1026 Acked Write 7 Done Done Done false 1027 Acked Write 4 Done Done Done false 1028 Acked Read 4 Done Done Done false 1029 Acked Read 7 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 6 6 6 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:13:16.565552679Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1382} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 48 48 48 84 84 84 F F F 1 020-039 50 50 50 88 88 88 T T T 2 040-059 50 50 50 87 87 87 F F F 3 060-079 49 49 49 85 85 85 T T T 4 080-099 48 48 48 83 83 83 T T T 5 100-119 45 45 45 76 76 76 F F F 6 120-139 48 48 48 82 82 82 F F F 7 140-159 50 50 50 86 86 86 F F F 8 160-179 50 50 50 88 88 88 F F F 9 180-199 50 50 50 87 87 87 F F F 10 200-219 49 49 49 85 85 85 F F F 11 220-239 48 48 48 82 82 82 F F F 12 240-259 50 50 50 88 88 88 F F F 13 260-279 50 50 50 87 87 87 F F F 14 280-299 45 45 45 76 76 76 F F F 15 300-319 49 49 49 85 85 85 F F F 16 320-339 50 50 50 87 87 87 F F F 17 340-359 50 50 50 88 88 88 F F F 18 360-379 45 45 45 76 76 76 F F F 19 380-399 46 46 46 80 80 80 T T T 20 400-419 50 50 50 88 88 88 F F F 21 420-439 49 49 49 85 85 85 F F F 22 440-459 45 45 45 76 76 76 F F F 23 460-479 45 45 45 76 76 76 F F F 24 480-499 48 48 48 84 84 84 F F F 25 500-519 50 50 50 87 87 87 F F F 26 520-539 48 48 48 83 83 83 F F F 27 540-559 34 34 34 52 52 52 F F F 28 560-579 48 48 48 84 84 84 F F F 29 580-599 48 48 48 84 84 84 F F F Max gen: 50, Max flush: 88 A Difference in extent metadata is expected here On loop 25 Verifying data now No change after verify Loop: 25 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 48 48 48 84 84 84 F F F 1 020-039 51 51 51 89 89 89 F F F 2 040-059 50 50 50 87 87 87 F F F 3 060-079 51 51 51 89 89 89 F F F 4 080-099 51 51 51 89 89 89 F F F 5 100-119 45 45 45 76 76 76 F F F 6 120-139 48 48 48 82 82 82 F F F 7 140-159 50 50 50 86 86 86 F F F 8 160-179 50 50 50 88 88 88 F F F 9 180-199 50 50 50 87 87 87 F F F 10 200-219 49 49 49 85 85 85 F F F 11 220-239 48 48 48 82 82 82 F F F 12 240-259 50 50 50 88 88 88 F F F 13 260-279 50 50 50 87 87 87 F F F 14 280-299 45 45 45 76 76 76 F F F 15 300-319 49 49 49 85 85 85 F F F 16 320-339 50 50 50 87 87 87 F F F 17 340-359 50 50 50 88 88 88 F F F 18 360-379 45 45 45 76 76 76 F F F 19 380-399 51 51 51 89 89 89 F F F 20 400-419 50 50 50 88 88 88 F F F 21 420-439 49 49 49 85 85 85 F F F 22 440-459 45 45 45 76 76 76 F F F 23 460-479 45 45 45 76 76 76 F F F 24 480-499 48 48 48 84 84 84 F F F 25 500-519 50 50 50 87 87 87 F F F 26 520-539 48 48 48 83 83 83 F F F 27 540-559 34 34 34 52 52 52 F F F 28 560-579 48 48 48 84 84 84 F F F 29 580-599 48 48 48 84 84 84 F F F Max gen: 51, Max flush: 89 Begin loop 26 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:20.800455909Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1390,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:20.844519073Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1390,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:20.845171294Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1390} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"abd5430f1508af0bc008143ea2566277ef714ec2\",\n git_commit_timestamp: \"2025-01-17T20:55:03.000000000Z\",\n git_branch: \"alan/tokio-to-140\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:20.845211728Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1390} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:20.845226296Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1390} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:20.845240615Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1390} {"msg":"Crucible 9d629cfd-55b4-4018-8375-219e576ac0cb has session id: f7c5a8ee-dd81-45ff-9ce8-95724ae88bf4","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:20.845727649Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1390,"session_id":"f7c5a8ee-dd81-45ff-9ce8-95724ae88bf4"} {"msg":"Upstairs opts: Upstairs UUID: 9d629cfd-55b4-4018-8375-219e576ac0cb, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:20.845767973Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1390,"session_id":"f7c5a8ee-dd81-45ff-9ce8-95724ae88bf4"} {"msg":"Crucible stats registered with UUID: 9d629cfd-55b4-4018-8375-219e576ac0cb","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:20.84579796Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1390,"session_id":"f7c5a8ee-dd81-45ff-9ce8-95724ae88bf4"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:13:20.845819843Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1390,"session_id":"f7c5a8ee-dd81-45ff-9ce8-95724ae88bf4"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:20.846246112Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1390,"task":"crutest"} {"msg":"The guest has requested activation with gen:52","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:20.846280453Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1390} {"msg":"9d629cfd-55b4-4018-8375-219e576ac0cb active request set","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:20.846301665Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1390,"session_id":"f7c5a8ee-dd81-45ff-9ce8-95724ae88bf4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:20.846325469Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1390,"client":"0","":"downstairs","session_id":"f7c5a8ee-dd81-45ff-9ce8-95724ae88bf4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:20.84636239Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1390,"client":"0","":"downstairs","session_id":"f7c5a8ee-dd81-45ff-9ce8-95724ae88bf4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:20.846392848Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1390,"client":"1","":"downstairs","session_id":"f7c5a8ee-dd81-45ff-9ce8-95724ae88bf4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:20.846424076Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1390,"client":"1","":"downstairs","session_id":"f7c5a8ee-dd81-45ff-9ce8-95724ae88bf4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:20.846451112Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1390,"client":"2","":"downstairs","session_id":"f7c5a8ee-dd81-45ff-9ce8-95724ae88bf4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:20.846475797Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1390,"client":"2","":"downstairs","session_id":"f7c5a8ee-dd81-45ff-9ce8-95724ae88bf4"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:20.84649888Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1390,"":"io task","client":"2","":"downstairs","session_id":"f7c5a8ee-dd81-45ff-9ce8-95724ae88bf4"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:20.846522034Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1390,"":"io task","client":"0","":"downstairs","session_id":"f7c5a8ee-dd81-45ff-9ce8-95724ae88bf4"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:20.846546028Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1390,"":"io task","client":"1","":"downstairs","session_id":"f7c5a8ee-dd81-45ff-9ce8-95724ae88bf4"} {"msg":"ds_connection connected from Ok(127.0.0.1:40196)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:20.846570472Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1390,"":"io task","client":"2","":"downstairs","session_id":"f7c5a8ee-dd81-45ff-9ce8-95724ae88bf4"} {"msg":"ds_connection connected from Ok(127.0.0.1:64897)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:20.84660193Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1390,"":"io task","client":"0","":"downstairs","session_id":"f7c5a8ee-dd81-45ff-9ce8-95724ae88bf4"} {"msg":"ds_connection connected from Ok(127.0.0.1:46033)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:20.846629176Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1390,"":"io task","client":"1","":"downstairs","session_id":"f7c5a8ee-dd81-45ff-9ce8-95724ae88bf4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:20.847174075Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1390,"client":"0","":"downstairs","session_id":"f7c5a8ee-dd81-45ff-9ce8-95724ae88bf4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:20.847214238Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1390,"client":"1","":"downstairs","session_id":"f7c5a8ee-dd81-45ff-9ce8-95724ae88bf4"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:20.847891244Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1390,"client":"1","":"downstairs","session_id":"f7c5a8ee-dd81-45ff-9ce8-95724ae88bf4"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:20.847930026Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1390,"client":"0","":"downstairs","session_id":"f7c5a8ee-dd81-45ff-9ce8-95724ae88bf4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:20.848088459Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1390,"client":"2","":"downstairs","session_id":"f7c5a8ee-dd81-45ff-9ce8-95724ae88bf4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:20.84848324Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1390,"session_id":"f7c5a8ee-dd81-45ff-9ce8-95724ae88bf4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:20.848509655Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1390,"session_id":"f7c5a8ee-dd81-45ff-9ce8-95724ae88bf4"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:20.84873983Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1390,"client":"2","":"downstairs","session_id":"f7c5a8ee-dd81-45ff-9ce8-95724ae88bf4"} {"msg":"[0]R flush_numbers[0..12]: [84, 89, 87, 89, 89, 76, 82, 86, 88, 87, 85, 82]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:20.849211466Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1390,"":"downstairs","":"downstairs","session_id":"f7c5a8ee-dd81-45ff-9ce8-95724ae88bf4"} {"msg":"[0]R generation[0..12]: [48, 51, 50, 51, 51, 45, 48, 50, 50, 50, 49, 48]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:20.849231177Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1390,"":"downstairs","":"downstairs","session_id":"f7c5a8ee-dd81-45ff-9ce8-95724ae88bf4"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:20.849246186Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1390,"":"downstairs","":"downstairs","session_id":"f7c5a8ee-dd81-45ff-9ce8-95724ae88bf4"} {"msg":"[1]R flush_numbers[0..12]: [84, 89, 87, 89, 89, 76, 82, 86, 88, 87, 85, 82]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:20.849260034Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1390,"":"downstairs","":"downstairs","session_id":"f7c5a8ee-dd81-45ff-9ce8-95724ae88bf4"} {"msg":"[1]R generation[0..12]: [48, 51, 50, 51, 51, 45, 48, 50, 50, 50, 49, 48]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:20.849276654Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1390,"":"downstairs","":"downstairs","session_id":"f7c5a8ee-dd81-45ff-9ce8-95724ae88bf4"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:20.849292783Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1390,"":"downstairs","":"downstairs","session_id":"f7c5a8ee-dd81-45ff-9ce8-95724ae88bf4"} {"msg":"[2]R flush_numbers[0..12]: [84, 89, 87, 89, 89, 76, 82, 86, 88, 87, 85, 82]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:20.849317017Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1390,"":"downstairs","":"downstairs","session_id":"f7c5a8ee-dd81-45ff-9ce8-95724ae88bf4"} {"msg":"[2]R generation[0..12]: [48, 51, 50, 51, 51, 45, 48, 50, 50, 50, 49, 48]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:20.849343403Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1390,"":"downstairs","":"downstairs","session_id":"f7c5a8ee-dd81-45ff-9ce8-95724ae88bf4"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:20.849371289Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":Wait for a query_work_queue command to finish before sending IO 1390,"":"downstairs","":"downstairs","session_id":"f7c5a8ee-dd81-45ff-9ce8-95724ae88bf4"} {"msg":"Max found gen is 52","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:20.849402647Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1390,"":"downstairs","":"downstairs","session_id":"f7c5a8ee-dd81-45ff-9ce8-95724ae88bf4"} {"msg":"Generation requested: 52 >= found:52","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:20.849427682Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1390,"":"downstairs","":"downstairs","session_id":"f7c5a8ee-dd81-45ff-9ce8-95724ae88bf4"} {"msg":"Next flush: 90","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:20.84944178Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1390,"":"downstairs","":"downstairs","session_id":"f7c5a8ee-dd81-45ff-9ce8-95724ae88bf4"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:20.849455188Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1390,"":"downstairs","":"downstairs","session_id":"f7c5a8ee-dd81-45ff-9ce8-95724ae88bf4"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:20.849477181Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1390,"session_id":"f7c5a8ee-dd81-45ff-9ce8-95724ae88bf4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:20.849500114Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1390,"client":"0","":"downstairs","session_id":"f7c5a8ee-dd81-45ff-9ce8-95724ae88bf4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:20.849522427Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1390,"client":"1","":"downstairs","session_id":"f7c5a8ee-dd81-45ff-9ce8-95724ae88bf4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:20.849538987Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1390,"client":"2","":"downstairs","session_id":"f7c5a8ee-dd81-45ff-9ce8-95724ae88bf4"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:20.849562411Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1390,"session_id":"f7c5a8ee-dd81-45ff-9ce8-95724ae88bf4"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:20.849585244Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1390,"session_id":"f7c5a8ee-dd81-45ff-9ce8-95724ae88bf4"} {"msg":"9d629cfd-55b4-4018-8375-219e576ac0cb is now active with session: f7c5a8ee-dd81-45ff-9ce8-95724ae88bf4","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:20.849614081Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1390,"session_id":"f7c5a8ee-dd81-45ff-9ce8-95724ae88bf4"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:20.84962871Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1390,"session_id":"f7c5a8ee-dd81-45ff-9ce8-95724ae88bf4"} {"msg":"The guest has finished waiting for activation with:52","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:20.849642818Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1390} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 194 len 24576 data: 9 7 6 5 6 6 02/30 Flush 03/30 Write block 489 len 32768 data: 4 4 4 3 4 4 4 5 04/30 Read block 282 len 24576 05/30 Write block 383 len 32768 data: 5 6 5 5 5 6 8 6 06/30 Read block 188 len 16384 07/30 Read block 420 len 4096 08/30 Write block 284 len 40960 data: 5 5 4 3 3 3 4 4 5 4 09/30 Write block 21 len 40960 data: 4 3 3 3 5 7 7 7 5 5 10/30 Flush 11/30 Write block 54 len 24576 data: 9 8 9 9 8 6 12/30 Write block 539 len 20480 data: 3 3 3 4 6 13/30 Write block 412 len 12288 data: 5 5 5 14/30 Read block 267 len 28672 15/30 Read block 135 len 28672 16/30 Read block 10 len 20480 17/30 Read block 344 len 36864 18/30 Read block 350 len 16384 19/30 Read block 84 len 8192 20/30 Read block 141 len 12288 21/30 Write block 33 len 40960 data: 7 6 7 4 5 5 5 4 4 3 22/30 Write block 61 len 8192 data: 5 3 23/30 Write block 75 len 16384 data: 9 9 8 7 24/30 Flush 25/30 Write block 92 len 12288 data: 4 4 5 26/30 Write block 334 len 40960 data: 6 7 8 7 7 7 7 5 5 9 27/30 Write block 18 len 28672 data: 5 4 2 5 4 4 4 28/30 Read block 472 len 20480 29/30 Write block 97 len 40960 data: 5 7 6 6 6 6 5 5 5 2 30/30 Write block 52 len 12288 data: 10 10 10 ---------------------------------------------------------------- Crucible gen:52 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 3 Done Done Done false 1025 Acked Write 10 Done Done Done false 1026 Acked Write 7 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Write 10 Sent Sent Sent false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:13:20.869420317Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1390} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 48 48 48 84 84 84 T T T 1 020-039 52 52 52 92 92 92 T T T 2 040-059 52 52 52 92 92 92 T T T 3 060-079 52 52 52 92 92 92 F F F 4 080-099 51 51 51 89 89 89 T T T 5 100-119 45 45 45 76 76 76 T T T 6 120-139 48 48 48 82 82 82 F F F 7 140-159 50 50 50 86 86 86 F F F 8 160-179 50 50 50 88 88 88 F F F 9 180-199 52 52 52 90 90 90 F F F 10 200-219 49 49 49 85 85 85 F F F 11 220-239 48 48 48 82 82 82 F F F 12 240-259 50 50 50 88 88 88 F F F 13 260-279 50 50 50 87 87 87 F F F 14 280-299 52 52 52 91 91 91 F F F 15 300-319 49 49 49 85 85 85 F F F 16 320-339 50 50 50 87 87 87 T T T 17 340-359 50 50 50 88 88 88 T T T 18 360-379 45 45 45 76 76 76 F F F 19 380-399 52 52 52 91 91 91 F F F 20 400-419 52 52 52 92 92 92 F F F 21 420-439 49 49 49 85 85 85 F F F 22 440-459 45 45 45 76 76 76 F F F 23 460-479 45 45 45 76 76 76 F F F 24 480-499 52 52 52 91 91 91 F F F 25 500-519 50 50 50 87 87 87 F F F 26 520-539 52 52 52 92 92 92 F F F 27 540-559 52 52 52 92 92 92 F F F 28 560-579 48 48 48 84 84 84 F F F 29 580-599 48 48 48 84 84 84 F F F Max gen: 52, Max flush: 92 A Difference in extent metadata is expected here On loop 26 Verifying data now No change after verify Loop: 26 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 53 53 53 93 93 93 F F F 1 020-039 53 53 53 93 93 93 F F F 2 040-059 53 53 53 93 93 93 F F F 3 060-079 52 52 52 92 92 92 F F F 4 080-099 53 53 53 93 93 93 F F F 5 100-119 53 53 53 93 93 93 F F F 6 120-139 48 48 48 82 82 82 F F F 7 140-159 50 50 50 86 86 86 F F F 8 160-179 50 50 50 88 88 88 F F F 9 180-199 52 52 52 90 90 90 F F F 10 200-219 49 49 49 85 85 85 F F F 11 220-239 48 48 48 82 82 82 F F F 12 240-259 50 50 50 88 88 88 F F F 13 260-279 50 50 50 87 87 87 F F F 14 280-299 52 52 52 91 91 91 F F F 15 300-319 49 49 49 85 85 85 F F F 16 320-339 53 53 53 93 93 93 F F F 17 340-359 53 53 53 93 93 93 F F F 18 360-379 45 45 45 76 76 76 F F F 19 380-399 52 52 52 91 91 91 F F F 20 400-419 52 52 52 92 92 92 F F F 21 420-439 49 49 49 85 85 85 F F F 22 440-459 45 45 45 76 76 76 F F F 23 460-479 45 45 45 76 76 76 F F F 24 480-499 52 52 52 91 91 91 F F F 25 500-519 50 50 50 87 87 87 F F F 26 520-539 52 52 52 92 92 92 F F F 27 540-559 52 52 52 92 92 92 F F F 28 560-579 48 48 48 84 84 84 F F F 29 580-599 48 48 48 84 84 84 F F F Max gen: 53, Max flush: 93 Begin loop 27 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:25.126020097Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1398,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:25.170350027Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1398,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:25.171014575Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1398} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"abd5430f1508af0bc008143ea2566277ef714ec2\",\n git_commit_timestamp: \"2025-01-17T20:55:03.000000000Z\",\n git_branch: \"alan/tokio-to-140\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:25.17105722Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1398} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:25.171075961Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1398} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:25.17109078Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1398} {"msg":"Crucible 74297b88-f437-4020-ad21-fac87bfba179 has session id: 416d271e-3d01-42ca-a32a-1dda1ce14774","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:25.171613655Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1398,"session_id":"416d271e-3d01-42ca-a32a-1dda1ce14774"} {"msg":"Upstairs opts: Upstairs UUID: 74297b88-f437-4020-ad21-fac87bfba179, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:25.171652438Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1398,"session_id":"416d271e-3d01-42ca-a32a-1dda1ce14774"} {"msg":"Crucible stats registered with UUID: 74297b88-f437-4020-ad21-fac87bfba179","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:25.171671309Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1398,"session_id":"416d271e-3d01-42ca-a32a-1dda1ce14774"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:13:25.171687549Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1398,"session_id":"416d271e-3d01-42ca-a32a-1dda1ce14774"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:25.172147678Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1398,"task":"crutest"} {"msg":"The guest has requested activation with gen:54","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:25.172171962Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1398} {"msg":"74297b88-f437-4020-ad21-fac87bfba179 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:25.172252139Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1398,"session_id":"416d271e-3d01-42ca-a32a-1dda1ce14774"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:25.172275532Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1398,"client":"0","":"downstairs","session_id":"416d271e-3d01-42ca-a32a-1dda1ce14774"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:25.172314065Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1398,"client":"0","":"downstairs","session_id":"416d271e-3d01-42ca-a32a-1dda1ce14774"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:25.172340691Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1398,"client":"1","":"downstairs","session_id":"416d271e-3d01-42ca-a32a-1dda1ce14774"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:25.172369617Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1398,"client":"1","":"downstairs","session_id":"416d271e-3d01-42ca-a32a-1dda1ce14774"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:25.172396113Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1398,"client":"2","":"downstairs","session_id":"416d271e-3d01-42ca-a32a-1dda1ce14774"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:25.172423139Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1398,"client":"2","":"downstairs","session_id":"416d271e-3d01-42ca-a32a-1dda1ce14774"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:25.172453906Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1398,"":"io task","client":"2","":"downstairs","session_id":"416d271e-3d01-42ca-a32a-1dda1ce14774"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:25.172481593Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1398,"":"io task","client":"0","":"downstairs","session_id":"416d271e-3d01-42ca-a32a-1dda1ce14774"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:25.17251048Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1398,"":"io task","client":"1","":"downstairs","session_id":"416d271e-3d01-42ca-a32a-1dda1ce14774"} {"msg":"ds_connection connected from Ok(127.0.0.1:49540)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:25.172539026Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1398,"":"io task","client":"2","":"downstairs","session_id":"416d271e-3d01-42ca-a32a-1dda1ce14774"} {"msg":"ds_connection connected from Ok(127.0.0.1:36439)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:25.172564741Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1398,"":"io task","client":"0","":"downstairs","session_id":"416d271e-3d01-42ca-a32a-1dda1ce14774"} {"msg":"ds_connection connected from Ok(127.0.0.1:59140)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:25.172589866Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1398,"":"io task","client":"1","":"downstairs","session_id":"416d271e-3d01-42ca-a32a-1dda1ce14774"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:25.17326346Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1398,"client":"2","":"downstairs","session_id":"416d271e-3d01-42ca-a32a-1dda1ce14774"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:25.173429457Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1398,"client":"0","":"downstairs","session_id":"416d271e-3d01-42ca-a32a-1dda1ce14774"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:25.173933061Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1398,"client":"2","":"downstairs","session_id":"416d271e-3d01-42ca-a32a-1dda1ce14774"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:25.17408462Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1398,"client":"0","":"downstairs","session_id":"416d271e-3d01-42ca-a32a-1dda1ce14774"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:25.174394521Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1398,"client":"1","":"downstairs","session_id":"416d271e-3d01-42ca-a32a-1dda1ce14774"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:25.174531241Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1398,"session_id":"416d271e-3d01-42ca-a32a-1dda1ce14774"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:25.174597169Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1398,"session_id":"416d271e-3d01-42ca-a32a-1dda1ce14774"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:25.17494021Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1398,"client":"1","":"downstairs","session_id":"416d271e-3d01-42ca-a32a-1dda1ce14774"} {"msg":"[0]R flush_numbers[0..12]: [93, 93, 93, 92, 93, 93, 82, 86, 88, 90, 85, 82]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:25.175416338Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1398,"":"downstairs","":"downstairs","session_id":"416d271e-3d01-42ca-a32a-1dda1ce14774"} {"msg":"[0]R generation[0..12]: [53, 53, 53, 52, 53, 53, 48, 50, 50, 52, 49, 48]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:25.17543625Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1398,"":"downstairs","":"downstairs","session_id":"416d271e-3d01-42ca-a32a-1dda1ce14774"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:25.175450668Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1398,"":"downstairs","":"downstairs","session_id":"416d271e-3d01-42ca-a32a-1dda1ce14774"} {"msg":"[1]R flush_numbers[0..12]: [93, 93, 93, 92, 93, 93, 82, 86, 88, 90, 85, 82]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:25.175464376Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1398,"":"downstairs","":"downstairs","session_id":"416d271e-3d01-42ca-a32a-1dda1ce14774"} {"msg":"[1]R generation[0..12]: [53, 53, 53, 52, 53, 53, 48, 50, 50, 52, 49, 48]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:25.175477694Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1398,"":"downstairs","":"downstairs","session_id":"416d271e-3d01-42ca-a32a-1dda1ce14774"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:25.175491452Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1398,"":"downstairs","":"downstairs","session_id":"416d271e-3d01-42ca-a32a-1dda1ce14774"} {"msg":"[2]R flush_numbers[0..12]: [93, 93, 93, 92, 93, 93, 82, 86, 88, 90, 85, 82]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:25.175504469Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1398,"":"downstairs","":"downstairs","session_id":"416d271e-3d01-42ca-a32a-1dda1ce14774"} {"msg":"[2]R generation[0..12]: [53, 53, 53, 52, 53, 53, 48, 50, 50, 52, 49, 48]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:25.175517487Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1398,"":"downstairs","":"downstairs","session_id":"416d271e-3d01-42ca-a32a-1dda1ce14774"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:25.175530875Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1398,"":"downstairs","":"downstairs","session_id":"416d271e-3d01-42ca-a32a-1dda1ce14774"} {"msg":"Max found gen is 54","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:25.175543372Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1398,"":"downstairs","":"downstairs","session_id":"416d271e-3d01-42ca-a32a-1dda1ce14774"} {"msg":"Generation requested: 54 >= found:54","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:25.17555619Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1398,"":"downstairs","":"downstairs","session_id":"416d271e-3d01-42ca-a32a-1dda1ce14774"} {"msg":"Next flush: 94","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:25.175568667Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1398,"":"downstairs","":"downstairs","session_id":"416d271e-3d01-42ca-a32a-1dda1ce14774"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-17T21:13:25.175581094Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1398,"":"downstairs","":"downstairs","session_id":"416d271e-3d01-42ca-a32a-1dda1ce14774"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:25.175596964Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1398,"session_id":"416d271e-3d01-42ca-a32a-1dda1ce14774"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:25.175610822Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1398,"client":"0","":"downstairs","session_id":"416d271e-3d01-42ca-a32a-1dda1ce14774"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:25.175623989Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1398,"client":"1","":"downstairs","session_id":"416d271e-3d01-42ca-a32a-1dda1ce14774"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:25.175636617Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1398,"client":"2","":"downstairs","session_id":"416d271e-3d01-42ca-a32a-1dda1ce14774"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:25.175649284Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1398,"session_id":"416d271e-3d01-42ca-a32a-1dda1ce14774"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:25.175661351Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1398,"session_id":"416d271e-3d01-42ca-a32a-1dda1ce14774"} {"msg":"74297b88-f437-4020-ad21-fac87bfba179 is now active with session: 416d271e-3d01-42ca-a32a-1dda1ce14774","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:25.175673679Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1398,"session_id":"416d271e-3d01-42ca-a32a-1dda1ce14774"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:25.175685185Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1398,"session_id":"416d271e-3d01-42ca-a32a-1dda1ce14774"} {"msg":"The guest has finished waiting for activation with:54","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:25.175696922Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1398} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 576 len 12288 data: 5 5 4 02/30 Write block 299 len 20480 data: 6 5 5 3 5 03/30 Read block 433 len 36864 04/30 Write block 305 len 8192 data: 5 4 05/30 Write block 514 len 40960 data: 7 10 10 7 8 8 8 8 6 9 06/30 Write block 402 len 36864 data: 5 7 6 6 5 4 3 4 4 07/30 Write block 426 len 32768 data: 5 6 7 7 7 8 7 5 08/30 Read block 391 len 32768 09/30 Read block 27 len 36864 10/30 Flush 11/30 Write block 270 len 12288 data: 5 4 4 12/30 Read block 188 len 20480 13/30 Write block 306 len 20480 data: 5 5 4 4 4 14/30 Write block 230 len 24576 data: 7 5 5 5 5 6 15/30 Read block 448 len 16384 16/30 Read block 70 len 12288 17/30 Read block 527 len 28672 18/30 Write block 454 len 32768 data: 5 5 5 6 6 5 5 6 19/30 Read block 460 len 12288 20/30 Read block 547 len 16384 21/30 Write block 233 len 36864 data: 6 6 7 6 5 3 3 4 6 22/30 Read block 297 len 32768 23/30 Read block 29 len 16384 24/30 Flush 25/30 Write block 98 len 40960 data: 8 7 7 7 7 6 6 6 3 3 26/30 Read block 356 len 36864 27/30 Read block 529 len 16384 28/30 Read block 73 len 20480 29/30 Read block 342 len 12288 30/30 Write block 452 len 8192 data: 5 4 ---------------------------------------------------------------- Crucible gen:54 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 10 Done Done Done false 1025 Acked Read 9 Done Done Done false 1026 Acked Read 4 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Read 3 Done Done Done false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:13:25.196220157Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1398} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 53 53 53 93 93 93 F F F 1 020-039 53 53 53 93 93 93 F F F 2 040-059 53 53 53 93 93 93 F F F 3 060-079 52 52 52 92 92 92 F F F 4 080-099 53 53 53 93 93 93 T T T 5 100-119 53 53 53 93 93 93 T T T 6 120-139 48 48 48 82 82 82 F F F 7 140-159 50 50 50 86 86 86 F F F 8 160-179 50 50 50 88 88 88 F F F 9 180-199 52 52 52 90 90 90 F F F 10 200-219 49 49 49 85 85 85 F F F 11 220-239 54 54 54 95 95 95 F F F 12 240-259 54 54 54 95 95 95 F F F 13 260-279 54 54 54 95 95 95 F F F 14 280-299 54 54 54 94 94 94 F F F 15 300-319 54 54 54 95 95 95 F F F 16 320-339 53 53 53 93 93 93 F F F 17 340-359 53 53 53 93 93 93 F F F 18 360-379 45 45 45 76 76 76 F F F 19 380-399 52 52 52 91 91 91 F F F 20 400-419 54 54 54 94 94 94 F F F 21 420-439 54 54 54 94 94 94 F F F 22 440-459 54 54 54 95 95 95 T T T 23 460-479 54 54 54 95 95 95 F F F 24 480-499 52 52 52 91 91 91 F F F 25 500-519 54 54 54 94 94 94 F F F 26 520-539 54 54 54 94 94 94 F F F 27 540-559 52 52 52 92 92 92 F F F 28 560-579 54 54 54 94 94 94 F F F 29 580-599 48 48 48 84 84 84 F F F Max gen: 54, Max flush: 95 A Difference in extent metadata is expected here On loop 27 Verifying data now No change after verify Loop: 27 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 53 53 53 93 93 93 F F F 1 020-039 53 53 53 93 93 93 F F F 2 040-059 53 53 53 93 93 93 F F F 3 060-079 52 52 52 92 92 92 F F F 4 080-099 55 55 55 96 96 96 F F F 5 100-119 55 55 55 96 96 96 F F F 6 120-139 48 48 48 82 82 82 F F F 7 140-159 50 50 50 86 86 86 F F F 8 160-179 50 50 50 88 88 88 F F F 9 180-199 52 52 52 90 90 90 F F F 10 200-219 49 49 49 85 85 85 F F F 11 220-239 54 54 54 95 95 95 F F F 12 240-259 54 54 54 95 95 95 F F F 13 260-279 54 54 54 95 95 95 F F F 14 280-299 54 54 54 94 94 94 F F F 15 300-319 54 54 54 95 95 95 F F F 16 320-339 53 53 53 93 93 93 F F F 17 340-359 53 53 53 93 93 93 F F F 18 360-379 45 45 45 76 76 76 F F F 19 380-399 52 52 52 91 91 91 F F F 20 400-419 54 54 54 94 94 94 F F F 21 420-439 54 54 54 94 94 94 F F F 22 440-459 55 55 55 96 96 96 F F F 23 460-479 54 54 54 95 95 95 F F F 24 480-499 52 52 52 91 91 91 F F F 25 500-519 54 54 54 94 94 94 F F F 26 520-539 54 54 54 94 94 94 F F F 27 540-559 52 52 52 92 92 92 F F F 28 560-579 54 54 54 94 94 94 F F F 29 580-599 48 48 48 84 84 84 F F F Max gen: 55, Max flush: 96 Begin loop 28 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:29.426744017Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1406,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:29.471104955Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1406,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:29.471752853Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1406} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"abd5430f1508af0bc008143ea2566277ef714ec2\",\n git_commit_timestamp: \"2025-01-17T20:55:03.000000000Z\",\n git_branch: \"alan/tokio-to-140\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:29.471802842Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1406} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:29.471820663Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1406} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:29.471838173Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1406} {"msg":"Crucible 745cc631-9335-4030-ba4e-476fd3a3d6a0 has session id: 04916cfb-485b-40b9-a738-b94e6118c324","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:29.472368053Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1406,"session_id":"04916cfb-485b-40b9-a738-b94e6118c324"} {"msg":"Upstairs opts: Upstairs UUID: 745cc631-9335-4030-ba4e-476fd3a3d6a0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:29.472429949Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1406,"session_id":"04916cfb-485b-40b9-a738-b94e6118c324"} {"msg":"Crucible stats registered with UUID: 745cc631-9335-4030-ba4e-476fd3a3d6a0","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:29.472460437Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1406,"session_id":"04916cfb-485b-40b9-a738-b94e6118c324"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:13:29.472485451Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1406,"session_id":"04916cfb-485b-40b9-a738-b94e6118c324"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:29.473000382Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1406,"task":"crutest"} {"msg":"The guest has requested activation with gen:56","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:29.473023266Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1406} {"msg":"745cc631-9335-4030-ba4e-476fd3a3d6a0 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:29.473083751Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1406,"session_id":"04916cfb-485b-40b9-a738-b94e6118c324"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:29.473100731Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1406,"client":"0","":"downstairs","session_id":"04916cfb-485b-40b9-a738-b94e6118c324"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:29.473124235Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1406,"client":"0","":"downstairs","session_id":"04916cfb-485b-40b9-a738-b94e6118c324"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:29.473141785Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1406,"client":"1","":"downstairs","session_id":"04916cfb-485b-40b9-a738-b94e6118c324"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:29.473155933Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1406,"client":"1","":"downstairs","session_id":"04916cfb-485b-40b9-a738-b94e6118c324"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:29.473168901Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1406,"client":"2","":"downstairs","session_id":"04916cfb-485b-40b9-a738-b94e6118c324"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:29.473182499Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1406,"client":"2","":"downstairs","session_id":"04916cfb-485b-40b9-a738-b94e6118c324"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:29.473195106Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1406,"":"io task","client":"2","":"downstairs","session_id":"04916cfb-485b-40b9-a738-b94e6118c324"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:29.473213367Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1406,"":"io task","client":"0","":"downstairs","session_id":"04916cfb-485b-40b9-a738-b94e6118c324"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:29.47323534Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1406,"":"io task","client":"1","":"downstairs","session_id":"04916cfb-485b-40b9-a738-b94e6118c324"} {"msg":"ds_connection connected from Ok(127.0.0.1:39109)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:29.473312835Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1406,"":"io task","client":"1","":"downstairs","session_id":"04916cfb-485b-40b9-a738-b94e6118c324"} {"msg":"ds_connection connected from Ok(127.0.0.1:38366)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:29.473345244Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1406,"":"io task","client":"0","":"downstairs","session_id":"04916cfb-485b-40b9-a738-b94e6118c324"} {"msg":"ds_connection connected from Ok(127.0.0.1:59924)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:29.473529802Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1406,"":"io task","client":"2","":"downstairs","session_id":"04916cfb-485b-40b9-a738-b94e6118c324"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:29.474100875Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1406,"client":"0","":"downstairs","session_id":"04916cfb-485b-40b9-a738-b94e6118c324"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:29.474128762Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1406,"client":"1","":"downstairs","session_id":"04916cfb-485b-40b9-a738-b94e6118c324"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:29.474144691Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1406,"client":"2","":"downstairs","session_id":"04916cfb-485b-40b9-a738-b94e6118c324"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:29.474693782Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1406,"client":"0","":"downstairs","session_id":"04916cfb-485b-40b9-a738-b94e6118c324"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:29.474858998Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1406,"client":"1","":"downstairs","session_id":"04916cfb-485b-40b9-a738-b94e6118c324"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:29.474887045Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1406,"client":"2","":"downstairs","session_id":"04916cfb-485b-40b9-a738-b94e6118c324"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:29.475222111Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1406,"session_id":"04916cfb-485b-40b9-a738-b94e6118c324"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:29.475440709Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1406,"session_id":"04916cfb-485b-40b9-a738-b94e6118c324"} {"msg":"[0]R flush_numbers[0..12]: [93, 93, 93, 92, 96, 96, 82, 86, 88, 90, 85, 95]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:29.475464643Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1406,"":"downstairs","":"downstairs","session_id":"04916cfb-485b-40b9-a738-b94e6118c324"} {"msg":"[0]R generation[0..12]: [53, 53, 53, 52, 55, 55, 48, 50, 50, 52, 49, 54]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:29.475479691Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1406,"":"downstairs","":"downstairs","session_id":"04916cfb-485b-40b9-a738-b94e6118c324"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:29.47549386Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1406,"":"downstairs","":"downstairs","session_id":"04916cfb-485b-40b9-a738-b94e6118c324"} {"msg":"[1]R flush_numbers[0..12]: [93, 93, 93, 92, 96, 96, 82, 86, 88, 90, 85, 95]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:29.475507228Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1406,"":"downstairs","":"downstairs","session_id":"04916cfb-485b-40b9-a738-b94e6118c324"} {"msg":"[1]R generation[0..12]: [53, 53, 53, 52, 55, 55, 48, 50, 50, 52, 49, 54]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:29.475520335Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1406,"":"downstairs","":"downstairs","session_id":"04916cfb-485b-40b9-a738-b94e6118c324"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:29.475534043Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1406,"":"downstairs","":"downstairs","session_id":"04916cfb-485b-40b9-a738-b94e6118c324"} {"msg":"[2]R flush_numbers[0..12]: [93, 93, 93, 92, 96, 96, 82, 86, 88, 90, 85, 95]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:29.475547251Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1406,"":"downstairs","":"downstairs","session_id":"04916cfb-485b-40b9-a738-b94e6118c324"} {"msg":"[2]R generation[0..12]: [53, 53, 53, 52, 55, 55, 48, 50, 50, 52, 49, 54]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:29.475560459Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1406,"":"downstairs","":"downstairs","session_id":"04916cfb-485b-40b9-a738-b94e6118c324"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:29.475573996Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1406,"":"downstairs","":"downstairs","session_id":"04916cfb-485b-40b9-a738-b94e6118c324"} {"msg":"Max found gen is 56","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:29.475586434Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1406,"":"downstairs","":"downstairs","session_id":"04916cfb-485b-40b9-a738-b94e6118c324"} {"msg":"Generation requested: 56 >= found:56","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:29.475603004Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1406,"":"downstairs","":"downstairs","session_id":"04916cfb-485b-40b9-a738-b94e6118c324"} {"msg":"Next flush: 97","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:29.475615751Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1406,"":"downstairs","":"downstairs","session_id":"04916cfb-485b-40b9-a738-b94e6118c324"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:29.475628308Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1406,"":"downstairs","":"downstairs","session_id":"04916cfb-485b-40b9-a738-b94e6118c324"} Wait for a query_work_queue command to finish before sending IO {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:29.475649391Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1406,"session_id":"04916cfb-485b-40b9-a738-b94e6118c324"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:29.475670893Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1406,"client":"0","":"downstairs","session_id":"04916cfb-485b-40b9-a738-b94e6118c324"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:29.475696708Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1406,"client":"1","":"downstairs","session_id":"04916cfb-485b-40b9-a738-b94e6118c324"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:29.475722403Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1406,"client":"2","":"downstairs","session_id":"04916cfb-485b-40b9-a738-b94e6118c324"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:29.475749999Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1406,"session_id":"04916cfb-485b-40b9-a738-b94e6118c324"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:29.475774824Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1406,"session_id":"04916cfb-485b-40b9-a738-b94e6118c324"} {"msg":"745cc631-9335-4030-ba4e-476fd3a3d6a0 is now active with session: 04916cfb-485b-40b9-a738-b94e6118c324","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:29.475801319Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1406,"session_id":"04916cfb-485b-40b9-a738-b94e6118c324"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:29.475824083Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1406,"session_id":"04916cfb-485b-40b9-a738-b94e6118c324"} {"msg":"The guest has finished waiting for activation with:56","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:29.475847416Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1406} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 198 len 24576 data: 7 7 6 5 4 4 02/30 Write block 130 len 16384 data: 5 4 5 4 03/30 Write block 396 len 8192 data: 7 7 04/30 Write block 370 len 12288 data: 4 4 7 05/30 Flush 06/30 Write block 399 len 12288 data: 6 7 7 07/30 Flush 08/30 Write block 157 len 40960 data: 4 4 5 5 5 5 4 5 6 7 09/30 Read block 425 len 8192 10/30 Read block 68 len 16384 11/30 Read block 449 len 36864 12/30 Write block 528 len 20480 data: 3 2 2 3 3 13/30 Write block 322 len 16384 data: 4 5 4 5 14/30 Write block 497 len 24576 data: 4 5 7 8 9 8 15/30 Read block 428 len 32768 16/30 Write block 415 len 28672 data: 5 4 4 3 3 2 3 17/30 Read block 226 len 20480 18/30 Write block 550 len 32768 data: 6 5 3 2 2 3 3 4 19/30 Read block 539 len 4096 20/30 Write block 523 len 20480 data: 10 7 6 6 5 21/30 Write block 265 len 28672 data: 5 5 5 4 5 6 5 22/30 Write block 463 len 32768 data: 6 7 9 7 10 8 8 7 23/30 Flush 24/30 Write block 187 len 32768 data: 3 4 4 6 6 7 9 10 25/30 Read block 453 len 12288 26/30 Read block 542 len 36864 27/30 Read block 524 len 32768 28/30 Read block 269 len 20480 29/30 Read block 594 len 8192 30/30 Write block 71 len 8192 data: 7 7 ---------------------------------------------------------------- Crucible gen:56 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 8 Done Done Done false 1024 Acked Read 3 Done Done Done false 1025 Acked Read 9 Done Done Done false 1026 Acked Read 8 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 6 6 6 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:13:29.495091034Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1406} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 53 53 53 93 93 93 F F F 1 020-039 53 53 53 93 93 93 F F F 2 040-059 53 53 53 93 93 93 F F F 3 060-079 52 52 52 92 92 92 T T T 4 080-099 55 55 55 96 96 96 F F F 5 100-119 55 55 55 96 96 96 F F F 6 120-139 56 56 56 97 97 97 F F F 7 140-159 56 56 56 99 99 99 F F F 8 160-179 56 56 56 99 99 99 F F F 9 180-199 56 56 56 97 97 97 T T T 10 200-219 56 56 56 97 97 97 F F F 11 220-239 54 54 54 95 95 95 F F F 12 240-259 54 54 54 95 95 95 F F F 13 260-279 56 56 56 99 99 99 F F F 14 280-299 54 54 54 94 94 94 F F F 15 300-319 54 54 54 95 95 95 F F F 16 320-339 56 56 56 99 99 99 F F F 17 340-359 53 53 53 93 93 93 F F F 18 360-379 56 56 56 97 97 97 F F F 19 380-399 56 56 56 98 98 98 F F F 20 400-419 56 56 56 99 99 99 F F F 21 420-439 56 56 56 99 99 99 F F F 22 440-459 55 55 55 96 96 96 F F F 23 460-479 56 56 56 99 99 99 F F F 24 480-499 56 56 56 99 99 99 F F F 25 500-519 56 56 56 99 99 99 F F F 26 520-539 56 56 56 99 99 99 F F F 27 540-559 56 56 56 99 99 99 F F F 28 560-579 54 54 54 94 94 94 F F F 29 580-599 48 48 48 84 84 84 F F F Max gen: 56, Max flush: 99 A Difference in extent metadata is expected here On loop 28 Verifying data now No change after verify Loop: 28 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 53 53 53 93 93 93 F F F 1 020-039 53 53 53 93 93 93 F F F 2 040-059 53 53 53 93 93 93 F F F 3 060-079 57 57 57 100 100 100 F F F 4 080-099 55 55 55 96 96 96 F F F 5 100-119 55 55 55 96 96 96 F F F 6 120-139 56 56 56 97 97 97 F F F 7 140-159 56 56 56 99 99 99 F F F 8 160-179 56 56 56 99 99 99 F F F 9 180-199 57 57 57 100 100 100 F F F 10 200-219 56 56 56 97 97 97 F F F 11 220-239 54 54 54 95 95 95 F F F 12 240-259 54 54 54 95 95 95 F F F 13 260-279 56 56 56 99 99 99 F F F 14 280-299 54 54 54 94 94 94 F F F 15 300-319 54 54 54 95 95 95 F F F 16 320-339 56 56 56 99 99 99 F F F 17 340-359 53 53 53 93 93 93 F F F 18 360-379 56 56 56 97 97 97 F F F 19 380-399 56 56 56 98 98 98 F F F 20 400-419 56 56 56 99 99 99 F F F 21 420-439 56 56 56 99 99 99 F F F 22 440-459 55 55 55 96 96 96 F F F 23 460-479 56 56 56 99 99 99 F F F 24 480-499 56 56 56 99 99 99 F F F 25 500-519 56 56 56 99 99 99 F F F 26 520-539 56 56 56 99 99 99 F F F 27 540-559 56 56 56 99 99 99 F F F 28 560-579 54 54 54 94 94 94 F F F 29 580-599 48 48 48 84 84 84 F F F Max gen: 57, Max flush: 100 Begin loop 29 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:33.721195431Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1414,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:13:33.764131546Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1414,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:33.764563828Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1414,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:33.79634036Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1414,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:33.796940591Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1414} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"abd5430f1508af0bc008143ea2566277ef714ec2\",\n git_commit_timestamp: \"2025-01-17T20:55:03.000000000Z\",\n git_branch: \"alan/tokio-to-140\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:33.796981305Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1414} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:33.796997375Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1414} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:33.797015525Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1414} {"msg":"Crucible da2500e1-4f6f-48ad-bb40-048cc42ad866 has session id: a538bff5-edaa-447b-85ee-ab2f7f13709a","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:33.797632606Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1414,"session_id":"a538bff5-edaa-447b-85ee-ab2f7f13709a"} {"msg":"Upstairs opts: Upstairs UUID: da2500e1-4f6f-48ad-bb40-048cc42ad866, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:33.797661052Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1414,"session_id":"a538bff5-edaa-447b-85ee-ab2f7f13709a"} {"msg":"Crucible stats registered with UUID: da2500e1-4f6f-48ad-bb40-048cc42ad866","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:33.797687938Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1414,"session_id":"a538bff5-edaa-447b-85ee-ab2f7f13709a"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:13:33.797711302Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1414,"session_id":"a538bff5-edaa-447b-85ee-ab2f7f13709a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:33.798121601Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1414,"task":"crutest"} {"msg":"The guest has requested activation with gen:58","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:33.798163086Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1414} {"msg":"da2500e1-4f6f-48ad-bb40-048cc42ad866 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:33.798223871Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1414,"session_id":"a538bff5-edaa-447b-85ee-ab2f7f13709a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:33.798250407Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1414,"client":"0","":"downstairs","session_id":"a538bff5-edaa-447b-85ee-ab2f7f13709a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:33.798274731Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1414,"client":"0","":"downstairs","session_id":"a538bff5-edaa-447b-85ee-ab2f7f13709a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:33.798288959Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1414,"client":"1","":"downstairs","session_id":"a538bff5-edaa-447b-85ee-ab2f7f13709a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:33.798306429Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1414,"client":"1","":"downstairs","session_id":"a538bff5-edaa-447b-85ee-ab2f7f13709a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:33.798330864Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1414,"client":"2","":"downstairs","session_id":"a538bff5-edaa-447b-85ee-ab2f7f13709a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:33.798357359Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1414,"client":"2","":"downstairs","session_id":"a538bff5-edaa-447b-85ee-ab2f7f13709a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:33.798381673Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1414,"":"io task","client":"2","":"downstairs","session_id":"a538bff5-edaa-447b-85ee-ab2f7f13709a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:33.798406988Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1414,"":"io task","client":"0","":"downstairs","session_id":"a538bff5-edaa-447b-85ee-ab2f7f13709a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:33.798433854Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1414,"":"io task","client":"1","":"downstairs","session_id":"a538bff5-edaa-447b-85ee-ab2f7f13709a"} {"msg":"ds_connection connected from Ok(127.0.0.1:57566)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:33.798460109Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1414,"":"io task","client":"2","":"downstairs","session_id":"a538bff5-edaa-447b-85ee-ab2f7f13709a"} {"msg":"ds_connection connected from Ok(127.0.0.1:49852)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:33.798484464Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1414,"":"io task","client":"0","":"downstairs","session_id":"a538bff5-edaa-447b-85ee-ab2f7f13709a"} {"msg":"ds_connection connected from Ok(127.0.0.1:43304)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:33.798513801Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1414,"":"io task","client":"1","":"downstairs","session_id":"a538bff5-edaa-447b-85ee-ab2f7f13709a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:33.799244538Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1414,"client":"2","":"downstairs","session_id":"a538bff5-edaa-447b-85ee-ab2f7f13709a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:33.799416048Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1414,"client":"1","":"downstairs","session_id":"a538bff5-edaa-447b-85ee-ab2f7f13709a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:33.79943653Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1414,"client":"0","":"downstairs","session_id":"a538bff5-edaa-447b-85ee-ab2f7f13709a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:33.799935732Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1414,"client":"2","":"downstairs","session_id":"a538bff5-edaa-447b-85ee-ab2f7f13709a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:33.799975115Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1414,"client":"1","":"downstairs","session_id":"a538bff5-edaa-447b-85ee-ab2f7f13709a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:33.800110224Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1414,"client":"0","":"downstairs","session_id":"a538bff5-edaa-447b-85ee-ab2f7f13709a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:33.800554223Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1414,"session_id":"a538bff5-edaa-447b-85ee-ab2f7f13709a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:33.800576066Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1414,"session_id":"a538bff5-edaa-447b-85ee-ab2f7f13709a"} {"msg":"[0]R flush_numbers[0..12]: [93, 93, 93, 100, 96, 96, 97, 99, 99, 100, 97, 95]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:33.80065181Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1414,"":"downstairs","":"downstairs","session_id":"a538bff5-edaa-447b-85ee-ab2f7f13709a"} {"msg":"[0]R generation[0..12]: [53, 53, 53, 57, 55, 55, 56, 56, 56, 57, 56, 54]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:33.800670321Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1414,"":"downstairs","":"downstairs","session_id":"a538bff5-edaa-447b-85ee-ab2f7f13709a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:33.800684399Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1414,"":"downstairs","":"downstairs","session_id":"a538bff5-edaa-447b-85ee-ab2f7f13709a"} {"msg":"[1]R flush_numbers[0..12]: [93, 93, 93, 100, 96, 96, 97, 99, 99, 100, 97, 95]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:33.800697847Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1414,"":"downstairs","":"downstairs","session_id":"a538bff5-edaa-447b-85ee-ab2f7f13709a"} {"msg":"[1]R generation[0..12]: [53, 53, 53, 57, 55, 55, 56, 56, 56, 57, 56, 54]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:33.800711195Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1414,"":"downstairs","":"downstairs","session_id":"a538bff5-edaa-447b-85ee-ab2f7f13709a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:33.800727875Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1414,"":"downstairs","":"downstairs","session_id":"a538bff5-edaa-447b-85ee-ab2f7f13709a"} {"msg":"[2]R flush_numbers[0..12]: [93, 93, 93, 100, 96, 96, 97, 99, 99, 100, 97, 95]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:33.800741373Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1414,"":"downstairs","":"downstairs","session_id":"a538bff5-edaa-447b-85ee-ab2f7f13709a"} {"msg":"[2]R generation[0..12]: [53, 53, 53, 57, 55, 55, 56, 56, 56, 57, 56, 54]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:33.80075458Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1414,"":"downstairs","":"downstairs","session_id":"a538bff5-edaa-447b-85ee-ab2f7f13709a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:33.800768028Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1414,"":"downstairs","":"downstairs","session_id":"a538bff5-edaa-447b-85ee-ab2f7f13709a"} {"msg":"Max found gen is 58","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:33.800780526Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1414,"":"downstairs","":"downstairs","session_id":"a538bff5-edaa-447b-85ee-ab2f7f13709a"} {"msg":"Generation requested: 58 >= found:58","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:33.800793163Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1414,"":"downstairs","":"downstairs","session_id":"a538bff5-edaa-447b-85ee-ab2f7f13709a"} {"msg":"Next flush: 101","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:33.80080561Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1414,"":"downstairs","":"downstairs","session_id":"a538bff5-edaa-447b-85ee-ab2f7f13709a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:33.800817977Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1414,Wait for a query_work_queue command to finish before sending IO "":"downstairs","":"downstairs","session_id":"a538bff5-edaa-447b-85ee-ab2f7f13709a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:33.800837359Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1414,"session_id":"a538bff5-edaa-447b-85ee-ab2f7f13709a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:33.800848996Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1414,"client":"0","":"downstairs","session_id":"a538bff5-edaa-447b-85ee-ab2f7f13709a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:33.800862113Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1414,"client":"1","":"downstairs","session_id":"a538bff5-edaa-447b-85ee-ab2f7f13709a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:33.800875081Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1414,"client":"2","":"downstairs","session_id":"a538bff5-edaa-447b-85ee-ab2f7f13709a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:33.800888089Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1414,"session_id":"a538bff5-edaa-447b-85ee-ab2f7f13709a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:33.800899975Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1414,"session_id":"a538bff5-edaa-447b-85ee-ab2f7f13709a"} {"msg":"da2500e1-4f6f-48ad-bb40-048cc42ad866 is now active with session: a538bff5-edaa-447b-85ee-ab2f7f13709a","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:33.800912263Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1414,"session_id":"a538bff5-edaa-447b-85ee-ab2f7f13709a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:33.800926181Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1414,"session_id":"a538bff5-edaa-447b-85ee-ab2f7f13709a"} {"msg":"The guest has finished waiting for activation with:58","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:33.800938118Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1414} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 175 len 24576 data: 7 5 5 4 5 6 02/30 Write block 208 len 32768 data: 4 5 3 3 3 3 3 3 03/30 Write block 369 len 24576 data: 3 5 5 8 7 8 04/30 Write block 326 len 32768 data: 6 5 4 4 5 5 5 4 05/30 Write block 476 len 20480 data: 4 4 3 3 4 06/30 Flush 07/30 Write block 521 len 32768 data: 9 7 11 8 7 7 6 4 08/30 Read block 196 len 28672 09/30 Write block 459 len 20480 data: 6 6 7 7 7 10/30 Write block 495 len 28672 data: 5 6 5 6 8 9 10 11/30 Flush 12/30 Write block 31 len 36864 data: 5 8 8 7 8 5 6 6 6 13/30 Read block 566 len 36864 14/30 Read block 538 len 28672 15/30 Read block 439 len 32768 16/30 Write block 428 len 12288 data: 8 8 8 17/30 Read block 248 len 40960 18/30 Read block 458 len 24576 19/30 Read block 83 len 12288 20/30 Write block 530 len 16384 data: 3 4 4 3 21/30 Read block 38 len 24576 22/30 Read block 69 len 16384 23/30 Read block 292 len 24576 24/30 Write block 115 len 20480 data: 8 5 5 5 5 25/30 Write block 155 len 12288 data: 4 4 5 26/30 Flush 27/30 Write block 420 len 40960 data: 3 4 2 2 2 4 6 7 9 9 28/30 Write block 10 len 20480 data: 7 9 9 9 8 29/30 Read block 536 len 24576 30/30 Read block 32 len 32768 ---------------------------------------------------------------- Crucible gen:58 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 10 Done Done Done false 1027 Acked Write 5 Done Done Done false 1028 Acked Read 6 Done Done Done false 1029 Acked Read 8 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:13:33.821582534Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1414} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 53 53 53 93 93 93 T T T 1 020-039 58 58 58 103 103 103 F F F 2 040-059 53 53 53 93 93 93 F F F 3 060-079 57 57 57 100 100 100 F F F 4 080-099 55 55 55 96 96 96 F F F 5 100-119 58 58 58 103 103 103 F F F 6 120-139 56 56 56 97 97 97 F F F 7 140-159 58 58 58 103 103 103 F F F 8 160-179 58 58 58 101 101 101 F F F 9 180-199 58 58 58 101 101 101 F F F 10 200-219 58 58 58 101 101 101 F F F 11 220-239 54 54 54 95 95 95 F F F 12 240-259 54 54 54 95 95 95 F F F 13 260-279 56 56 56 99 99 99 F F F 14 280-299 54 54 54 94 94 94 F F F 15 300-319 54 54 54 95 95 95 F F F 16 320-339 58 58 58 101 101 101 F F F 17 340-359 53 53 53 93 93 93 F F F 18 360-379 58 58 58 101 101 101 F F F 19 380-399 56 56 56 98 98 98 F F F 20 400-419 56 56 56 99 99 99 F F F 21 420-439 58 58 58 103 103 103 T T T 22 440-459 58 58 58 102 102 102 F F F 23 460-479 58 58 58 102 102 102 F F F 24 480-499 58 58 58 102 102 102 F F F 25 500-519 58 58 58 102 102 102 F F F 26 520-539 58 58 58 103 103 103 F F F 27 540-559 56 56 56 99 99 99 F F F 28 560-579 54 54 54 94 94 94 F F F 29 580-599 48 48 48 84 84 84 F F F Max gen: 58, Max flush: 103 A Difference in extent metadata is expected here On loop 29 Verifying data now No change after verify Loop: 29 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 59 59 59 104 104 104 F F F 1 020-039 58 58 58 103 103 103 F F F 2 040-059 53 53 53 93 93 93 F F F 3 060-079 57 57 57 100 100 100 F F F 4 080-099 55 55 55 96 96 96 F F F 5 100-119 58 58 58 103 103 103 F F F 6 120-139 56 56 56 97 97 97 F F F 7 140-159 58 58 58 103 103 103 F F F 8 160-179 58 58 58 101 101 101 F F F 9 180-199 58 58 58 101 101 101 F F F 10 200-219 58 58 58 101 101 101 F F F 11 220-239 54 54 54 95 95 95 F F F 12 240-259 54 54 54 95 95 95 F F F 13 260-279 56 56 56 99 99 99 F F F 14 280-299 54 54 54 94 94 94 F F F 15 300-319 54 54 54 95 95 95 F F F 16 320-339 58 58 58 101 101 101 F F F 17 340-359 53 53 53 93 93 93 F F F 18 360-379 58 58 58 101 101 101 F F F 19 380-399 56 56 56 98 98 98 F F F 20 400-419 56 56 56 99 99 99 F F F 21 420-439 59 59 59 104 104 104 F F F 22 440-459 58 58 58 102 102 102 F F F 23 460-479 58 58 58 102 102 102 F F F 24 480-499 58 58 58 102 102 102 F F F 25 500-519 58 58 58 102 102 102 F F F 26 520-539 58 58 58 103 103 103 F F F 27 540-559 56 56 56 99 99 99 F F F 28 560-579 54 54 54 94 94 94 F F F 29 580-599 48 48 48 84 84 84 F F F Max gen: 59, Max flush: 104 Begin loop 30 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:38.077975283Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1422,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:38.122313478Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1422,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:38.122974864Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1422} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"abd5430f1508af0bc008143ea2566277ef714ec2\",\n git_commit_timestamp: \"2025-01-17T20:55:03.000000000Z\",\n git_branch: \"alan/tokio-to-140\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:38.123022692Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1422} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:38.123040423Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1422} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:38.123062296Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1422} {"msg":"Crucible e97280e9-848c-43bb-8e58-b70086eb3177 has session id: e7692c38-fb05-4d23-b9d7-ad4c7210be6d","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:38.123608585Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1422,"session_id":"e7692c38-fb05-4d23-b9d7-ad4c7210be6d"} {"msg":"Upstairs opts: Upstairs UUID: e97280e9-848c-43bb-8e58-b70086eb3177, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:38.123655752Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1422,"session_id":"e7692c38-fb05-4d23-b9d7-ad4c7210be6d"} {"msg":"Crucible stats registered with UUID: e97280e9-848c-43bb-8e58-b70086eb3177","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:38.123683699Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1422,"session_id":"e7692c38-fb05-4d23-b9d7-ad4c7210be6d"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:13:38.123708703Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1422,"session_id":"e7692c38-fb05-4d23-b9d7-ad4c7210be6d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:38.124197909Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1422,"task":"crutest"} {"msg":"The guest has requested activation with gen:60","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:38.124232449Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1422} {"msg":"e97280e9-848c-43bb-8e58-b70086eb3177 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:38.124306143Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1422,"session_id":"e7692c38-fb05-4d23-b9d7-ad4c7210be6d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:38.124338882Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1422,"client":"0","":"downstairs","session_id":"e7692c38-fb05-4d23-b9d7-ad4c7210be6d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:38.124373792Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1422,"client":"0","":"downstairs","session_id":"e7692c38-fb05-4d23-b9d7-ad4c7210be6d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:38.124396075Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1422,"client":"1","":"downstairs","session_id":"e7692c38-fb05-4d23-b9d7-ad4c7210be6d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:38.124418788Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1422,"client":"1","":"downstairs","session_id":"e7692c38-fb05-4d23-b9d7-ad4c7210be6d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:38.124441522Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1422,"client":"2","":"downstairs","session_id":"e7692c38-fb05-4d23-b9d7-ad4c7210be6d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:38.124464885Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1422,"client":"2","":"downstairs","session_id":"e7692c38-fb05-4d23-b9d7-ad4c7210be6d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:38.12449013Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1422,"":"io task","client":"2","":"downstairs","session_id":"e7692c38-fb05-4d23-b9d7-ad4c7210be6d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:38.124512513Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1422,"":"io task","client":"0","":"downstairs","session_id":"e7692c38-fb05-4d23-b9d7-ad4c7210be6d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:38.124534356Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1422,"":"io task","client":"1","":"downstairs","session_id":"e7692c38-fb05-4d23-b9d7-ad4c7210be6d"} {"msg":"ds_connection connected from Ok(127.0.0.1:45445)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:38.124556138Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1422,"":"io task","client":"2","":"downstairs","session_id":"e7692c38-fb05-4d23-b9d7-ad4c7210be6d"} {"msg":"ds_connection connected from Ok(127.0.0.1:51162)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:38.124577831Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1422,"":"io task","client":"0","":"downstairs","session_id":"e7692c38-fb05-4d23-b9d7-ad4c7210be6d"} {"msg":"ds_connection connected from Ok(127.0.0.1:58745)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:38.124691638Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1422,"":"io task","client":"1","":"downstairs","session_id":"e7692c38-fb05-4d23-b9d7-ad4c7210be6d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:38.125390946Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1422,"client":"0","":"downstairs","session_id":"e7692c38-fb05-4d23-b9d7-ad4c7210be6d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:38.125427838Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1422,"client":"1","":"downstairs","session_id":"e7692c38-fb05-4d23-b9d7-ad4c7210be6d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:38.125798055Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1422,"client":"2","":"downstairs","session_id":"e7692c38-fb05-4d23-b9d7-ad4c7210be6d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:38.126091946Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1422,"client":"1","":"downstairs","session_id":"e7692c38-fb05-4d23-b9d7-ad4c7210be6d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:38.126118011Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1422,"client":"0","":"downstairs","session_id":"e7692c38-fb05-4d23-b9d7-ad4c7210be6d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:38.12640664Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1422,"client":"2","":"downstairs","session_id":"e7692c38-fb05-4d23-b9d7-ad4c7210be6d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:38.126720684Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1422,"session_id":"e7692c38-fb05-4d23-b9d7-ad4c7210be6d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:38.126743877Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1422,"session_id":"e7692c38-fb05-4d23-b9d7-ad4c7210be6d"} {"msg":"[0]R flush_numbers[0..12]: [104, 103, 93, 100, 96, 103, 97, 103, 101, 101, 101, 95]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:38.126946275Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1422,"":"downstairs","":"downstairs","session_id":"e7692c38-fb05-4d23-b9d7-ad4c7210be6d"} {"msg":"[0]R generation[0..12]: [59, 58, 53, 57, 55, 58, 56, 58, 58, 58, 58, 54]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:38.126985509Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1422,"":"downstairs","":"downstairs","session_id":"e7692c38-fb05-4d23-b9d7-ad4c7210be6d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:38.127029044Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1422,"":"downstairs","":"downstairs","session_id":"e7692c38-fb05-4d23-b9d7-ad4c7210be6d"} {"msg":"[1]R flush_numbers[0..12]: [104, 103, 93, 100, 96, 103, 97, 103, 101, 101, 101, 95]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:38.127055539Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1422,"":"downstairs","":"downstairs","session_id":"e7692c38-fb05-4d23-b9d7-ad4c7210be6d"} {"msg":"[1]R generation[0..12]: [59, 58, 53, 57, 55, 58, 56, 58, 58, 58, 58, 54]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:38.127081365Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1422,"":"downstairs","":"downstairs","session_id":"e7692c38-fb05-4d23-b9d7-ad4c7210be6d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:38.127105969Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1422,"":"downstairs","":"downstairs","session_id":"e7692c38-fb05-4d23-b9d7-ad4c7210be6d"} {"msg":"[2]R flush_numbers[0..12]: [104, 103, 93, 100, 96, 103, 97, 103, 101, 101, 101, 95]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:38.127130383Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1422,"":"downstairs","":"downstairs","session_id":"e7692c38-fb05-4d23-b9d7-ad4c7210be6d"} {"msg":"[2]R generation[0..12]: [59, 58, 53, 57, 55, 58, 56, 58, 58, 58, 58, 54]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:38.127157519Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1422,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO e7692c38-fb05-4d23-b9d7-ad4c7210be6d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:38.12719292Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1422,"":"downstairs","":"downstairs","session_id":"e7692c38-fb05-4d23-b9d7-ad4c7210be6d"} {"msg":"Max found gen is 60","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:38.127220256Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1422,"":"downstairs","":"downstairs","session_id":"e7692c38-fb05-4d23-b9d7-ad4c7210be6d"} {"msg":"Generation requested: 60 >= found:60","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:38.127246471Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1422,"":"downstairs","":"downstairs","session_id":"e7692c38-fb05-4d23-b9d7-ad4c7210be6d"} {"msg":"Next flush: 105","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:38.127272066Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1422,"":"downstairs","":"downstairs","session_id":"e7692c38-fb05-4d23-b9d7-ad4c7210be6d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:38.127297271Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1422,"":"downstairs","":"downstairs","session_id":"e7692c38-fb05-4d23-b9d7-ad4c7210be6d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:38.127323556Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1422,"session_id":"e7692c38-fb05-4d23-b9d7-ad4c7210be6d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:38.1273475Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1422,"client":"0","":"downstairs","session_id":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 e7692c38-fb05-4d23-b9d7-ad4c7210be6d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:38.127379359Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1422,"client":"1","":"downstairs","session_id":"e7692c38-fb05-4d23-b9d7-ad4c7210be6d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:38.127405184Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1422,"client":"2","":"downstairs","session_id":"e7692c38-fb05-4d23-b9d7-ad4c7210be6d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:38.127421713Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1422,"session_id":"e7692c38-fb05-4d23-b9d7-ad4c7210be6d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:38.127434001Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1422,"session_id":"e7692c38-fb05-4d23-b9d7-ad4c7210be6d"} {"msg":"e97280e9-848c-43bb-8e58-b70086eb3177 is now active with session: e7692c38-fb05-4d23-b9d7-ad4c7210be6d","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:38.127446518Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1422,"session_id":"e7692c38-fb05-4d23-b9d7-ad4c7210be6d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:38.127458105Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1422,"session_id":"e7692c38-fb05-4d23-b9d7-ad4c7210be6d"} {"msg":"The guest has finished waiting for activation with:60","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:38.127469861Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1422} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 90 len 28672 data: 4 4 5 5 6 6 6 02/30 Read block 100 len 20480 03/30 Write block 597 len 4096 data: 3 04/30 Read block 97 len 36864 05/30 Read block 451 len 36864 06/30 Read block 578 len 28672 07/30 Write block 370 len 16384 data: 6 6 9 8 08/30 Read block 18 len 32768 09/30 Read block 85 len 12288 10/30 Flush 11/30 Write block 558 len 4096 data: 5 12/30 Read block 516 len 28672 13/30 Write block 255 len 20480 data: 3 5 5 6 6 14/30 Read block 216 len 20480 15/30 Write block 153 len 12288 data: 4 5 5 16/30 Flush 17/30 Write block 58 len 8192 data: 9 7 18/30 Read block 218 len 28672 19/30 Flush 20/30 Write block 202 len 4096 data: 5 21/30 Flush 22/30 Write block 397 len 24576 data: 8 8 7 8 8 6 23/30 Read block 385 len 40960 24/30 Write block 58 len 28672 data: 10 8 6 6 4 3 3 25/30 Write block 556 len 28672 data: 4 5 6 5 6 7 7 26/30 Flush 27/30 Write block 459 len 32768 data: 7 7 8 8 8 8 10 8 28/30 Read block 366 len 8192 29/30 Read block 127 len 40960 30/30 Write block 170 len 32768 data: 6 6 6 6 6 8 6 6 ---------------------------------------------------------------- Crucible gen:60 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 8 Done Done Done false 1027 Acked Read 2 Done Done Done false 1028 Acked Read 10 Done Done Done false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:13:38.146588786Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1422} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 59 59 59 104 104 104 F F F 1 020-039 58 58 58 103 103 103 F F F 2 040-059 60 60 60 109 109 109 F F F 3 060-079 60 60 60 109 109 109 F F F 4 080-099 60 60 60 105 105 105 F F F 5 100-119 58 58 58 103 103 103 F F F 6 120-139 56 56 56 97 97 97 F F F 7 140-159 60 60 60 106 106 106 F F F 8 160-179 58 58 58 101 101 101 T T T 9 180-199 58 58 58 101 101 101 F F F 10 200-219 60 60 60 108 108 108 F F F 11 220-239 54 54 54 95 95 95 F F F 12 240-259 60 60 60 106 106 106 F F F 13 260-279 56 56 56 99 99 99 F F F 14 280-299 54 54 54 94 94 94 F F F 15 300-319 54 54 54 95 95 95 F F F 16 320-339 58 58 58 101 101 101 F F F 17 340-359 53 53 53 93 93 93 F F F 18 360-379 60 60 60 105 105 105 F F F 19 380-399 60 60 60 109 109 109 F F F 20 400-419 60 60 60 109 109 109 F F F 21 420-439 59 59 59 104 104 104 F F F 22 440-459 58 58 58 102 102 102 T T T 23 460-479 58 58 58 102 102 102 T T T 24 480-499 58 58 58 102 102 102 F F F 25 500-519 58 58 58 102 102 102 F F F 26 520-539 58 58 58 103 103 103 F F F 27 540-559 60 60 60 109 109 109 F F F 28 560-579 60 60 60 109 109 109 F F F 29 580-599 60 60 60 105 105 105 F F F Max gen: 60, Max flush: 109 A Difference in extent metadata is expected here On loop 30 Verifying data now No change after verify Loop: 30 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 59 59 59 104 104 104 F F F 1 020-039 58 58 58 103 103 103 F F F 2 040-059 60 60 60 109 109 109 F F F 3 060-079 60 60 60 109 109 109 F F F 4 080-099 60 60 60 105 105 105 F F F 5 100-119 58 58 58 103 103 103 F F F 6 120-139 56 56 56 97 97 97 F F F 7 140-159 60 60 60 106 106 106 F F F 8 160-179 61 61 61 110 110 110 F F F 9 180-199 58 58 58 101 101 101 F F F 10 200-219 60 60 60 108 108 108 F F F 11 220-239 54 54 54 95 95 95 F F F 12 240-259 60 60 60 106 106 106 F F F 13 260-279 56 56 56 99 99 99 F F F 14 280-299 54 54 54 94 94 94 F F F 15 300-319 54 54 54 95 95 95 F F F 16 320-339 58 58 58 101 101 101 F F F 17 340-359 53 53 53 93 93 93 F F F 18 360-379 60 60 60 105 105 105 F F F 19 380-399 60 60 60 109 109 109 F F F 20 400-419 60 60 60 109 109 109 F F F 21 420-439 59 59 59 104 104 104 F F F 22 440-459 61 61 61 110 110 110 F F F 23 460-479 61 61 61 110 110 110 F F F 24 480-499 58 58 58 102 102 102 F F F 25 500-519 58 58 58 102 102 102 F F F 26 520-539 58 58 58 103 103 103 F F F 27 540-559 60 60 60 109 109 109 F F F 28 560-579 60 60 60 109 109 109 F F F 29 580-599 60 60 60 105 105 105 F F F Max gen: 61, Max flush: 110 Begin loop 31 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:42.373802971Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1430,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:42.417962411Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1430,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:42.418680521Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1430} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"abd5430f1508af0bc008143ea2566277ef714ec2\",\n git_commit_timestamp: \"2025-01-17T20:55:03.000000000Z\",\n git_branch: \"alan/tokio-to-140\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:42.418724737Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1430} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:42.418746089Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1430} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:42.418761398Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1430} {"msg":"Crucible 30f89c40-a3b4-484d-89b6-21ca88d6ba8f has session id: bdadb29b-4645-4d87-b2d7-5b843a92447f","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:42.419379169Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1430,"session_id":"bdadb29b-4645-4d87-b2d7-5b843a92447f"} {"msg":"Upstairs opts: Upstairs UUID: 30f89c40-a3b4-484d-89b6-21ca88d6ba8f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:42.419423555Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1430,"session_id":"bdadb29b-4645-4d87-b2d7-5b843a92447f"} {"msg":"Crucible stats registered with UUID: 30f89c40-a3b4-484d-89b6-21ca88d6ba8f","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:42.419446969Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1430,"session_id":"bdadb29b-4645-4d87-b2d7-5b843a92447f"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:13:42.419468382Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1430,"session_id":"bdadb29b-4645-4d87-b2d7-5b843a92447f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:42.419888127Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1430,"task":"crutest"} {"msg":"The guest has requested activation with gen:62","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:42.419920596Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1430} {"msg":"30f89c40-a3b4-484d-89b6-21ca88d6ba8f active request set","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:42.419983563Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1430,"session_id":"bdadb29b-4645-4d87-b2d7-5b843a92447f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:42.420023166Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1430,"client":"0","":"downstairs","session_id":"bdadb29b-4645-4d87-b2d7-5b843a92447f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:42.4200473Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1430,"client":"0","":"downstairs","session_id":"bdadb29b-4645-4d87-b2d7-5b843a92447f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:42.420061518Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1430,"client":"1","":"downstairs","session_id":"bdadb29b-4645-4d87-b2d7-5b843a92447f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:42.42008233Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1430,"client":"1","":"downstairs","session_id":"bdadb29b-4645-4d87-b2d7-5b843a92447f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:42.420105454Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1430,"client":"2","":"downstairs","session_id":"bdadb29b-4645-4d87-b2d7-5b843a92447f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:42.420131709Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1430,"client":"2","":"downstairs","session_id":"bdadb29b-4645-4d87-b2d7-5b843a92447f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:42.420155563Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1430,"":"io task","client":"2","":"downstairs","session_id":"bdadb29b-4645-4d87-b2d7-5b843a92447f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:42.420182309Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1430,"":"io task","client":"0","":"downstairs","session_id":"bdadb29b-4645-4d87-b2d7-5b843a92447f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:42.420202681Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1430,"":"io task","client":"1","":"downstairs","session_id":"bdadb29b-4645-4d87-b2d7-5b843a92447f"} {"msg":"ds_connection connected from Ok(127.0.0.1:54441)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:42.420225584Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1430,"":"io task","client":"2","":"downstairs","session_id":"bdadb29b-4645-4d87-b2d7-5b843a92447f"} {"msg":"ds_connection connected from Ok(127.0.0.1:33949)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:42.420249058Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1430,"":"io task","client":"0","":"downstairs","session_id":"bdadb29b-4645-4d87-b2d7-5b843a92447f"} {"msg":"ds_connection connected from Ok(127.0.0.1:55272)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:42.420275043Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1430,"":"io task","client":"1","":"downstairs","session_id":"bdadb29b-4645-4d87-b2d7-5b843a92447f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:42.42097104Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1430,"client":"2","":"downstairs","session_id":"bdadb29b-4645-4d87-b2d7-5b843a92447f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:42.421012414Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1430,"client":"0","":"downstairs","session_id":"bdadb29b-4645-4d87-b2d7-5b843a92447f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:42.421572581Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1430,"client":"1","":"downstairs","session_id":"bdadb29b-4645-4d87-b2d7-5b843a92447f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:42.421611904Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1430,"client":"0","":"downstairs","session_id":"bdadb29b-4645-4d87-b2d7-5b843a92447f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:42.421641962Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1430,"client":"2","":"downstairs","session_id":"bdadb29b-4645-4d87-b2d7-5b843a92447f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:42.422179086Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1430,"session_id":"bdadb29b-4645-4d87-b2d7-5b843a92447f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:42.42220439Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1430,"session_id":"bdadb29b-4645-4d87-b2d7-5b843a92447f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:42.422218509Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1430,"client":"1","":"downstairs","session_id":"bdadb29b-4645-4d87-b2d7-5b843a92447f"} {"msg":"[0]R flush_numbers[0..12]: [104, 103, 109, 109, 105, 103, 97, 106, 110, 101, 108, 95]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:42.422710516Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1430,"":"downstairs","":"downstairs","session_id":"bdadb29b-4645-4d87-b2d7-5b843a92447f"} {"msg":"[0]R generation[0..12]: [59, 58, 60, 60, 60, 58, 56, 60, 61, 58, 60, 54]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:42.422730098Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1430,"":"downstairs","":"downstairs","session_id":"bdadb29b-4645-4d87-b2d7-5b843a92447f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:42.422744276Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1430,"":"downstairs","":"downstairs","session_id":"bdadb29b-4645-4d87-b2d7-5b843a92447f"} {"msg":"[1]R flush_numbers[0..12]: [104, 103, 109, 109, 105, 103, 97, 106, 110, 101, 108, 95]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:42.422758204Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1430,"":"downstairs","":"downstairs","session_id":"bdadb29b-4645-4d87-b2d7-5b843a92447f"} {"msg":"[1]R generation[0..12]: [59, 58, 60, 60, 60, 58, 56, 60, 61, 58, 60, 54]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:42.422771672Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1430,"":"downstairs","":"downstairs","session_id":"bdadb29b-4645-4d87-b2d7-5b843a92447f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:42.4227854Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1430,"":"downstairs","":"downstairs","session_id":"bdadb29b-4645-4d87-b2d7-5b843a92447f"} {"msg":"[2]R flush_numbers[0..12]: [104, 103, 109, 109, 105, 103, 97, 106, 110, 101, 108, 95]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:42.422798808Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1430,"":"downstairs","":"downstairs","session_id":"bdadb29b-4645-4d87-b2d7-5b843a92447f"} {"msg":"[2]R generation[0..12]: [59, 58, 60, 60, 60, 58, 56, 60, 61, 58, 60, 54]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:42.422811966Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1430,"":"downstairs","":"downstairs","session_id":"bdadb29b-4645-4d87-b2d7-5b843a92447f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:42.422825264Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1430,"":"downstairs","":"downstairs","session_id":"bdadb29b-4645-4d87-b2d7-5b843a92447f"} {"msg":"Max found gen is 62","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:42.422837751Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1430,"":"downstairs","":"downstairs","session_id":"bdadb29b-4645-4d87-b2d7-5b843a92447f"} {"msg":"Generation requested: 62 >= found:62","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:42.422850308Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1430,"":"downstairs","":"downstairs","session_id":"bdadb29b-4645-4d87-b2d7-5b843a92447f"} {"msg":"Next flush: 111","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:42.422866728Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1430,"":"downstairs","":"downstairs","session_id":"bdadb29b-4645-4d87-b2d7-5b843a92447f"} {Wait for a query_work_queue command to finish before sending IO "msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:42.422886449Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1430,"":"downstairs","":"downstairs","session_id":"bdadb29b-4645-4d87-b2d7-5b843a92447f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:42.422899307Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1430,"session_id":"bdadb29b-4645-4d87-b2d7-5b843a92447f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:42.422910854Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1430,"client":"0","":"downstairs","session_id":"bdadb29b-4645-4d87-b2d7-5b843a92447f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:42.422923791Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1430,"client":"1","":"downstairs","session_id":"bdadb29b-4645-4d87-b2d7-5b843a92447f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:42.422936569Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1430,"client":"2","":"downstairs","session_id":"bdadb29b-4645-4d87-b2d7-5b843a92447f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:42.422949226Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1430,"session_id":"bdadb29b-4645-4d87-b2d7-5b843a92447f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:42.422960953Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1430,"session_id":"bdadb29b-4645-4d87-b2d7-5b843a92447f"} {"msg":"30f89c40-a3b4-484d-89b6-21ca88d6ba8f is now active with session: bdadb29b-4645-4d87-b2d7-5b843a92447f","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:42.42297319Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1430,"session_id":"bdadb29b-4645-4d87-b2d7-5b843a92447f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:42.422984787Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1430,"session_id":"bdadb29b-4645-4d87-b2d7-5b843a92447f"} {"msg":"The guest has finished waiting for activation with:62","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:42.422996674Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1430} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 422 len 12288 data: 3 3 3 02/30 Read block 414 len 12288 03/30 Read block 413 len 24576 04/30 Read block 341 len 36864 05/30 Flush 06/30 Write block 334 len 16384 data: 7 8 9 8 07/30 Flush 08/30 Write block 338 len 24576 data: 8 8 8 6 6 10 09/30 Flush 10/30 Write block 528 len 8192 data: 5 3 11/30 Read block 62 len 12288 12/30 Write block 507 len 4096 data: 8 13/30 Read block 18 len 32768 14/30 Read block 417 len 8192 15/30 Read block 152 len 40960 16/30 Read block 564 len 20480 17/30 Read block 455 len 4096 18/30 Read block 218 len 28672 19/30 Flush 20/30 Flush 21/30 Write block 589 len 24576 data: 5 5 3 4 4 3 22/30 Write block 386 len 16384 data: 6 6 7 9 23/30 Flush 24/30 Write block 595 len 16384 data: 3 3 4 3 25/30 Write block 575 len 4096 data: 4 26/30 Write block 251 len 12288 data: 4 4 2 27/30 Write block 106 len 4096 data: 4 28/30 Write block 415 len 40960 data: 6 5 5 4 4 4 5 4 4 4 29/30 Read block 259 len 16384 30/30 Read block 314 len 28672 ---------------------------------------------------------------- Crucible gen:62 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 4 Done Done Done false 1024 Acked Write 1 Done Done Done false 1025 Acked Write 3 Done Done Done false 1026 Acked Write 1 Done Done Done false 1027 Acked Write 10 Done Done Done false 1028 Acked Read 4 Done Done Done false 1029 Acked Read 7 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 7 7 7 21 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:13:42.44163986Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1430} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 59 59 59 104 104 104 F F F 1 020-039 58 58 58 103 103 103 F F F 2 040-059 60 60 60 109 109 109 F F F 3 060-079 60 60 60 109 109 109 F F F 4 080-099 60 60 60 105 105 105 F F F 5 100-119 58 58 58 103 103 103 T T T 6 120-139 56 56 56 97 97 97 F F F 7 140-159 60 60 60 106 106 106 F F F 8 160-179 61 61 61 110 110 110 F F F 9 180-199 58 58 58 101 101 101 F F F 10 200-219 60 60 60 108 108 108 F F F 11 220-239 54 54 54 95 95 95 F F F 12 240-259 60 60 60 106 106 106 T T T 13 260-279 56 56 56 99 99 99 F F F 14 280-299 54 54 54 94 94 94 F F F 15 300-319 54 54 54 95 95 95 F F F 16 320-339 62 62 62 113 113 113 F F F 17 340-359 62 62 62 113 113 113 F F F 18 360-379 60 60 60 105 105 105 F F F 19 380-399 62 62 62 116 116 116 F F F 20 400-419 60 60 60 109 109 109 T T T 21 420-439 62 62 62 111 111 111 T T T 22 440-459 61 61 61 110 110 110 F F F 23 460-479 61 61 61 110 110 110 F F F 24 480-499 58 58 58 102 102 102 F F F 25 500-519 62 62 62 114 114 114 F F F 26 520-539 62 62 62 114 114 114 F F F 27 540-559 60 60 60 109 109 109 F F F 28 560-579 60 60 60 109 109 109 T T T 29 580-599 62 62 62 116 116 116 T T T Max gen: 62, Max flush: 116 A Difference in extent metadata is expected here On loop 31 Verifying data now No change after verify Loop: 31 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 59 59 59 104 104 104 F F F 1 020-039 58 58 58 103 103 103 F F F 2 040-059 60 60 60 109 109 109 F F F 3 060-079 60 60 60 109 109 109 F F F 4 080-099 60 60 60 105 105 105 F F F 5 100-119 63 63 63 117 117 117 F F F 6 120-139 56 56 56 97 97 97 F F F 7 140-159 60 60 60 106 106 106 F F F 8 160-179 61 61 61 110 110 110 F F F 9 180-199 58 58 58 101 101 101 F F F 10 200-219 60 60 60 108 108 108 F F F 11 220-239 54 54 54 95 95 95 F F F 12 240-259 63 63 63 117 117 117 F F F 13 260-279 56 56 56 99 99 99 F F F 14 280-299 54 54 54 94 94 94 F F F 15 300-319 54 54 54 95 95 95 F F F 16 320-339 62 62 62 113 113 113 F F F 17 340-359 62 62 62 113 113 113 F F F 18 360-379 60 60 60 105 105 105 F F F 19 380-399 62 62 62 116 116 116 F F F 20 400-419 63 63 63 117 117 117 F F F 21 420-439 63 63 63 117 117 117 F F F 22 440-459 61 61 61 110 110 110 F F F 23 460-479 61 61 61 110 110 110 F F F 24 480-499 58 58 58 102 102 102 F F F 25 500-519 62 62 62 114 114 114 F F F 26 520-539 62 62 62 114 114 114 F F F 27 540-559 60 60 60 109 109 109 F F F 28 560-579 63 63 63 117 117 117 F F F 29 580-599 63 63 63 117 117 117 F F F Max gen: 63, Max flush: 117 Begin loop 32 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:46.690767583Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1438,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:46.735022669Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1438,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:46.735661823Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1438} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"abd5430f1508af0bc008143ea2566277ef714ec2\",\n git_commit_timestamp: \"2025-01-17T20:55:03.000000000Z\",\n git_branch: \"alan/tokio-to-140\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:46.735701696Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1438} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:46.735717045Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1438} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:46.735731294Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1438} {"msg":"Crucible 47244154-ddd4-424b-8b86-e3bec21bdb2c has session id: 7e174162-a5c9-42e8-90f6-474aecbe4917","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:46.736314044Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1438,"session_id":"7e174162-a5c9-42e8-90f6-474aecbe4917"} {"msg":"Upstairs opts: Upstairs UUID: 47244154-ddd4-424b-8b86-e3bec21bdb2c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:46.736353737Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1438,"session_id":"7e174162-a5c9-42e8-90f6-474aecbe4917"} {"msg":"Crucible stats registered with UUID: 47244154-ddd4-424b-8b86-e3bec21bdb2c","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:46.736376461Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1438,"session_id":"7e174162-a5c9-42e8-90f6-474aecbe4917"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:13:46.736397663Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1438,"session_id":"7e174162-a5c9-42e8-90f6-474aecbe4917"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:46.736850088Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1438,"task":"crutest"} {"msg":"The guest has requested activation with gen:64","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:46.73688846Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1438} {"msg":"47244154-ddd4-424b-8b86-e3bec21bdb2c active request set","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:46.736921749Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1438,"session_id":"7e174162-a5c9-42e8-90f6-474aecbe4917"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:46.736945733Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1438,"client":"0","":"downstairs","session_id":"7e174162-a5c9-42e8-90f6-474aecbe4917"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:46.736987758Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1438,"client":"0","":"downstairs","session_id":"7e174162-a5c9-42e8-90f6-474aecbe4917"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:46.737017595Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1438,"client":"1","":"downstairs","session_id":"7e174162-a5c9-42e8-90f6-474aecbe4917"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:46.737046732Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1438,"client":"1","":"downstairs","session_id":"7e174162-a5c9-42e8-90f6-474aecbe4917"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:46.737072547Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1438,"client":"2","":"downstairs","session_id":"7e174162-a5c9-42e8-90f6-474aecbe4917"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:46.737101504Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1438,"client":"2","":"downstairs","session_id":"7e174162-a5c9-42e8-90f6-474aecbe4917"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:46.737123407Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1438,"":"io task","client":"2","":"downstairs","session_id":"7e174162-a5c9-42e8-90f6-474aecbe4917"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:46.7371453Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1438,"":"io task","client":"0","":"downstairs","session_id":"7e174162-a5c9-42e8-90f6-474aecbe4917"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:46.737167153Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1438,"":"io task","client":"1","":"downstairs","session_id":"7e174162-a5c9-42e8-90f6-474aecbe4917"} {"msg":"ds_connection connected from Ok(127.0.0.1:56062)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:46.737189246Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1438,"":"io task","client":"1","":"downstairs","session_id":"7e174162-a5c9-42e8-90f6-474aecbe4917"} {"msg":"ds_connection connected from Ok(127.0.0.1:50525)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:46.737483988Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1438,"":"io task","client":"2","":"downstairs","session_id":"7e174162-a5c9-42e8-90f6-474aecbe4917"} {"msg":"ds_connection connected from Ok(127.0.0.1:61067)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:46.737524752Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1438,"":"io task","client":"0","":"downstairs","session_id":"7e174162-a5c9-42e8-90f6-474aecbe4917"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:46.737960847Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1438,"client":"0","":"downstairs","session_id":"7e174162-a5c9-42e8-90f6-474aecbe4917"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:46.738011146Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1438,"client":"2","":"downstairs","session_id":"7e174162-a5c9-42e8-90f6-474aecbe4917"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:46.738441648Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1438,"client":"1","":"downstairs","session_id":"7e174162-a5c9-42e8-90f6-474aecbe4917"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:46.738641455Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1438,"client":"2","":"downstairs","session_id":"7e174162-a5c9-42e8-90f6-474aecbe4917"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:46.738673083Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1438,"client":"0","":"downstairs","session_id":"7e174162-a5c9-42e8-90f6-474aecbe4917"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:46.739036806Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1438,"client":"1","":"downstairs","session_id":"7e174162-a5c9-42e8-90f6-474aecbe4917"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:46.73919802Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1438,"session_id":"7e174162-a5c9-42e8-90f6-474aecbe4917"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:46.73923197Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1438,"session_id":"7e174162-a5c9-42e8-90f6-474aecbe4917"} {"msg":"[0]R flush_numbers[0..12]: [104, 103, 109, 109, 105, 117, 97, 106, 110, 101, 108, 95]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:46.739528923Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1438,"":"downstairs","":"downstairs","session_id":"7e174162-a5c9-42e8-90f6-474aecbe4917"} {"msg":"[0]R generation[0..12]: [59, 58, 60, 60, 60, 63, 56, 60, 61, 58, 60, 54]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:46.739552217Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1438,"":"downstairs","":"downstairs","session_id":"7e174162-a5c9-42e8-90f6-474aecbe4917"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:46.739567096Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1438,"":"downstairs","":"downstairs","session_id":"7e174162-a5c9-42e8-90f6-474aecbe4917"} {"msg":"[1]R flush_numbers[0..12]: [104, 103, 109, 109, 105, 117, 97, 106, 110, 101, 108, 95]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:46.739581084Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1438,"":"downstairs","":"downstairs","session_id":"7e174162-a5c9-42e8-90f6-474aecbe4917"} {"msg":"[1]R generation[0..12]: [59, 58, 60, 60, 60, 63, 56, 60, 61, 58, 60, 54]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:46.739594532Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1438,"":"downstairs","":"downstairs","session_id":"7e174162-a5c9-42e8-90f6-474aecbe4917"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:46.73960828Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1438,"":"downstairs","":"downstairs","session_id":"7e174162-a5c9-42e8-90f6-474aecbe4917"} {"msg":"[2]R flush_numbers[0..12]: [104, 103, 109, 109, 105, 117, 97, 106, 110, 101, 108, 95]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:46.739621817Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1438,"":"downstairs","":"downstairs","session_id":"7e174162-a5c9-42e8-90f6-474aecbe4917"} {"msg":"[2]R generation[0..12]: [59, 58, 60, 60, 60, 63, 56, 60, 61, 58, 60, 54]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:46.739639038Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1438,"":"downstairs","":"downstairs","session_id":"7e174162-a5c9-42e8-90f6-474aecbe4917"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:46.739652886Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1438,"":"downstairs","":"downstairs","session_id":"7e174162-a5c9-42e8-90f6-474aecbe4917"} {"msg":"Max found gen is 64","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:46.739665543Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1438,"":"downstairs","":"downstairs","session_id":"7e174162-a5c9-42e8-90f6-474aecbe4917"} {"msg":"Generation requested: 64 >= found:64","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:46.73967817Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1438,"":"downstairs","":"downstairs","session_id":"7e174162-a5c9-42e8-90f6-474aecbe4917"} {"msg":"Next flush: 118","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-17T21:13:46.739690648Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1438,"":"downstairs","":"downstairs","session_id":"7e174162-a5c9-42e8-90f6-474aecbe4917"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:46.739708478Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1438,"":"downstairs","":"downstairs","session_id":"7e174162-a5c9-42e8-90f6-474aecbe4917"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:46.739721576Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1438,"session_id":"7e174162-a5c9-42e8-90f6-474aecbe4917"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:46.739733333Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1438,"client":"0","":"downstairs","session_id":"7e174162-a5c9-42e8-90f6-474aecbe4917"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:46.73974637Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1438,"client":"1","":"downstairs","session_id":"7e174162-a5c9-42e8-90f6-474aecbe4917"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:46.739759028Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1438,"client":"2","":"downstairs","session_id":"7e174162-a5c9-42e8-90f6-474aecbe4917"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:46.739771795Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1438,"session_id":"7e174162-a5c9-42e8-90f6-474aecbe4917"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:46.739783622Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1438,"session_id":"7e174162-a5c9-42e8-90f6-474aecbe4917"} {"msg":"47244154-ddd4-424b-8b86-e3bec21bdb2c is now active with session: 7e174162-a5c9-42e8-90f6-474aecbe4917","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:46.739795999Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1438,"session_id":"7e174162-a5c9-42e8-90f6-474aecbe4917"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:46.739807626Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1438,"session_id":"7e174162-a5c9-42e8-90f6-474aecbe4917"} {"msg":"The guest has finished waiting for activation with:64","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:46.739819293Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1438} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 289 len 32768 data: 4 5 5 6 5 4 6 6 02/30 Flush 03/30 Write block 195 len 8192 data: 8 7 04/30 Write block 552 len 32768 data: 4 3 3 4 5 6 7 6 05/30 Read block 67 len 28672 06/30 Write block 16 len 28672 data: 8 6 6 5 3 6 5 07/30 Read block 592 len 24576 08/30 Read block 211 len 24576 09/30 Read block 330 len 36864 10/30 Flush 11/30 Write block 497 len 16384 data: 6 7 9 10 12/30 Write block 456 len 28672 data: 6 7 7 8 8 9 9 13/30 Write block 359 len 20480 data: 8 6 5 5 4 14/30 Read block 389 len 4096 15/30 Write block 102 len 40960 data: 8 7 7 7 5 4 3 5 5 6 16/30 Write block 437 len 40960 data: 3 2 2 2 2 2 4 5 5 4 17/30 Read block 470 len 4096 18/30 Write block 5 len 12288 data: 4 3 3 19/30 Read block 348 len 8192 20/30 Read block 527 len 40960 21/30 Read block 166 len 28672 22/30 Read block 198 len 16384 23/30 Write block 357 len 20480 data: 8 8 9 7 6 24/30 Write block 5 len 28672 data: 5 4 4 3 5 8 10 25/30 Write block 587 len 4096 data: 4 26/30 Read block 545 len 36864 27/30 Write block 374 len 32768 data: 9 9 9 11 11 10 7 6 28/30 Read block 557 len 32768 29/30 Read block 93 len 36864 30/30 Read block 492 len 40960 ---------------------------------------------------------------- Crucible gen:64 GIO:true work queues: Upstairs:0 downstairs:20 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1010 Acked Write 4 Done Done Done false 1011 Acked Write 7 Done Done Done false 1012 Acked Write 5 Done Done Done false 1013 Acked Read 1 Done Done Done false 1014 Acked Write 10 Done Done Done false 1015 Acked Write 10 Done Done Done false 1016 Acked Read 1 Done Done Done false 1017 Acked Write 3 Done Done Done false 1018 Acked Read 2 Done Done Done false 1019 Acked Read 10 Done Done Done false 1020 Acked Read 7 Done Done Done false 1021 Acked Read 4 Done Done Done false 1022 Acked Write 5 Done Done Done false 1023 Acked Write 7 Done Done Done false 1024 Acked Write 1 Done Done Done false 1025 Acked Read 9 Done Done Done false 1026 Acked Write 8 Done Done Done false 1027 Acked Read 8 Done Done Done false 1028 Acked Read 9 Done Done Done false 1029 Acked Read 10 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 20 20 20 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1009 1009 1009 Downstairs last five completed: 1009 1008 1007 1006 1005 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:13:46.760455694Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1438} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 64 64 64 119 119 119 T T T 1 020-039 64 64 64 119 119 119 F F F 2 040-059 60 60 60 109 109 109 F F F 3 060-079 60 60 60 109 109 109 F F F 4 080-099 60 60 60 105 105 105 F F F 5 100-119 63 63 63 117 117 117 T T T 6 120-139 56 56 56 97 97 97 F F F 7 140-159 60 60 60 106 106 106 F F F 8 160-179 61 61 61 110 110 110 F F F 9 180-199 64 64 64 119 119 119 F F F 10 200-219 60 60 60 108 108 108 F F F 11 220-239 54 54 54 95 95 95 F F F 12 240-259 63 63 63 117 117 117 F F F 13 260-279 56 56 56 99 99 99 F F F 14 280-299 64 64 64 118 118 118 F F F 15 300-319 54 54 54 95 95 95 F F F 16 320-339 62 62 62 113 113 113 F F F 17 340-359 62 62 62 113 113 113 T T T 18 360-379 60 60 60 105 105 105 T T T 19 380-399 62 62 62 116 116 116 T T T 20 400-419 63 63 63 117 117 117 F F F 21 420-439 63 63 63 117 117 117 T T T 22 440-459 61 61 61 110 110 110 T T T 23 460-479 61 61 61 110 110 110 T T T 24 480-499 58 58 58 102 102 102 T T T 25 500-519 62 62 62 114 114 114 T T T 26 520-539 62 62 62 114 114 114 F F F 27 540-559 64 64 64 119 119 119 F F F 28 560-579 63 63 63 117 117 117 F F F 29 580-599 63 63 63 117 117 117 T T T Max gen: 64, Max flush: 119 A Difference in extent metadata is expected here On loop 32 Verifying data now No change after verify Loop: 32 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 65 65 65 120 120 120 F F F 1 020-039 64 64 64 119 119 119 F F F 2 040-059 60 60 60 109 109 109 F F F 3 060-079 60 60 60 109 109 109 F F F 4 080-099 60 60 60 105 105 105 F F F 5 100-119 65 65 65 120 120 120 F F F 6 120-139 56 56 56 97 97 97 F F F 7 140-159 60 60 60 106 106 106 F F F 8 160-179 61 61 61 110 110 110 F F F 9 180-199 64 64 64 119 119 119 F F F 10 200-219 60 60 60 108 108 108 F F F 11 220-239 54 54 54 95 95 95 F F F 12 240-259 63 63 63 117 117 117 F F F 13 260-279 56 56 56 99 99 99 F F F 14 280-299 64 64 64 118 118 118 F F F 15 300-319 54 54 54 95 95 95 F F F 16 320-339 62 62 62 113 113 113 F F F 17 340-359 65 65 65 120 120 120 F F F 18 360-379 65 65 65 120 120 120 F F F 19 380-399 65 65 65 120 120 120 F F F 20 400-419 63 63 63 117 117 117 F F F 21 420-439 65 65 65 120 120 120 F F F 22 440-459 65 65 65 120 120 120 F F F 23 460-479 65 65 65 120 120 120 F F F 24 480-499 65 65 65 120 120 120 F F F 25 500-519 65 65 65 120 120 120 F F F 26 520-539 62 62 62 114 114 114 F F F 27 540-559 64 64 64 119 119 119 F F F 28 560-579 63 63 63 117 117 117 F F F 29 580-599 65 65 65 120 120 120 F F F Max gen: 65, Max flush: 120 Begin loop 33 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:51.037899687Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1446,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:51.082180468Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1446,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:51.08283431Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1446} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"abd5430f1508af0bc008143ea2566277ef714ec2\",\n git_commit_timestamp: \"2025-01-17T20:55:03.000000000Z\",\n git_branch: \"alan/tokio-to-140\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:51.082873393Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1446} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:51.082892324Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1446} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:51.082907722Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1446} {"msg":"Crucible 06dc7e27-8d11-4674-92e4-1e842b06c4ac has session id: 1bc3f072-7e84-4bfc-bfed-be92d490d4f5","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:51.083479867Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1446,"session_id":"1bc3f072-7e84-4bfc-bfed-be92d490d4f5"} {"msg":"Upstairs opts: Upstairs UUID: 06dc7e27-8d11-4674-92e4-1e842b06c4ac, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:51.08351978Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1446,"session_id":"1bc3f072-7e84-4bfc-bfed-be92d490d4f5"} {"msg":"Crucible stats registered with UUID: 06dc7e27-8d11-4674-92e4-1e842b06c4ac","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:51.083546015Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1446,"session_id":"1bc3f072-7e84-4bfc-bfed-be92d490d4f5"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:13:51.083568349Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1446,"session_id":"1bc3f072-7e84-4bfc-bfed-be92d490d4f5"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:51.083958136Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1446,"task":"crutest"} {"msg":"The guest has requested activation with gen:66","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:51.083983771Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1446} {"msg":"06dc7e27-8d11-4674-92e4-1e842b06c4ac active request set","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:51.084069472Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1446,"session_id":"1bc3f072-7e84-4bfc-bfed-be92d490d4f5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:51.084095367Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1446,"client":"0","":"downstairs","session_id":"1bc3f072-7e84-4bfc-bfed-be92d490d4f5"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:51.084121252Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1446,"client":"0","":"downstairs","session_id":"1bc3f072-7e84-4bfc-bfed-be92d490d4f5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:51.08413538Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1446,"client":"1","":"downstairs","session_id":"1bc3f072-7e84-4bfc-bfed-be92d490d4f5"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:51.084149368Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1446,"client":"1","":"downstairs","session_id":"1bc3f072-7e84-4bfc-bfed-be92d490d4f5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:51.084162276Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1446,"client":"2","":"downstairs","session_id":"1bc3f072-7e84-4bfc-bfed-be92d490d4f5"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:51.084175784Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1446,"client":"2","":"downstairs","session_id":"1bc3f072-7e84-4bfc-bfed-be92d490d4f5"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:51.084191793Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1446,"":"io task","client":"2","":"downstairs","session_id":"1bc3f072-7e84-4bfc-bfed-be92d490d4f5"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:51.084207852Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1446,"":"io task","client":"0","":"downstairs","session_id":"1bc3f072-7e84-4bfc-bfed-be92d490d4f5"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:51.084232257Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1446,"":"io task","client":"1","":"downstairs","session_id":"1bc3f072-7e84-4bfc-bfed-be92d490d4f5"} {"msg":"ds_connection connected from Ok(127.0.0.1:49263)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:51.084320808Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1446,"":"io task","client":"0","":"downstairs","session_id":"1bc3f072-7e84-4bfc-bfed-be92d490d4f5"} {"msg":"ds_connection connected from Ok(127.0.0.1:35480)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:51.084356279Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1446,"":"io task","client":"2","":"downstairs","session_id":"1bc3f072-7e84-4bfc-bfed-be92d490d4f5"} {"msg":"ds_connection connected from Ok(127.0.0.1:57422)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:51.084383065Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1446,"":"io task","client":"1","":"downstairs","session_id":"1bc3f072-7e84-4bfc-bfed-be92d490d4f5"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:51.085146831Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1446,"client":"0","":"downstairs","session_id":"1bc3f072-7e84-4bfc-bfed-be92d490d4f5"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:51.085174978Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1446,"client":"1","":"downstairs","session_id":"1bc3f072-7e84-4bfc-bfed-be92d490d4f5"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:51.085712792Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1446,"client":"1","":"downstairs","session_id":"1bc3f072-7e84-4bfc-bfed-be92d490d4f5"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:51.085869053Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1446,"client":"0","":"downstairs","session_id":"1bc3f072-7e84-4bfc-bfed-be92d490d4f5"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:51.086011737Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1446,"client":"2","":"downstairs","session_id":"1bc3f072-7e84-4bfc-bfed-be92d490d4f5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:51.086243542Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1446,"session_id":"1bc3f072-7e84-4bfc-bfed-be92d490d4f5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:51.08635875Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1446,"session_id":"1bc3f072-7e84-4bfc-bfed-be92d490d4f5"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:51.086652141Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1446,"client":"2","":"downstairs","session_id":"1bc3f072-7e84-4bfc-bfed-be92d490d4f5"} {"msg":"[0]R flush_numbers[0..12]: [120, 119, 109, 109, 105, 120, 97, 106, 110, 119, 108, 95]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:51.087160988Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1446,"":"downstairs","":"downstairs","session_id":"1bc3f072-7e84-4bfc-bfed-be92d490d4f5"} {"msg":"[0]R generation[0..12]: [65, 64, 60, 60, 60, 65, 56, 60, 61, 64, 60, 54]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:51.08718101Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1446,"":"downstairs","":"downstairs","session_id":"1bc3f072-7e84-4bfc-bfed-be92d490d4f5"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:51.087196239Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1446,"":"downstairs","":"downstairs","session_id":"1bc3f072-7e84-4bfc-bfed-be92d490d4f5"} {"msg":"[1]R flush_numbers[0..12]: [120, 119, 109, 109, 105, 120, 97, 106, 110, 119, 108, 95]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:51.087210127Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1446,"":"downstairs","":"downstairs","session_id":"1bc3f072-7e84-4bfc-bfed-be92d490d4f5"} {"msg":"[1]R generation[0..12]: [65, 64, 60, 60, 60, 65, 56, 60, 61, 64, 60, 54]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:51.087223275Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1446,"":"downstairs","":"downstairs","session_id":"1bc3f072-7e84-4bfc-bfed-be92d490d4f5"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:51.087236773Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1446,"":"downstairs","":"downstairs","session_id":"1bc3f072-7e84-4bfc-bfed-be92d490d4f5"} {"msg":"[2]R flush_numbers[0..12]: [120, 119, 109, 109, 105, 120, 97, 106, 110, 119, 108, 95]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:51.087250171Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1446,"":"downstairs","":"downstairs","session_id":"1bc3f072-7e84-4bfc-bfed-be92d490d4f5"} {"msg":"[2]R generation[0..12]: [65, 64, 60, 60, 60, 65, 56, 60, 61, 64, 60, 54]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:51.087263268Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1446,"":"downstairs","":"downstairs","session_id":"1bc3f072-7e84-4bfc-bfed-be92d490d4f5"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:51.087276646Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1446,"":"downstairs","":"downstairs","session_id":"1bc3f072-7e84-4bfc-bfed-be92d490d4f5"} {"msg":"Max found gen is 66","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:51.087289183Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1446,"":"downstairs","":"downstairs","session_id":"1bc3f072-7e84-4bfc-bfed-be92d490d4f5"} {"msg":"Generation requested: 66 >= found:66","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:51.087301991Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1446,"":"downstairs","":"downstairs","session_id":"1bc3f072-7e84-4bfc-bfed-be92d490d4f5"} {"msg":"Next flush: 121","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:51.087314258Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1446,"":"downstairs","":"downstairs","session_id":"1bc3f072-7e84-4bfc-bfed-be92d490d4f5"} {"msg":"All extents match","v":Wait for a query_work_queue command to finish before sending IO 0,"name":"crucible","level":30,"time":"2025-01-17T21:13:51.087332909Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1446,"":"downstairs","":"downstairs","session_id":"1bc3f072-7e84-4bfc-bfed-be92d490d4f5"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:51.087345767Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1446,"session_id":"1bc3f072-7e84-4bfc-bfed-be92d490d4f5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:51.087359524Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1446,"client":"0","":"downstairs","session_id":"1bc3f072-7e84-4bfc-bfed-be92d490d4f5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:51.087372402Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1446,"client":"1","":"downstairs","session_id":"1bc3f072-7e84-4bfc-bfed-be92d490d4f5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:51.087385179Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1446,"client":"2","":"downstairs","session_id":"1bc3f072-7e84-4bfc-bfed-be92d490d4f5"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:51.087397977Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1446,"session_id":"1bc3f072-7e84-4bfc-bfed-be92d490d4f5"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:51.087409974Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1446,"session_id":"1bc3f072-7e84-4bfc-bfed-be92d490d4f5"} {"msg":"06dc7e27-8d11-4674-92e4-1e842b06c4ac is now active with session: 1bc3f072-7e84-4bfc-bfed-be92d490d4f5","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:51.087422441Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1446,"session_id":"1bc3f072-7e84-4bfc-bfed-be92d490d4f5"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:51.087434168Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1446,"session_id":"1bc3f072-7e84-4bfc-bfed-be92d490d4f5"} {"msg":"The guest has finished waiting for activation with:66","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:51.087445875Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1446} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 268 len 28672 data: 5 6 7 6 5 5 5 02/30 Write block 316 len 32768 data: 5 5 5 6 6 5 5 6 03/30 Flush 04/30 Write block 489 len 40960 data: 5 5 5 4 5 5 6 7 7 8 05/30 Write block 343 len 28672 data: 11 7 6 6 7 6 4 06/30 Flush 07/30 Write block 348 len 12288 data: 7 5 7 08/30 Write block 247 len 20480 data: 7 7 5 5 5 09/30 Read block 189 len 12288 10/30 Write block 13 len 32768 data: 10 9 7 9 7 7 6 4 11/30 Write block 378 len 24576 data: 12 11 8 7 5 6 12/30 Read block 537 len 32768 13/30 Read block 424 len 12288 14/30 Write block 105 len 16384 data: 8 6 5 4 15/30 Write block 234 len 40960 data: 7 8 7 6 4 4 5 7 7 7 16/30 Read block 303 len 12288 17/30 Read block 540 len 36864 18/30 Write block 190 len 40960 data: 7 7 8 10 11 9 8 6 8 8 19/30 Flush 20/30 Write block 577 len 8192 data: 6 5 21/30 Read block 245 len 16384 22/30 Write block 494 len 20480 data: 6 7 8 8 9 23/30 Read block 563 len 36864 24/30 Read block 254 len 8192 25/30 Write block 59 len 12288 data: 9 7 7 26/30 Write block 100 len 8192 data: 8 8 27/30 Read block 573 len 40960 28/30 Read block 298 len 12288 29/30 Read block 384 len 32768 30/30 Write block 562 len 40960 data: 8 7 5 4 5 4 4 3 3 2 ---------------------------------------------------------------- Crucible gen:66 GIO:true work queues: Upstairs:0 downstairs:11 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1019 Acked Write 2 Done Done Done false 1020 Acked Read 4 Done Done Done false 1021 Acked Write 5 Done Done Done false 1022 Acked Read 9 Done Done Done false 1023 Acked Read 2 Done Done Done false 1024 Acked Write 3 Done Done Done false 1025 Acked Write 2 Done Done Done false 1026 Acked Read 10 Done Done Done false 1027 Acked Read 3 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 10 10 10 30 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1018 1018 1018 Downstairs last five completed: 1018 1017 1016 1015 1014 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:13:51.107732312Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1446} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 66 66 66 123 123 123 F F F 1 020-039 66 66 66 123 123 123 F F F 2 040-059 60 60 60 109 109 109 T T T 3 060-079 60 60 60 109 109 109 T T T 4 080-099 60 60 60 105 105 105 F F F 5 100-119 66 66 66 123 123 123 T T T 6 120-139 56 56 56 97 97 97 F F F 7 140-159 60 60 60 106 106 106 F F F 8 160-179 61 61 61 110 110 110 F F F 9 180-199 66 66 66 123 123 123 F F F 10 200-219 60 60 60 108 108 108 F F F 11 220-239 66 66 66 123 123 123 F F F 12 240-259 66 66 66 123 123 123 F F F 13 260-279 66 66 66 121 121 121 F F F 14 280-299 64 64 64 118 118 118 F F F 15 300-319 66 66 66 121 121 121 F F F 16 320-339 66 66 66 121 121 121 F F F 17 340-359 66 66 66 123 123 123 F F F 18 360-379 66 66 66 123 123 123 F F F 19 380-399 66 66 66 123 123 123 F F F 20 400-419 63 63 63 117 117 117 F F F 21 420-439 65 65 65 120 120 120 F F F 22 440-459 65 65 65 120 120 120 F F F 23 460-479 65 65 65 120 120 120 F F F 24 480-499 66 66 66 122 122 122 T T T 25 500-519 65 65 65 120 120 120 F F F 26 520-539 62 62 62 114 114 114 F F F 27 540-559 64 64 64 119 119 119 F F F 28 560-579 63 63 63 117 117 117 T T T 29 580-599 65 65 65 120 120 120 F F F Max gen: 66, Max flush: 123 A Difference in extent metadata is expected here On loop 33 Verifying data now No change after verify Loop: 33 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 66 66 66 123 123 123 F F F 1 020-039 66 66 66 123 123 123 F F F 2 040-059 67 67 67 124 124 124 F F F 3 060-079 67 67 67 124 124 124 F F F 4 080-099 60 60 60 105 105 105 F F F 5 100-119 67 67 67 124 124 124 F F F 6 120-139 56 56 56 97 97 97 F F F 7 140-159 60 60 60 106 106 106 F F F 8 160-179 61 61 61 110 110 110 F F F 9 180-199 66 66 66 123 123 123 F F F 10 200-219 60 60 60 108 108 108 F F F 11 220-239 66 66 66 123 123 123 F F F 12 240-259 66 66 66 123 123 123 F F F 13 260-279 66 66 66 121 121 121 F F F 14 280-299 64 64 64 118 118 118 F F F 15 300-319 66 66 66 121 121 121 F F F 16 320-339 66 66 66 121 121 121 F F F 17 340-359 66 66 66 123 123 123 F F F 18 360-379 66 66 66 123 123 123 F F F 19 380-399 66 66 66 123 123 123 F F F 20 400-419 63 63 63 117 117 117 F F F 21 420-439 65 65 65 120 120 120 F F F 22 440-459 65 65 65 120 120 120 F F F 23 460-479 65 65 65 120 120 120 F F F 24 480-499 67 67 67 124 124 124 F F F 25 500-519 65 65 65 120 120 120 F F F 26 520-539 62 62 62 114 114 114 F F F 27 540-559 64 64 64 119 119 119 F F F 28 560-579 67 67 67 124 124 124 F F F 29 580-599 65 65 65 120 120 120 F F F Max gen: 67, Max flush: 124 Begin loop 34 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:55.348766284Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1454,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:55.393719218Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1454,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:55.394353018Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1454} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"abd5430f1508af0bc008143ea2566277ef714ec2\",\n git_commit_timestamp: \"2025-01-17T20:55:03.000000000Z\",\n git_branch: \"alan/tokio-to-140\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:55.394392742Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1454} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:55.39440754Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1454} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:55.394421909Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1454} {"msg":"Crucible 05f2d3e2-bda3-4578-b82f-04af7320c8df has session id: a5ba7c38-a5fc-4a5b-bdd8-f95488ee240e","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:55.39486867Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1454,"session_id":"a5ba7c38-a5fc-4a5b-bdd8-f95488ee240e"} {"msg":"Upstairs opts: Upstairs UUID: 05f2d3e2-bda3-4578-b82f-04af7320c8df, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:55.394906672Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1454,"session_id":"a5ba7c38-a5fc-4a5b-bdd8-f95488ee240e"} {"msg":"Crucible stats registered with UUID: 05f2d3e2-bda3-4578-b82f-04af7320c8df","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:55.394929325Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1454,"session_id":"a5ba7c38-a5fc-4a5b-bdd8-f95488ee240e"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:13:55.394949977Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1454,"session_id":"a5ba7c38-a5fc-4a5b-bdd8-f95488ee240e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:55.395483009Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1454,"task":"crutest"} {"msg":"The guest has requested activation with gen:68","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:55.395507843Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1454} {"msg":"05f2d3e2-bda3-4578-b82f-04af7320c8df active request set","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:55.395575673Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1454,"session_id":"a5ba7c38-a5fc-4a5b-bdd8-f95488ee240e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:55.39560436Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1454,"client":"0","":"downstairs","session_id":"a5ba7c38-a5fc-4a5b-bdd8-f95488ee240e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:55.395640101Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1454,"client":"0","":"downstairs","session_id":"a5ba7c38-a5fc-4a5b-bdd8-f95488ee240e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:55.395669798Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1454,"client":"1","":"downstairs","session_id":"a5ba7c38-a5fc-4a5b-bdd8-f95488ee240e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:55.395719507Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1454,"client":"1","":"downstairs","session_id":"a5ba7c38-a5fc-4a5b-bdd8-f95488ee240e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:55.395747333Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1454,"client":"2","":"downstairs","session_id":"a5ba7c38-a5fc-4a5b-bdd8-f95488ee240e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:55.395774619Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1454,"client":"2","":"downstairs","session_id":"a5ba7c38-a5fc-4a5b-bdd8-f95488ee240e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:55.395800564Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1454,"":"io task","client":"2","":"downstairs","session_id":"a5ba7c38-a5fc-4a5b-bdd8-f95488ee240e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:55.3958276Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1454,"":"io task","client":"0","":"downstairs","session_id":"a5ba7c38-a5fc-4a5b-bdd8-f95488ee240e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:55.395852925Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1454,"":"io task","client":"1","":"downstairs","session_id":"a5ba7c38-a5fc-4a5b-bdd8-f95488ee240e"} {"msg":"ds_connection connected from Ok(127.0.0.1:35090)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:55.395876999Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1454,"":"io task","client":"1","":"downstairs","session_id":"a5ba7c38-a5fc-4a5b-bdd8-f95488ee240e"} {"msg":"ds_connection connected from Ok(127.0.0.1:38939)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:55.395984912Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1454,"":"io task","client":"0","":"downstairs","session_id":"a5ba7c38-a5fc-4a5b-bdd8-f95488ee240e"} {"msg":"ds_connection connected from Ok(127.0.0.1:41061)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:55.396020053Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1454,"":"io task","client":"2","":"downstairs","session_id":"a5ba7c38-a5fc-4a5b-bdd8-f95488ee240e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:55.396666971Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1454,"client":"2","":"downstairs","session_id":"a5ba7c38-a5fc-4a5b-bdd8-f95488ee240e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:55.396695318Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1454,"client":"0","":"downstairs","session_id":"a5ba7c38-a5fc-4a5b-bdd8-f95488ee240e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:55.396748719Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1454,"client":"1","":"downstairs","session_id":"a5ba7c38-a5fc-4a5b-bdd8-f95488ee240e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:55.397368851Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1454,"client":"1","":"downstairs","session_id":"a5ba7c38-a5fc-4a5b-bdd8-f95488ee240e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:55.397397498Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1454,"client":"2","":"downstairs","session_id":"a5ba7c38-a5fc-4a5b-bdd8-f95488ee240e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:55.397416709Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1454,"client":"0","":"downstairs","session_id":"a5ba7c38-a5fc-4a5b-bdd8-f95488ee240e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:55.397918642Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1454,"session_id":"a5ba7c38-a5fc-4a5b-bdd8-f95488ee240e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:55.398013858Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1454,"session_id":"a5ba7c38-a5fc-4a5b-bdd8-f95488ee240e"} {"msg":"[0]R flush_numbers[0..12]: [123, 123, 124, 124, 105, 124, 97, 106, 110, 123, 108, 123]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:55.398128875Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1454,"":"downstairs","":"downstairs","session_id":"a5ba7c38-a5fc-4a5b-bdd8-f95488ee240e"} {"msg":"[0]R generation[0..12]: [66, 66, 67, 67, 60, 67, 56, 60, 61, 66, 60, 66]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:55.398148337Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1454,"":"downstairs","":"downstairs","session_id":"a5ba7c38-a5fc-4a5b-bdd8-f95488ee240e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:55.398162685Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1454,"":"downstairs","":"downstairs","session_id":"a5ba7c38-a5fc-4a5b-bdd8-f95488ee240e"} {"msg":"[1]R flush_numbers[0..12]: [123, 123, 124, 124, 105, 124, 97, 106, 110, 123, 108, 123]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:55.398176503Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1454,"":"downstairs","":"downstairs","session_id":"a5ba7c38-a5fc-4a5b-bdd8-f95488ee240e"} {"msg":"[1]R generation[0..12]: [66, 66, 67, 67, 60, 67, 56, 60, 61, 66, 60, 66]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:55.398190161Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1454,"":"downstairs","":"downstairs","session_id":"a5ba7c38-a5fc-4a5b-bdd8-f95488ee240e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:55.398203949Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1454,"":"downstairs","":"downstairs","session_id":"a5ba7c38-a5fc-4a5b-bdd8-f95488ee240e"} {"msg":"[2]R flush_numbers[0..12]: [123, 123, 124, 124, 105, 124, 97, 106, 110, 123, 108, 123]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:55.398217357Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1454,"":"downstairs","":"downstairs","session_id":"a5ba7c38-a5fc-4a5b-bdd8-f95488ee240e"} {"msg":"[2]R generation[0..12]: [66, 66, 67, 67, 60, 67, 56, 60, 61, 66, 60, 66]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:55.398230465Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1454,"":"downstairs","":"downstairs","session_id":"a5ba7c38-a5fc-4a5b-bdd8-f95488ee240e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:55.398247525Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1454,"":"downstairs","":"downstairs","session_id":"a5ba7c38-a5fc-4a5b-bdd8-f95488ee240e"} {"msg":"Max found gen is 68","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:55.398260462Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1454,"":"downstairs","":"downstairs","session_id":"a5ba7c38-a5fc-4a5b-bdd8-f95488ee240e"} {"msg":"Generation requested: 68 >= found:68","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:55.39827332Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1454,"":"downstairs","":"downstairs","session_id":"a5ba7c38-a5fc-4a5b-bdd8-f95488ee240e"} {"msg":"Next flush: 125","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:55.398286147Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1454,"":"downstairs","":"downstairs","session_id":"a5ba7c38-a5fc-4a5b-bdd8-f95488ee240e"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-17T21:13:55.398298655Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1454,"":"downstairs","":"downstairs","session_id":"a5ba7c38-a5fc-4a5b-bdd8-f95488ee240e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:55.398315975Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1454,"session_id":"a5ba7c38-a5fc-4a5b-bdd8-f95488ee240e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:55.398327892Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1454,"client":"0","":"downstairs","session_id":"a5ba7c38-a5fc-4a5b-bdd8-f95488ee240e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:55.398340809Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1454,"client":"1","":"downstairs","session_id":"a5ba7c38-a5fc-4a5b-bdd8-f95488ee240e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:55.398353597Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1454,"client":"2","":"downstairs","session_id":"a5ba7c38-a5fc-4a5b-bdd8-f95488ee240e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:55.398366334Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1454,"session_id":"a5ba7c38-a5fc-4a5b-bdd8-f95488ee240e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:55.398378311Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1454,"session_id":"a5ba7c38-a5fc-4a5b-bdd8-f95488ee240e"} {"msg":"05f2d3e2-bda3-4578-b82f-04af7320c8df is now active with session: a5ba7c38-a5fc-4a5b-bdd8-f95488ee240e","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:55.398390878Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1454,"session_id":"a5ba7c38-a5fc-4a5b-bdd8-f95488ee240e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:55.398402415Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1454,"session_id":"a5ba7c38-a5fc-4a5b-bdd8-f95488ee240e"} {"msg":"The guest has finished waiting for activation with:68","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:55.398414302Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1454} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 469 len 32768 data: 9 8 6 7 5 4 5 5 02/30 Read block 247 len 24576 03/30 Flush 04/30 Write block 208 len 24576 data: 5 6 4 4 4 4 05/30 Read block 436 len 36864 06/30 Write block 290 len 40960 data: 6 6 7 6 5 7 7 6 5 7 07/30 Read block 443 len 20480 08/30 Write block 161 len 12288 data: 6 6 5 09/30 Write block 574 len 4096 data: 3 10/30 Write block 18 len 12288 data: 8 7 5 11/30 Write block 142 len 8192 data: 5 5 12/30 Write block 592 len 4096 data: 5 13/30 Write block 566 len 4096 data: 6 14/30 Read block 365 len 36864 15/30 Write block 117 len 8192 data: 6 6 16/30 Write block 147 len 16384 data: 5 5 5 5 17/30 Read block 293 len 28672 18/30 Write block 441 len 16384 data: 3 3 5 6 19/30 Write block 584 len 28672 data: 4 4 4 5 5 6 6 20/30 Read block 509 len 40960 21/30 Flush 22/30 Write block 78 len 4096 data: 8 23/30 Read block 229 len 40960 24/30 Write block 257 len 4096 data: 6 25/30 Read block 19 len 40960 26/30 Read block 66 len 12288 27/30 Write block 276 len 12288 data: 6 5 4 28/30 Read block 135 len 12288 29/30 Read block 396 len 8192 30/30 Read block 137 len 28672 ---------------------------------------------------------------- Crucible gen:68 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 1 Done Done Done false 1022 Acked Read 10 Done Done Done false 1023 Acked Write 1 Done Done Done false 1024 Acked Read 10 Done Done Done false 1025 Acked Read 3 Done Done Done false 1026 Acked Write 3 Done Done Done false 1027 Acked Read 3 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Read 7 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 0 1 Done 9 8 9 26 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1020 1020 1020 Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:13:55.416466753Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1454} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 68 68 68 126 126 126 F F F 1 020-039 68 68 68 126 126 126 F F F 2 040-059 67 67 67 124 124 124 F F F 3 060-079 67 67 67 124 124 124 T T T 4 080-099 60 60 60 105 105 105 F F F 5 100-119 68 68 68 126 126 126 F F F 6 120-139 56 56 56 97 97 97 F F F 7 140-159 68 68 68 126 126 126 F F F 8 160-179 68 68 68 126 126 126 F F F 9 180-199 66 66 66 123 123 123 F F F 10 200-219 68 68 68 126 126 126 F F F 11 220-239 66 66 66 123 123 123 F F F 12 240-259 66 66 66 123 123 123 T T T 13 260-279 66 66 66 121 121 121 T T T 14 280-299 68 68 68 126 126 126 F F F 15 300-319 66 66 66 121 121 121 F F F 16 320-339 66 66 66 121 121 121 F F F 17 340-359 66 66 66 123 123 123 F F F 18 360-379 66 66 66 123 123 123 F F F 19 380-399 66 66 66 123 123 123 F F F 20 400-419 63 63 63 117 117 117 F F F 21 420-439 65 65 65 120 120 120 F F F 22 440-459 68 68 68 126 126 126 F F F 23 460-479 68 68 68 125 125 125 F F F 24 480-499 67 67 67 124 124 124 F F F 25 500-519 65 65 65 120 120 120 F F F 26 520-539 62 62 62 114 114 114 F F F 27 540-559 64 64 64 119 119 119 F F F 28 560-579 68 68 68 126 126 126 F F F 29 580-599 68 68 68 126 126 126 F F F Max gen: 68, Max flush: 126 A Difference in extent metadata is expected here On loop 34 Verifying data now No change after verify Loop: 34 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 68 68 68 126 126 126 F F F 1 020-039 68 68 68 126 126 126 F F F 2 040-059 67 67 67 124 124 124 F F F 3 060-079 69 69 69 127 127 127 F F F 4 080-099 60 60 60 105 105 105 F F F 5 100-119 68 68 68 126 126 126 F F F 6 120-139 56 56 56 97 97 97 F F F 7 140-159 68 68 68 126 126 126 F F F 8 160-179 68 68 68 126 126 126 F F F 9 180-199 66 66 66 123 123 123 F F F 10 200-219 68 68 68 126 126 126 F F F 11 220-239 66 66 66 123 123 123 F F F 12 240-259 69 69 69 127 127 127 F F F 13 260-279 69 69 69 127 127 127 F F F 14 280-299 68 68 68 126 126 126 F F F 15 300-319 66 66 66 121 121 121 F F F 16 320-339 66 66 66 121 121 121 F F F 17 340-359 66 66 66 123 123 123 F F F 18 360-379 66 66 66 123 123 123 F F F 19 380-399 66 66 66 123 123 123 F F F 20 400-419 63 63 63 117 117 117 F F F 21 420-439 65 65 65 120 120 120 F F F 22 440-459 68 68 68 126 126 126 F F F 23 460-479 68 68 68 125 125 125 F F F 24 480-499 67 67 67 124 124 124 F F F 25 500-519 65 65 65 120 120 120 F F F 26 520-539 62 62 62 114 114 114 F F F 27 540-559 64 64 64 119 119 119 F F F 28 560-579 68 68 68 126 126 126 F F F 29 580-599 68 68 68 126 126 126 F F F Max gen: 69, Max flush: 127 Begin loop 35 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:59.644756296Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1462,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:13:59.688330934Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1462,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:59.688756493Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1462,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:59.721210101Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1462,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:59.721854498Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1462} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"abd5430f1508af0bc008143ea2566277ef714ec2\",\n git_commit_timestamp: \"2025-01-17T20:55:03.000000000Z\",\n git_branch: \"alan/tokio-to-140\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:59.721895101Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1462} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:59.721912271Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1462} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:59.721929502Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1462} {"msg":"Crucible bf88c22e-da75-4adf-a640-d7effd77e0ef has session id: a64631d7-a959-4da1-bc65-139b463ada7c","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:59.722488638Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1462,"session_id":"a64631d7-a959-4da1-bc65-139b463ada7c"} {"msg":"Upstairs opts: Upstairs UUID: bf88c22e-da75-4adf-a640-d7effd77e0ef, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:59.722536426Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1462,"session_id":"a64631d7-a959-4da1-bc65-139b463ada7c"} {"msg":"Crucible stats registered with UUID: bf88c22e-da75-4adf-a640-d7effd77e0ef","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:59.722562982Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1462,"session_id":"a64631d7-a959-4da1-bc65-139b463ada7c"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:13:59.722579191Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1462,"session_id":"a64631d7-a959-4da1-bc65-139b463ada7c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:59.723097164Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1462,"task":"crutest"} {"msg":"The guest has requested activation with gen:70","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:59.723120177Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1462} {"msg":"bf88c22e-da75-4adf-a640-d7effd77e0ef active request set","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:59.723191189Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1462,"session_id":"a64631d7-a959-4da1-bc65-139b463ada7c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:59.723208109Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1462,"client":"0","":"downstairs","session_id":"a64631d7-a959-4da1-bc65-139b463ada7c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:59.723231413Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1462,"client":"0","":"downstairs","session_id":"a64631d7-a959-4da1-bc65-139b463ada7c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:59.72324517Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1462,"client":"1","":"downstairs","session_id":"a64631d7-a959-4da1-bc65-139b463ada7c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:59.723259609Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1462,"client":"1","":"downstairs","session_id":"a64631d7-a959-4da1-bc65-139b463ada7c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:59.723272457Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1462,"client":"2","":"downstairs","session_id":"a64631d7-a959-4da1-bc65-139b463ada7c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:59.723285864Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1462,"client":"2","":"downstairs","session_id":"a64631d7-a959-4da1-bc65-139b463ada7c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:59.723298302Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1462,"":"io task","client":"2","":"downstairs","session_id":"a64631d7-a959-4da1-bc65-139b463ada7c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:59.72331168Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1462,"":"io task","client":"0","":"downstairs","session_id":"a64631d7-a959-4da1-bc65-139b463ada7c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:59.723327739Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1462,"":"io task","client":"1","":"downstairs","session_id":"a64631d7-a959-4da1-bc65-139b463ada7c"} {"msg":"ds_connection connected from Ok(127.0.0.1:52263)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:59.723629215Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1462,"":"io task","client":"0","":"downstairs","session_id":"a64631d7-a959-4da1-bc65-139b463ada7c"} {"msg":"ds_connection connected from Ok(127.0.0.1:42464)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:59.723670509Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1462,"":"io task","client":"2","":"downstairs","session_id":"a64631d7-a959-4da1-bc65-139b463ada7c"} {"msg":"ds_connection connected from Ok(127.0.0.1:36686)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:59.723699466Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1462,"":"io task","client":"1","":"downstairs","session_id":"a64631d7-a959-4da1-bc65-139b463ada7c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:59.724287059Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1462,"client":"1","":"downstairs","session_id":"a64631d7-a959-4da1-bc65-139b463ada7c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:59.724316747Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1462,"client":"2","":"downstairs","session_id":"a64631d7-a959-4da1-bc65-139b463ada7c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:59.724333006Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1462,"client":"0","":"downstairs","session_id":"a64631d7-a959-4da1-bc65-139b463ada7c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:59.724969889Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1462,"client":"0","":"downstairs","session_id":"a64631d7-a959-4da1-bc65-139b463ada7c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:59.725123519Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1462,"client":"1","":"downstairs","session_id":"a64631d7-a959-4da1-bc65-139b463ada7c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:59.725152045Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1462,"client":"2","":"downstairs","session_id":"a64631d7-a959-4da1-bc65-139b463ada7c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:59.725531577Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1462,"session_id":"a64631d7-a959-4da1-bc65-139b463ada7c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:59.725668987Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1462,"session_id":"a64631d7-a959-4da1-bc65-139b463ada7c"} {"msg":"[0]R flush_numbers[0..12]: [126, 126, 124, 127, 105, 126, 97, 126, 126, 123, 126, 123]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:59.725789328Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1462,"":"downstairs","":"downstairs","session_id":"a64631d7-a959-4da1-bc65-139b463ada7c"} {"msg":"[0]R generation[0..12]: [68, 68, 67, 69, 60, 68, 56, 68, 68, 66, 68, 66]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:59.72581008Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1462,"":"downstairs","":"downstairs","session_id":"a64631d7-a959-4da1-bc65-139b463ada7c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:59.72582802Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1462,"":"downstairs","":"downstairs","session_id":"a64631d7-a959-4da1-bc65-139b463ada7c"} {"msg":"[1]R flush_numbers[0..12]: [126, 126, 124, 127, 105, 126, 97, 126, 126, 123, 126, 123]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:59.725842108Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1462,"":"downstairs","":"downstairs","session_id":"a64631d7-a959-4da1-bc65-139b463ada7c"} {"msg":"[1]R generation[0..12]: [68, 68, 67, 69, 60, 68, 56, 68, 68, 66, 68, 66]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:59.725855606Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1462,"":"downstairs","":"downstairs","session_id":"a64631d7-a959-4da1-bc65-139b463ada7c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:59.725869384Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1462,"":"downstairs","":"downstairs","session_id":"a64631d7-a959-4da1-bc65-139b463ada7c"} {"msg":"[2]R flush_numbers[0..12]: [126, 126, 124, 127, 105, 126, 97, 126, 126, 123, 126, 123]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:59.725883162Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1462,"":"downstairs","":"downstairs","session_id":"a64631d7-a959-4da1-bc65-139b463ada7c"} {"msg":"[2]R generation[0..12]: [68, 68, 67, 69, 60, 68, 56, 68, 68, 66, 68, 66]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:59.72589671Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1462,"":"downstairs","":"downstairs","session_id":"a64631d7-a959-4da1-bc65-139b463ada7c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:59.725910378Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1462,"":"downstairs","":"downstairs","session_id":"a64631d7-a959-4da1-bc65-139b463ada7c"} {"msg":"Max found gen is 70","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:59.725922906Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1462,"":"downstairs","":"downstairs","session_id":"a64631d7-a959-4da1-bc65-139b463ada7c"} {"msg":"Generation requested: 70 >= found:70","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:59.725935683Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1462,"":"downstairs","":"downstairs","session_id":"a64631d7-a959-4da1-bc65-139b463ada7c"} Wait for a query_work_queue command to finish before sending IO {"msg":"Next flush: 128","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:59.725956195Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1462,"":"downstairs","":"downstairs","session_id":"a64631d7-a959-4da1-bc65-139b463ada7c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:59.725980139Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1462,"":"downstairs","":"downstairs","session_id":"a64631d7-a959-4da1-bc65-139b463ada7c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:59.726005374Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1462,"session_id":"a64631d7-a959-4da1-bc65-139b463ada7c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:59.726029008Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1462,"client":"0","":"downstairs","session_id":"a64631d7-a959-4da1-bc65-139b463ada7c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:59.726054683Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1462,"client":"1","":"downstairs","session_id":"a64631d7-a959-4da1-bc65-139b463ada7c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:59.72608487Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1462,"client":"2","":"downstairs","session_id":"a64631d7-a959-4da1-bc65-139b463ada7c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:59.726113467Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1462,"session_id":"a64631d7-a959-4da1-bc65-139b463ada7c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:59.726138512Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1462,"session_id":"a64631d7-a959-4da1-bc65-139b463ada7c"} {"msg":"bf88c22e-da75-4adf-a640-d7effd77e0ef is now active with session: a64631d7-a959-4da1-bc65-139b463ada7c","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:59.726165037Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1462,"session_id":"a64631d7-a959-4da1-bc65-139b463ada7c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2025-01-17T21:13:59.726188781Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1462,"session_id":"a64631d7-a959-4da1-bc65-139b463ada7c"} {"msg":"The guest has finished waiting for activation with:70","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:13:59.726214126Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1462} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 473 len 36864 data: 6 5 6 6 5 4 4 5 5 02/30 Read block 339 len 12288 03/30 Read block 320 len 28672 04/30 Write block 248 len 12288 data: 8 6 6 05/30 Write block 523 len 28672 data: 12 9 8 8 7 6 4 06/30 Write block 6 len 12288 data: 5 5 4 07/30 Read block 559 len 36864 08/30 Write block 152 len 28672 data: 5 5 6 6 5 6 5 09/30 Write block 377 len 16384 data: 12 13 12 9 10/30 Write block 379 len 28672 data: 13 10 8 6 7 7 6 11/30 Read block 339 len 28672 12/30 Read block 589 len 12288 13/30 Read block 288 len 20480 14/30 Write block 583 len 8192 data: 4 5 15/30 Read block 582 len 12288 16/30 Write block 402 len 4096 data: 7 17/30 Flush 18/30 Write block 30 len 4096 data: 6 19/30 Write block 313 len 8192 data: 5 4 20/30 Read block 70 len 40960 21/30 Read block 373 len 24576 22/30 Write block 30 len 24576 data: 7 6 9 9 8 9 23/30 Read block 391 len 16384 24/30 Read block 120 len 12288 25/30 Write block 526 len 12288 data: 9 8 7 26/30 Read block 427 len 12288 27/30 Read block 383 len 24576 28/30 Write block 404 len 12288 data: 7 7 6 29/30 Write block 89 len 8192 data: 3 5 30/30 Read block 585 len 4096 ---------------------------------------------------------------- Crucible gen:70 GIO:true work queues: Upstairs:0 downstairs:13 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1017 Acked Write 1 Done Done Done false 1018 Acked Write 2 Done Done Done false 1019 Acked Read 10 Done Done Done false 1020 Acked Read 6 Done Done Done false 1021 Acked Write 6 Done Done Done false 1022 Acked Read 4 Done Done Done false 1023 Acked Read 3 Done Done Done false 1024 Acked Write 3 Done Done Done false 1025 Acked Read 3 Done Done Done false 1026 Acked Read 6 Done Done Done false 1027 Acked Write 3 Done Done Done false 1028 Acked Write 2 Done Done Done false 1029 Acked Read 1 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 13 13 13 39 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1016 1016 1016 Downstairs last five completed: 1016 1015 1014 1013 1012 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:13:59.744752301Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1462} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 70 70 70 128 128 128 F F F 1 020-039 68 68 68 126 126 126 T T T 2 040-059 67 67 67 124 124 124 F F F 3 060-079 69 69 69 127 127 127 F F F 4 080-099 60 60 60 105 105 105 T T T 5 100-119 68 68 68 126 126 126 F F F 6 120-139 56 56 56 97 97 97 F F F 7 140-159 70 70 70 128 128 128 F F F 8 160-179 68 68 68 126 126 126 F F F 9 180-199 66 66 66 123 123 123 F F F 10 200-219 68 68 68 126 126 126 F F F 11 220-239 66 66 66 123 123 123 F F F 12 240-259 70 70 70 128 128 128 F F F 13 260-279 69 69 69 127 127 127 F F F 14 280-299 68 68 68 126 126 126 F F F 15 300-319 66 66 66 121 121 121 T T T 16 320-339 66 66 66 121 121 121 F F F 17 340-359 66 66 66 123 123 123 F F F 18 360-379 70 70 70 128 128 128 F F F 19 380-399 70 70 70 128 128 128 F F F 20 400-419 70 70 70 128 128 128 T T T 21 420-439 65 65 65 120 120 120 F F F 22 440-459 68 68 68 126 126 126 F F F 23 460-479 70 70 70 128 128 128 F F F 24 480-499 70 70 70 128 128 128 F F F 25 500-519 65 65 65 120 120 120 F F F 26 520-539 70 70 70 128 128 128 T T T 27 540-559 64 64 64 119 119 119 F F F 28 560-579 68 68 68 126 126 126 F F F 29 580-599 70 70 70 128 128 128 F F F Max gen: 70, Max flush: 128 A Difference in extent metadata is expected here On loop 35 Verifying data now No change after verify Loop: 35 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 70 70 70 128 128 128 F F F 1 020-039 71 71 71 129 129 129 F F F 2 040-059 67 67 67 124 124 124 F F F 3 060-079 69 69 69 127 127 127 F F F 4 080-099 71 71 71 129 129 129 F F F 5 100-119 68 68 68 126 126 126 F F F 6 120-139 56 56 56 97 97 97 F F F 7 140-159 70 70 70 128 128 128 F F F 8 160-179 68 68 68 126 126 126 F F F 9 180-199 66 66 66 123 123 123 F F F 10 200-219 68 68 68 126 126 126 F F F 11 220-239 66 66 66 123 123 123 F F F 12 240-259 70 70 70 128 128 128 F F F 13 260-279 69 69 69 127 127 127 F F F 14 280-299 68 68 68 126 126 126 F F F 15 300-319 71 71 71 129 129 129 F F F 16 320-339 66 66 66 121 121 121 F F F 17 340-359 66 66 66 123 123 123 F F F 18 360-379 70 70 70 128 128 128 F F F 19 380-399 70 70 70 128 128 128 F F F 20 400-419 71 71 71 129 129 129 F F F 21 420-439 65 65 65 120 120 120 F F F 22 440-459 68 68 68 126 126 126 F F F 23 460-479 70 70 70 128 128 128 F F F 24 480-499 70 70 70 128 128 128 F F F 25 500-519 65 65 65 120 120 120 F F F 26 520-539 71 71 71 129 129 129 F F F 27 540-559 64 64 64 119 119 119 F F F 28 560-579 68 68 68 126 126 126 F F F 29 580-599 70 70 70 128 128 128 F F F Max gen: 71, Max flush: 129 Begin loop 36 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:04.014259556Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1470,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:04.058663829Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1470,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:04.059300091Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1470} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"abd5430f1508af0bc008143ea2566277ef714ec2\",\n git_commit_timestamp: \"2025-01-17T20:55:03.000000000Z\",\n git_branch: \"alan/tokio-to-140\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:04.059341325Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1470} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:04.059356994Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1470} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:04.059371482Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1470} {"msg":"Crucible 3744349a-d93f-4bfe-8a8b-c1ecda749af0 has session id: ae3bfbc3-4f87-4499-b8b0-bd00a3208b40","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:04.059891916Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1470,"session_id":"ae3bfbc3-4f87-4499-b8b0-bd00a3208b40"} {"msg":"Upstairs opts: Upstairs UUID: 3744349a-d93f-4bfe-8a8b-c1ecda749af0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:04.059929699Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1470,"session_id":"ae3bfbc3-4f87-4499-b8b0-bd00a3208b40"} {"msg":"Crucible stats registered with UUID: 3744349a-d93f-4bfe-8a8b-c1ecda749af0","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:04.059958385Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1470,"session_id":"ae3bfbc3-4f87-4499-b8b0-bd00a3208b40"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:14:04.059972103Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1470,"session_id":"ae3bfbc3-4f87-4499-b8b0-bd00a3208b40"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:04.060419305Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1470,"task":"crutest"} {"msg":"The guest has requested activation with gen:72","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:04.060443679Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1470} {"msg":"3744349a-d93f-4bfe-8a8b-c1ecda749af0 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:04.060502584Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1470,"session_id":"ae3bfbc3-4f87-4499-b8b0-bd00a3208b40"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:04.060519743Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1470,"client":"0","":"downstairs","session_id":"ae3bfbc3-4f87-4499-b8b0-bd00a3208b40"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:04.060544148Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1470,"client":"0","":"downstairs","session_id":"ae3bfbc3-4f87-4499-b8b0-bd00a3208b40"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:04.060561688Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1470,"client":"1","":"downstairs","session_id":"ae3bfbc3-4f87-4499-b8b0-bd00a3208b40"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:04.060579468Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1470,"client":"1","":"downstairs","session_id":"ae3bfbc3-4f87-4499-b8b0-bd00a3208b40"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:04.060602812Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1470,"client":"2","":"downstairs","session_id":"ae3bfbc3-4f87-4499-b8b0-bd00a3208b40"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:04.060629338Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1470,"client":"2","":"downstairs","session_id":"ae3bfbc3-4f87-4499-b8b0-bd00a3208b40"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:04.060655283Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1470,"":"io task","client":"2","":"downstairs","session_id":"ae3bfbc3-4f87-4499-b8b0-bd00a3208b40"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:04.060681378Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1470,"":"io task","client":"0","":"downstairs","session_id":"ae3bfbc3-4f87-4499-b8b0-bd00a3208b40"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:04.060706163Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1470,"":"io task","client":"1","":"downstairs","session_id":"ae3bfbc3-4f87-4499-b8b0-bd00a3208b40"} {"msg":"ds_connection connected from Ok(127.0.0.1:51954)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:04.060729206Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1470,"":"io task","client":"2","":"downstairs","session_id":"ae3bfbc3-4f87-4499-b8b0-bd00a3208b40"} {"msg":"ds_connection connected from Ok(127.0.0.1:35192)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:04.06075361Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1470,"":"io task","client":"1","":"downstairs","session_id":"ae3bfbc3-4f87-4499-b8b0-bd00a3208b40"} {"msg":"ds_connection connected from Ok(127.0.0.1:46600)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:04.060781106Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1470,"":"io task","client":"0","":"downstairs","session_id":"ae3bfbc3-4f87-4499-b8b0-bd00a3208b40"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:04.061432607Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1470,"client":"1","":"downstairs","session_id":"ae3bfbc3-4f87-4499-b8b0-bd00a3208b40"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:04.061470259Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1470,"client":"0","":"downstairs","session_id":"ae3bfbc3-4f87-4499-b8b0-bd00a3208b40"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:04.062117948Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1470,"client":"1","":"downstairs","session_id":"ae3bfbc3-4f87-4499-b8b0-bd00a3208b40"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:04.062158091Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1470,"client":"0","":"downstairs","session_id":"ae3bfbc3-4f87-4499-b8b0-bd00a3208b40"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:04.062717868Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1470,"session_id":"ae3bfbc3-4f87-4499-b8b0-bd00a3208b40"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:04.062753449Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1470,"session_id":"ae3bfbc3-4f87-4499-b8b0-bd00a3208b40"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:04.062938027Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1470,"client":"2","":"downstairs","session_id":"ae3bfbc3-4f87-4499-b8b0-bd00a3208b40"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:04.063464014Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1470,"client":"2","":"downstairs","session_id":"ae3bfbc3-4f87-4499-b8b0-bd00a3208b40"} {"msg":"[0]R flush_numbers[0..12]: [128, 129, 124, 127, 129, 126, 97, 128, 126, 123, 126, 123]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:04.063948478Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1470,"":"downstairs","":"downstairs","session_id":"ae3bfbc3-4f87-4499-b8b0-bd00a3208b40"} {"msg":"[0]R generation[0..12]: [70, 71, 67, 69, 71, 68, 56, 70, 68, 66, 68, 66]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:04.063972621Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1470,"":"downstairs","":"downstairs","session_id":"ae3bfbc3-4f87-4499-b8b0-bd00a3208b40"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:04.063988311Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1470,"":"downstairs","":"downstairs","session_id":"ae3bfbc3-4f87-4499-b8b0-bd00a3208b40"} {"msg":"[1]R flush_numbers[0..12]: [128, 129, 124, 127, 129, 126, 97, 128, 126, 123, 126, 123]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:04.064002709Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1470,"":"downstairs","":"downstairs","session_id":"ae3bfbc3-4f87-4499-b8b0-bd00a3208b40"} {"msg":"[1]R generation[0..12]: [70, 71, 67, 69, 71, 68, 56, 70, 68, 66, 68, 66]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:04.064016497Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1470,"":"downstairs","":"downstairs","session_id":"ae3bfbc3-4f87-4499-b8b0-bd00a3208b40"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:04.064030265Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1470,"":"downstairs","":"downstairs","session_id":"ae3bfbc3-4f87-4499-b8b0-bd00a3208b40"} {"msg":"[2]R flush_numbers[0..12]: [128, 129, 124, 127, 129, 126, 97, 128, 126, 123, 126, 123]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:04.064043663Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1470,"":"downstairs","":"downstairs","session_id":"ae3bfbc3-4f87-4499-b8b0-bd00a3208b40"} {"msg":"[2]R generation[0..12]: [70, 71, 67, 69, 71, 68, 56, 70, 68, 66, 68, 66]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:04.064057151Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1470,"":"downstairs","":"downstairs","session_id":"ae3bfbc3-4f87-4499-b8b0-bd00a3208b40"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:04.064070809Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1470,"":"downstairs","":"downstairs","session_id":"ae3bfbc3-4f87-4499-b8b0-bd00a3208b40"} {"msg":"Max found gen is 72","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:04.064083346Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1470,"":"downstairs","":"downstairs","session_id":"ae3bfbc3-4f87-4499-b8b0-bd00a3208b40"} {"msg":"Generation requested: 72 >= found:72","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:04.064099416Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1470,"":"downstairs","":"downstairs","session_id":"ae3bfbc3-4f87-4499-b8b0-bd00a3208b40"} {"msg":"Next flush: 130","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-17T21:14:04.064112353Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1470,"":"downstairs","":"downstairs","session_id":"ae3bfbc3-4f87-4499-b8b0-bd00a3208b40"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:04.064131674Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1470,"":"downstairs","":"downstairs","session_id":"ae3bfbc3-4f87-4499-b8b0-bd00a3208b40"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:04.064144722Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1470,"session_id":"ae3bfbc3-4f87-4499-b8b0-bd00a3208b40"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:04.064156519Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1470,"client":"0","":"downstairs","session_id":"ae3bfbc3-4f87-4499-b8b0-bd00a3208b40"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:04.064169647Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1470,"client":"1","":"downstairs","session_id":"ae3bfbc3-4f87-4499-b8b0-bd00a3208b40"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:04.064182374Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1470,"client":"2","":"downstairs","session_id":"ae3bfbc3-4f87-4499-b8b0-bd00a3208b40"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:04.064195082Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1470,"session_id":"ae3bfbc3-4f87-4499-b8b0-bd00a3208b40"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:04.064207049Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1470,"session_id":"ae3bfbc3-4f87-4499-b8b0-bd00a3208b40"} {"msg":"3744349a-d93f-4bfe-8a8b-c1ecda749af0 is now active with session: ae3bfbc3-4f87-4499-b8b0-bd00a3208b40","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:04.064219566Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1470,"session_id":"ae3bfbc3-4f87-4499-b8b0-bd00a3208b40"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:04.064231293Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1470,"session_id":"ae3bfbc3-4f87-4499-b8b0-bd00a3208b40"} {"msg":"The guest has finished waiting for activation with:72","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:04.06424349Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1470} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 125 len 24576 data: 3 3 4 4 4 6 02/30 Write block 320 len 28672 data: 7 6 6 7 5 6 7 03/30 Read block 124 len 28672 04/30 Flush 05/30 Write block 35 len 8192 data: 10 6 06/30 Flush 07/30 Write block 504 len 40960 data: 9 9 7 9 7 9 8 5 5 6 08/30 Write block 138 len 40960 data: 5 6 6 5 6 6 5 4 4 6 09/30 Read block 103 len 36864 10/30 Read block 577 len 4096 11/30 Write block 90 len 28672 data: 6 5 6 6 7 7 7 12/30 Write block 46 len 40960 data: 3 3 3 4 5 8 11 11 11 9 13/30 Flush 14/30 Write block 359 len 4096 data: 10 15/30 Write block 560 len 28672 data: 7 8 9 8 6 5 7 16/30 Read block 228 len 24576 17/30 Write block 560 len 36864 data: 8 9 10 9 7 6 8 5 5 18/30 Read block 78 len 36864 19/30 Write block 50 len 4096 data: 6 20/30 Write block 199 len 28672 data: 9 7 6 6 5 4 4 21/30 Read block 11 len 12288 22/30 Read block 316 len 28672 23/30 Write block 264 len 32768 data: 5 6 6 6 6 7 8 7 24/30 Read block 26 len 28672 25/30 Flush 26/30 Write block 111 len 36864 data: 7 7 7 8 9 6 7 7 6 27/30 Read block 520 len 32768 28/30 Read block 213 len 20480 29/30 Write block 349 len 8192 data: 6 8 30/30 Read block 461 len 12288 ---------------------------------------------------------------- Crucible gen:72 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 9 Done Done Done false 1026 Acked Read 8 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Write 2 Done Sent Sent false 1029 Acked Read 3 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 2 2 4 Done 5 3 3 11 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:14:04.085079638Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1470} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 70 70 70 128 128 128 F F F 1 020-039 72 72 72 131 131 131 F F F 2 040-059 72 72 72 133 133 133 F F F 3 060-079 69 69 69 127 127 127 F F F 4 080-099 72 72 72 132 132 132 F F F 5 100-119 68 68 68 126 126 126 T T T 6 120-139 72 72 72 132 132 132 F F F 7 140-159 72 72 72 132 132 132 F F F 8 160-179 68 68 68 126 126 126 F F F 9 180-199 72 72 72 133 133 133 F F F 10 200-219 72 72 72 133 133 133 F F F 11 220-239 66 66 66 123 123 123 F F F 12 240-259 70 70 70 128 128 128 F F F 13 260-279 72 72 72 133 133 133 F F F 14 280-299 68 68 68 126 126 126 F F F 15 300-319 71 71 71 129 129 129 F F F 16 320-339 72 72 72 130 130 130 F F F 17 340-359 72 72 72 133 133 133 T T T 18 360-379 70 70 70 128 128 128 F F F 19 380-399 70 70 70 128 128 128 F F F 20 400-419 71 71 71 129 129 129 F F F 21 420-439 65 65 65 120 120 120 F F F 22 440-459 68 68 68 126 126 126 F F F 23 460-479 70 70 70 128 128 128 F F F 24 480-499 70 70 70 128 128 128 F F F 25 500-519 72 72 72 132 132 132 F F F 26 520-539 71 71 71 129 129 129 F F F 27 540-559 64 64 64 119 119 119 F F F 28 560-579 72 72 72 133 133 133 F F F 29 580-599 70 70 70 128 128 128 F F F Max gen: 72, Max flush: 133 A Difference in extent metadata is expected here On loop 36 Verifying data now No change after verify Loop: 36 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 70 70 70 128 128 128 F F F 1 020-039 72 72 72 131 131 131 F F F 2 040-059 72 72 72 133 133 133 F F F 3 060-079 69 69 69 127 127 127 F F F 4 080-099 72 72 72 132 132 132 F F F 5 100-119 73 73 73 134 134 134 F F F 6 120-139 72 72 72 132 132 132 F F F 7 140-159 72 72 72 132 132 132 F F F 8 160-179 68 68 68 126 126 126 F F F 9 180-199 72 72 72 133 133 133 F F F 10 200-219 72 72 72 133 133 133 F F F 11 220-239 66 66 66 123 123 123 F F F 12 240-259 70 70 70 128 128 128 F F F 13 260-279 72 72 72 133 133 133 F F F 14 280-299 68 68 68 126 126 126 F F F 15 300-319 71 71 71 129 129 129 F F F 16 320-339 72 72 72 130 130 130 F F F 17 340-359 73 73 73 134 134 134 F F F 18 360-379 70 70 70 128 128 128 F F F 19 380-399 70 70 70 128 128 128 F F F 20 400-419 71 71 71 129 129 129 F F F 21 420-439 65 65 65 120 120 120 F F F 22 440-459 68 68 68 126 126 126 F F F 23 460-479 70 70 70 128 128 128 F F F 24 480-499 70 70 70 128 128 128 F F F 25 500-519 72 72 72 132 132 132 F F F 26 520-539 71 71 71 129 129 129 F F F 27 540-559 64 64 64 119 119 119 F F F 28 560-579 72 72 72 133 133 133 F F F 29 580-599 70 70 70 128 128 128 F F F Max gen: 73, Max flush: 134 Begin loop 37 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:08.307055491Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1478,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:08.352328912Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1478,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:08.353132182Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1478} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"abd5430f1508af0bc008143ea2566277ef714ec2\",\n git_commit_timestamp: \"2025-01-17T20:55:03.000000000Z\",\n git_branch: \"alan/tokio-to-140\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:08.353174146Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1478} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:08.353189885Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1478} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:08.353204944Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1478} {"msg":"Crucible 2676f67b-e105-4bd1-9510-12d5b012bdbe has session id: d9e0135d-f9af-40ea-9251-2b9d17f5faca","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:08.353726889Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1478,"session_id":"d9e0135d-f9af-40ea-9251-2b9d17f5faca"} {"msg":"Upstairs opts: Upstairs UUID: 2676f67b-e105-4bd1-9510-12d5b012bdbe, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:08.353759809Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1478,"session_id":"d9e0135d-f9af-40ea-9251-2b9d17f5faca"} {"msg":"Crucible stats registered with UUID: 2676f67b-e105-4bd1-9510-12d5b012bdbe","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:08.353781862Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1478,"session_id":"d9e0135d-f9af-40ea-9251-2b9d17f5faca"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:14:08.353808797Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1478,"session_id":"d9e0135d-f9af-40ea-9251-2b9d17f5faca"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:08.354269607Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1478,"task":"crutest"} {"msg":"The guest has requested activation with gen:74","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:08.35429268Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1478} {"msg":"2676f67b-e105-4bd1-9510-12d5b012bdbe active request set","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:08.354376249Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1478,"session_id":"d9e0135d-f9af-40ea-9251-2b9d17f5faca"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:08.354393599Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1478,"client":"0","":"downstairs","session_id":"d9e0135d-f9af-40ea-9251-2b9d17f5faca"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:08.354418574Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1478,"client":"0","":"downstairs","session_id":"d9e0135d-f9af-40ea-9251-2b9d17f5faca"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:08.354432472Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1478,"client":"1","":"downstairs","session_id":"d9e0135d-f9af-40ea-9251-2b9d17f5faca"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:08.35444667Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1478,"client":"1","":"downstairs","session_id":"d9e0135d-f9af-40ea-9251-2b9d17f5faca"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:08.354460318Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1478,"client":"2","":"downstairs","session_id":"d9e0135d-f9af-40ea-9251-2b9d17f5faca"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:08.354482021Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1478,"client":"2","":"downstairs","session_id":"d9e0135d-f9af-40ea-9251-2b9d17f5faca"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:08.354504084Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1478,"":"io task","client":"2","":"downstairs","session_id":"d9e0135d-f9af-40ea-9251-2b9d17f5faca"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:08.354523835Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1478,"":"io task","client":"0","":"downstairs","session_id":"d9e0135d-f9af-40ea-9251-2b9d17f5faca"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:08.354547449Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1478,"":"io task","client":"1","":"downstairs","session_id":"d9e0135d-f9af-40ea-9251-2b9d17f5faca"} {"msg":"ds_connection connected from Ok(127.0.0.1:35786)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:08.354587272Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1478,"":"io task","client":"2","":"downstairs","session_id":"d9e0135d-f9af-40ea-9251-2b9d17f5faca"} {"msg":"ds_connection connected from Ok(127.0.0.1:41862)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:08.354612427Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1478,"":"io task","client":"0","":"downstairs","session_id":"d9e0135d-f9af-40ea-9251-2b9d17f5faca"} {"msg":"ds_connection connected from Ok(127.0.0.1:62022)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:08.354636791Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1478,"":"io task","client":"1","":"downstairs","session_id":"d9e0135d-f9af-40ea-9251-2b9d17f5faca"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:08.355342914Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1478,"client":"0","":"downstairs","session_id":"d9e0135d-f9af-40ea-9251-2b9d17f5faca"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:08.355382517Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1478,"client":"2","":"downstairs","session_id":"d9e0135d-f9af-40ea-9251-2b9d17f5faca"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:08.355563883Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1478,"client":"1","":"downstairs","session_id":"d9e0135d-f9af-40ea-9251-2b9d17f5faca"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:08.356126502Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1478,"client":"2","":"downstairs","session_id":"d9e0135d-f9af-40ea-9251-2b9d17f5faca"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:08.356165505Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1478,"client":"0","":"downstairs","session_id":"d9e0135d-f9af-40ea-9251-2b9d17f5faca"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:08.356197543Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1478,"client":"1","":"downstairs","session_id":"d9e0135d-f9af-40ea-9251-2b9d17f5faca"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:08.356689331Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1478,"session_id":"d9e0135d-f9af-40ea-9251-2b9d17f5faca"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:08.356795563Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1478,"session_id":"d9e0135d-f9af-40ea-9251-2b9d17f5faca"} {"msg":"[0]R flush_numbers[0..12]: [128, 131, 133, 127, 132, 134, 132, 132, 126, 133, 133, 123]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:08.356860441Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1478,"":"downstairs","":"downstairs","session_id":"d9e0135d-f9af-40ea-9251-2b9d17f5faca"} {"msg":"[0]R generation[0..12]: [70, 72, 72, 69, 72, 73, 72, 72, 68, 72, 72, 66]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:08.356888007Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1478,"":"downstairs","":"downstairs","session_id":"d9e0135d-f9af-40ea-9251-2b9d17f5faca"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:08.356904236Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1478,"":"downstairs","":"downstairs","session_id":"d9e0135d-f9af-40ea-9251-2b9d17f5faca"} {"msg":"[1]R flush_numbers[0..12]: [128, 131, 133, 127, 132, 134, 132, 132, 126, 133, 133, 123]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:08.356918955Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1478,"":"downstairs","":"downstairs","session_id":"d9e0135d-f9af-40ea-9251-2b9d17f5faca"} {"msg":"[1]R generation[0..12]: [70, 72, 72, 69, 72, 73, 72, 72, 68, 72, 72, 66]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:08.356932733Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1478,"":"downstairs","":"downstairs","session_id":"d9e0135d-f9af-40ea-9251-2b9d17f5faca"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:08.356946481Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1478,"":"downstairs","":"downstairs","session_id":"d9e0135d-f9af-40ea-9251-2b9d17f5faca"} {"msg":"[2]R flush_numbers[0..12]: [128, 131, 133, 127, 132, 134, 132, 132, 126, 133, 133, 123]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:08.356962871Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1478,"":"downstairs","":"downstairs","session_id":"d9e0135d-f9af-40ea-9251-2b9d17f5faca"} {"msg":"[2]R generation[0..12]: [70, 72, 72, 69, 72, 73, 72, 72, 68, 72, 72, 66]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:08.356976649Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1478,"":"downstairs","":"downstairs","session_id":"d9e0135d-f9af-40ea-9251-2b9d17f5faca"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:08.356990287Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1478,"":"downstairs","":"downstairs","session_id":"d9e0135d-f9af-40ea-9251-2b9d17f5faca"} {"msg":"Max found gen is 74","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:08.357003014Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1478,"":"downstairs","":"downstairs","session_id":"d9e0135d-f9af-40ea-9251-2b9d17f5faca"} {"msg":"Generation requested: 74 >= found:74","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:08.357015832Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1478,"":"downstairs","":"downstairs","session_id":"d9e0135d-f9af-40ea-9251-2b9d17f5faca"} {"msg":"Next flush: 135","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:08.357028429Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1478,"":"downstairs","":"downstairs","session_id":"d9e0135d-f9af-40ea-9251-2b9d17f5faca"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:08.357040936Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1478,"":"downstairs","":"downstairs","session_id":"d9e0135d-f9af-40ea-9251-2b9d17f5faca"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-17T21:14:08.357053674Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1478,"session_id":"d9e0135d-f9af-40ea-9251-2b9d17f5faca"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:08.357071014Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1478,"client":"0","":"downstairs","session_id":"d9e0135d-f9af-40ea-9251-2b9d17f5faca"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:08.357084132Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1478,"client":"1","":"downstairs","session_id":"d9e0135d-f9af-40ea-9251-2b9d17f5faca"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:08.357096949Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1478,"client":"2","":"downstairs","session_id":"d9e0135d-f9af-40ea-9251-2b9d17f5faca"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:08.357109767Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1478,"session_id":"d9e0135d-f9af-40ea-9251-2b9d17f5faca"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:08.357121654Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1478,"session_id":"d9e0135d-f9af-40ea-9251-2b9d17f5faca"} {"msg":"2676f67b-e105-4bd1-9510-12d5b012bdbe is now active with session: d9e0135d-f9af-40ea-9251-2b9d17f5faca","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:08.357134221Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1478,"session_id":"d9e0135d-f9af-40ea-9251-2b9d17f5faca"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:08.357145848Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1478,"session_id":"d9e0135d-f9af-40ea-9251-2b9d17f5faca"} {"msg":"The guest has finished waiting for activation with:74","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:08.357159476Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1478} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 199 len 16384 data: 10 8 7 7 03/30 Read block 558 len 8192 04/30 Read block 296 len 16384 05/30 Read block 290 len 4096 06/30 Write block 505 len 28672 data: 10 8 10 8 10 9 6 07/30 Read block 345 len 40960 08/30 Write block 523 len 8192 data: 13 10 09/30 Read block 238 len 40960 10/30 Write block 566 len 32768 data: 9 6 6 4 4 3 2 2 11/30 Flush 12/30 Write block 511 len 32768 data: 7 6 7 8 11 11 8 9 13/30 Write block 371 len 20480 data: 7 10 9 10 10 14/30 Write block 546 len 24576 data: 7 7 7 6 7 6 15/30 Write block 471 len 24576 data: 7 8 7 6 7 7 16/30 Write block 108 len 28672 data: 5 6 6 8 8 8 9 17/30 Read block 191 len 32768 18/30 Read block 106 len 12288 19/30 Write block 334 len 36864 data: 8 9 10 9 9 9 9 7 7 20/30 Read block 450 len 16384 21/30 Read block 299 len 32768 22/30 Write block 253 len 36864 data: 3 2 4 6 7 7 7 6 6 23/30 Write block 374 len 12288 data: 11 11 10 24/30 Write block 490 len 24576 data: 6 6 5 6 7 8 25/30 Read block 102 len 36864 26/30 Read block 337 len 36864 27/30 Write block 252 len 20480 data: 5 4 3 5 7 28/30 Write block 301 len 8192 data: 6 4 29/30 Read block 43 len 16384 30/30 Read block 460 len 16384 ---------------------------------------------------------------- Crucible gen:74 GIO:true work queues: Upstairs:0 downstairs:19 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1011 Acked Write 8 Done Done Done false 1012 Acked Write 5 Done Done Done false 1013 Acked Write 6 Done Done Done false 1014 Acked Write 6 Done Done Done false 1015 Acked Write 7 Done Done Done false 1016 Acked Read 8 Done Done Done false 1017 Acked Read 3 Done Done Done false 1018 Acked Write 9 Done Done Done false 1019 Acked Read 4 Done Done Done false 1020 Acked Read 8 Done Done Done false 1021 Acked Write 9 Done Done Done false 1022 Acked Write 3 Done Done Done false 1023 Acked Write 6 Done Done Done false 1024 Acked Read 9 Done Done Done false 1025 Acked Read 9 Done Done Done false 1026 Acked Write 5 Done Done Done false 1027 Acked Write 2 Done Done Done false 1028 Acked Read 4 Done Done Done false 1029 Acked Read 4 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 19 18 18 55 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1010 1010 1010 Downstairs last five completed: 1010 1009 1008 1007 1006 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:14:08.377208994Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1478} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 70 70 70 128 128 128 F F F 1 020-039 72 72 72 131 131 131 F F F 2 040-059 72 72 72 133 133 133 F F F 3 060-079 69 69 69 127 127 127 F F F 4 080-099 72 72 72 132 132 132 F F F 5 100-119 73 73 73 134 134 134 T T T 6 120-139 72 72 72 132 132 132 F F F 7 140-159 72 72 72 132 132 132 F F F 8 160-179 68 68 68 126 126 126 F F F 9 180-199 74 74 74 136 136 136 F F F 10 200-219 74 74 74 136 136 136 F F F 11 220-239 66 66 66 123 123 123 F F F 12 240-259 70 70 70 128 128 128 T T T 13 260-279 72 72 72 133 133 133 T T T 14 280-299 68 68 68 126 126 126 F F F 15 300-319 71 71 71 129 129 129 T T T 16 320-339 72 72 72 130 130 130 T T T 17 340-359 73 73 73 134 134 134 T T T 18 360-379 70 70 70 128 128 128 T T T 19 380-399 70 70 70 128 128 128 F F F 20 400-419 71 71 71 129 129 129 F F F 21 420-439 65 65 65 120 120 120 F F F 22 440-459 68 68 68 126 126 126 F F F 23 460-479 70 70 70 128 128 128 T T T 24 480-499 70 70 70 128 128 128 T T T 25 500-519 74 74 74 136 136 136 T T T 26 520-539 74 74 74 136 136 136 F F F 27 540-559 64 64 64 119 119 119 T T T 28 560-579 74 74 74 136 136 136 F F F 29 580-599 70 70 70 128 128 128 F F F Max gen: 74, Max flush: 136 A Difference in extent metadata is expected here On loop 37 Verifying data now No change after verify Loop: 37 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 70 70 70 128 128 128 F F F 1 020-039 72 72 72 131 131 131 F F F 2 040-059 72 72 72 133 133 133 F F F 3 060-079 69 69 69 127 127 127 F F F 4 080-099 72 72 72 132 132 132 F F F 5 100-119 75 75 75 137 137 137 F F F 6 120-139 72 72 72 132 132 132 F F F 7 140-159 72 72 72 132 132 132 F F F 8 160-179 68 68 68 126 126 126 F F F 9 180-199 74 74 74 136 136 136 F F F 10 200-219 74 74 74 136 136 136 F F F 11 220-239 66 66 66 123 123 123 F F F 12 240-259 75 75 75 137 137 137 F F F 13 260-279 75 75 75 137 137 137 F F F 14 280-299 68 68 68 126 126 126 F F F 15 300-319 75 75 75 137 137 137 F F F 16 320-339 75 75 75 137 137 137 F F F 17 340-359 75 75 75 137 137 137 F F F 18 360-379 75 75 75 137 137 137 F F F 19 380-399 70 70 70 128 128 128 F F F 20 400-419 71 71 71 129 129 129 F F F 21 420-439 65 65 65 120 120 120 F F F 22 440-459 68 68 68 126 126 126 F F F 23 460-479 75 75 75 137 137 137 F F F 24 480-499 75 75 75 137 137 137 F F F 25 500-519 75 75 75 137 137 137 F F F 26 520-539 74 74 74 136 136 136 F F F 27 540-559 75 75 75 137 137 137 F F F 28 560-579 74 74 74 136 136 136 F F F 29 580-599 70 70 70 128 128 128 F F F Max gen: 75, Max flush: 137 Begin loop 38 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:12.649865708Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1486,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:12.694044709Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1486,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:12.694762369Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1486} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"abd5430f1508af0bc008143ea2566277ef714ec2\",\n git_commit_timestamp: \"2025-01-17T20:55:03.000000000Z\",\n git_branch: \"alan/tokio-to-140\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:12.694812958Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1486} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:12.694834821Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1486} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:12.694853142Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1486} {"msg":"Crucible 359b9c52-dec4-4238-b8db-11591d780e66 has session id: c138c2ef-ef36-4a5c-a52d-7e9fcf2bd55d","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:12.695471053Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1486,"session_id":"c138c2ef-ef36-4a5c-a52d-7e9fcf2bd55d"} {"msg":"Upstairs opts: Upstairs UUID: 359b9c52-dec4-4238-b8db-11591d780e66, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:12.695511407Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1486,"session_id":"c138c2ef-ef36-4a5c-a52d-7e9fcf2bd55d"} {"msg":"Crucible stats registered with UUID: 359b9c52-dec4-4238-b8db-11591d780e66","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:12.69553447Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1486,"session_id":"c138c2ef-ef36-4a5c-a52d-7e9fcf2bd55d"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:14:12.695554402Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1486,"session_id":"c138c2ef-ef36-4a5c-a52d-7e9fcf2bd55d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:12.696040606Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1486,"task":"crutest"} {"msg":"The guest has requested activation with gen:76","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:12.696074736Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1486} {"msg":"359b9c52-dec4-4238-b8db-11591d780e66 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:12.696099741Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1486,"session_id":"c138c2ef-ef36-4a5c-a52d-7e9fcf2bd55d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:12.696122224Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1486,"client":"0","":"downstairs","session_id":"c138c2ef-ef36-4a5c-a52d-7e9fcf2bd55d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:12.696154513Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1486,"client":"0","":"downstairs","session_id":"c138c2ef-ef36-4a5c-a52d-7e9fcf2bd55d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:12.696175795Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1486,"client":"1","":"downstairs","session_id":"c138c2ef-ef36-4a5c-a52d-7e9fcf2bd55d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:12.696198678Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1486,"client":"1","":"downstairs","session_id":"c138c2ef-ef36-4a5c-a52d-7e9fcf2bd55d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:12.696226064Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1486,"client":"2","":"downstairs","session_id":"c138c2ef-ef36-4a5c-a52d-7e9fcf2bd55d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:12.69625219Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1486,"client":"2","":"downstairs","session_id":"c138c2ef-ef36-4a5c-a52d-7e9fcf2bd55d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:12.696277044Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1486,"":"io task","client":"2","":"downstairs","session_id":"c138c2ef-ef36-4a5c-a52d-7e9fcf2bd55d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:12.696301068Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1486,"":"io task","client":"0","":"downstairs","session_id":"c138c2ef-ef36-4a5c-a52d-7e9fcf2bd55d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:12.696326443Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1486,"":"io task","client":"1","":"downstairs","session_id":"c138c2ef-ef36-4a5c-a52d-7e9fcf2bd55d"} {"msg":"ds_connection connected from Ok(127.0.0.1:49967)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:12.696564072Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1486,"":"io task","client":"0","":"downstairs","session_id":"c138c2ef-ef36-4a5c-a52d-7e9fcf2bd55d"} {"msg":"ds_connection connected from Ok(127.0.0.1:34229)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:12.696582132Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1486,"":"io task","client":"2","":"downstairs","session_id":"c138c2ef-ef36-4a5c-a52d-7e9fcf2bd55d"} {"msg":"ds_connection connected from Ok(127.0.0.1:59498)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:12.696596971Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1486,"":"io task","client":"1","":"downstairs","session_id":"c138c2ef-ef36-4a5c-a52d-7e9fcf2bd55d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:12.697149774Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1486,"client":"2","":"downstairs","session_id":"c138c2ef-ef36-4a5c-a52d-7e9fcf2bd55d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:12.697209079Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1486,"client":"0","":"downstairs","session_id":"c138c2ef-ef36-4a5c-a52d-7e9fcf2bd55d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:12.697884884Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1486,"client":"0","":"downstairs","session_id":"c138c2ef-ef36-4a5c-a52d-7e9fcf2bd55d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:12.697918213Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1486,"client":"1","":"downstairs","session_id":"c138c2ef-ef36-4a5c-a52d-7e9fcf2bd55d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:12.697944689Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1486,"client":"2","":"downstairs","session_id":"c138c2ef-ef36-4a5c-a52d-7e9fcf2bd55d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:12.698481413Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1486,"session_id":"c138c2ef-ef36-4a5c-a52d-7e9fcf2bd55d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:12.698507928Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1486,"client":"1","":"downstairs","session_id":"c138c2ef-ef36-4a5c-a52d-7e9fcf2bd55d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:12.698526789Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1486,"session_id":"c138c2ef-ef36-4a5c-a52d-7e9fcf2bd55d"} {"msg":"[0]R flush_numbers[0..12]: [128, 131, 133, 127, 132, 137, 132, 132, 126, 136, 136, 123]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:12.69905891Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1486,"":"downstairs","":"downstairs","session_id":"c138c2ef-ef36-4a5c-a52d-7e9fcf2bd55d"} {"msg":"[0]R generation[0..12]: [70, 72, 72, 69, 72, 75, 72, 72, 68, 74, 74, 66]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:12.699082264Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1486,"":"downstairs","":"downstairs","session_id":"c138c2ef-ef36-4a5c-a52d-7e9fcf2bd55d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:12.699096962Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1486,"":"downstairs","":"downstairs","session_id":"c138c2ef-ef36-4a5c-a52d-7e9fcf2bd55d"} {"msg":"[1]R flush_numbers[0..12]: [128, 131, 133, 127, 132, 137, 132, 132, 126, 136, 136, 123]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:12.69911077Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1486,"":"downstairs","":"downstairs","session_id":"c138c2ef-ef36-4a5c-a52d-7e9fcf2bd55d"} {"msg":"[1]R generation[0..12]: [70, 72, 72, 69, 72, 75, 72, 72, 68, 74, 74, 66]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:12.699124258Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1486,"":"downstairs","":"downstairs","session_id":"c138c2ef-ef36-4a5c-a52d-7e9fcf2bd55d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:12.699138026Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1486,"":"downstairs","":"downstairs","session_id":"c138c2ef-ef36-4a5c-a52d-7e9fcf2bd55d"} {"msg":"[2]R flush_numbers[0..12]: [128, 131, 133, 127, 132, 137, 132, 132, 126, 136, 136, 123]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:12.699151244Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1486,"":"downstairs","":"downstairs","session_id":"c138c2ef-ef36-4a5c-a52d-7e9fcf2bd55d"} {"msg":"[2]R generation[0..12]: [70, 72, 72, 69, 72, 75, 72, 72, 68, 74, 74, 66]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:12.699164262Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1486,"":"downstairs","":"downstairs","session_id":"c138c2ef-ef36-4a5c-a52d-7e9fcf2bd55d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:12.69917874Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1486,"":"downstairs","":"downstairs","session_id":"c138c2ef-ef36-4a5c-a52d-7e9fcf2bd55d"} {"msg":"Max found gen is 76","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:12.699191508Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1486,"":"downstairs","":"downstairs","session_id":"c138c2ef-ef36-4a5c-a52d-7e9fcf2bd55d"} {"msg":"Generation requested: 76 >= found:76","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:12.699205165Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1486,"":"downstairs","":"downstairs","session_id":"c138c2ef-ef36-4a5c-a52d-7e9fcf2bd55d"} Wait for a query_work_queue command to finish before sending IO {"msg":"Next flush: 138","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:12.699223546Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1486,"":"downstairs","":"downstairs","session_id":"c138c2ef-ef36-4a5c-a52d-7e9fcf2bd55d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:12.699236144Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1486,"":"downstairs","":"downstairs","session_id":"c138c2ef-ef36-4a5c-a52d-7e9fcf2bd55d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:12.699251663Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1486,"session_id":"c138c2ef-ef36-4a5c-a52d-7e9fcf2bd55d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:12.69926347Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1486,"client":"0","":"downstairs","session_id":"c138c2ef-ef36-4a5c-a52d-7e9fcf2bd55d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:12.699276738Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1486,"client":"1","":"downstairs","session_id":"c138c2ef-ef36-4a5c-a52d-7e9fcf2bd55d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:12.699289755Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1486,"client":"2","":"downstairs","session_id":"c138c2ef-ef36-4a5c-a52d-7e9fcf2bd55d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:12.699302622Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1486,"session_id":"c138c2ef-ef36-4a5c-a52d-7e9fcf2bd55d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:12.699314529Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1486,"session_id":"c138c2ef-ef36-4a5c-a52d-7e9fcf2bd55d"} {"msg":"359b9c52-dec4-4238-b8db-11591d780e66 is now active with session: c138c2ef-ef36-4a5c-a52d-7e9fcf2bd55d","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:12.699326987Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1486,"session_id":"c138c2ef-ef36-4a5c-a52d-7e9fcf2bd55d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:12.699338434Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1486,"session_id":"c138c2ef-ef36-4a5c-a52d-7e9fcf2bd55d"} {"msg":"The guest has finished waiting for activation with:76","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:12.69935015Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1486} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 233 len 24576 data: 7 8 9 8 7 5 02/30 Read block 364 len 4096 03/30 Write block 548 len 8192 data: 8 7 04/30 Read block 331 len 4096 05/30 Write block 239 len 8192 data: 5 6 06/30 Write block 434 len 20480 data: 5 4 3 4 3 07/30 Read block 229 len 28672 08/30 Read block 594 len 24576 09/30 Write block 488 len 8192 data: 4 6 10/30 Read block 147 len 32768 11/30 Read block 484 len 36864 12/30 Read block 511 len 36864 13/30 Write block 64 len 12288 data: 4 4 4 14/30 Write block 354 len 12288 data: 7 7 8 15/30 Write block 40 len 36864 data: 5 5 4 3 3 3 4 4 4 16/30 Flush 17/30 Write block 150 len 40960 data: 6 6 6 6 7 7 6 7 6 6 18/30 Read block 520 len 8192 19/30 Write block 155 len 32768 data: 8 7 8 7 7 6 7 7 20/30 Write block 156 len 40960 data: 8 9 8 8 7 8 8 6 6 7 21/30 Flush 22/30 Write block 44 len 20480 data: 4 4 5 5 5 23/30 Write block 423 len 20480 data: 5 5 5 7 8 24/30 Write block 92 len 20480 data: 7 7 8 8 8 25/30 Read block 382 len 12288 26/30 Write block 508 len 8192 data: 9 11 27/30 Write block 176 len 16384 data: 7 7 5 6 28/30 Write block 30 len 36864 data: 8 7 10 10 9 11 7 7 7 29/30 Read block 4 len 32768 30/30 Write block 495 len 40960 data: 9 9 9 10 10 11 11 9 9 10 ---------------------------------------------------------------- Crucible gen:76 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 5 Done Done Done false 1022 Acked Write 5 Done Done Done false 1023 Acked Write 5 Done Done Done false 1024 Acked Read 3 Done Done Done false 1025 Acked Write 2 Done Done Done false 1026 Acked Write 4 Done Done Done false 1027 Acked Write 9 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 8 8 8 24 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1020 1020 1020 Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:14:12.718594718Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1486} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 70 70 70 128 128 128 F F F 1 020-039 72 72 72 131 131 131 T T T 2 040-059 76 76 76 138 138 138 T T T 3 060-079 76 76 76 138 138 138 F F F 4 080-099 72 72 72 132 132 132 T T T 5 100-119 75 75 75 137 137 137 F F F 6 120-139 72 72 72 132 132 132 F F F 7 140-159 76 76 76 139 139 139 F F F 8 160-179 76 76 76 139 139 139 T T T 9 180-199 74 74 74 136 136 136 F F F 10 200-219 74 74 74 136 136 136 F F F 11 220-239 76 76 76 138 138 138 F F F 12 240-259 76 76 76 138 138 138 F F F 13 260-279 75 75 75 137 137 137 F F F 14 280-299 68 68 68 126 126 126 F F F 15 300-319 75 75 75 137 137 137 F F F 16 320-339 75 75 75 137 137 137 F F F 17 340-359 76 76 76 138 138 138 F F F 18 360-379 75 75 75 137 137 137 F F F 19 380-399 70 70 70 128 128 128 F F F 20 400-419 71 71 71 129 129 129 F F F 21 420-439 76 76 76 138 138 138 T T T 22 440-459 68 68 68 126 126 126 F F F 23 460-479 75 75 75 137 137 137 F F F 24 480-499 76 76 76 138 138 138 T T T 25 500-519 75 75 75 137 137 137 T T T 26 520-539 74 74 74 136 136 136 F F F 27 540-559 76 76 76 138 138 138 F F F 28 560-579 74 74 74 136 136 136 F F F 29 580-599 70 70 70 128 128 128 F F F Max gen: 76, Max flush: 139 A Difference in extent metadata is expected here On loop 38 Verifying data now No change after verify Loop: 38 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 70 70 70 128 128 128 F F F 1 020-039 77 77 77 140 140 140 F F F 2 040-059 77 77 77 140 140 140 F F F 3 060-079 76 76 76 138 138 138 F F F 4 080-099 77 77 77 140 140 140 F F F 5 100-119 75 75 75 137 137 137 F F F 6 120-139 72 72 72 132 132 132 F F F 7 140-159 76 76 76 139 139 139 F F F 8 160-179 77 77 77 140 140 140 F F F 9 180-199 74 74 74 136 136 136 F F F 10 200-219 74 74 74 136 136 136 F F F 11 220-239 76 76 76 138 138 138 F F F 12 240-259 76 76 76 138 138 138 F F F 13 260-279 75 75 75 137 137 137 F F F 14 280-299 68 68 68 126 126 126 F F F 15 300-319 75 75 75 137 137 137 F F F 16 320-339 75 75 75 137 137 137 F F F 17 340-359 76 76 76 138 138 138 F F F 18 360-379 75 75 75 137 137 137 F F F 19 380-399 70 70 70 128 128 128 F F F 20 400-419 71 71 71 129 129 129 F F F 21 420-439 77 77 77 140 140 140 F F F 22 440-459 68 68 68 126 126 126 F F F 23 460-479 75 75 75 137 137 137 F F F 24 480-499 77 77 77 140 140 140 F F F 25 500-519 77 77 77 140 140 140 F F F 26 520-539 74 74 74 136 136 136 F F F 27 540-559 76 76 76 138 138 138 F F F 28 560-579 74 74 74 136 136 136 F F F 29 580-599 70 70 70 128 128 128 F F F Max gen: 77, Max flush: 140 Begin loop 39 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:16.973423572Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1494,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:17.018679832Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1494,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:17.019430341Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1494} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"abd5430f1508af0bc008143ea2566277ef714ec2\",\n git_commit_timestamp: \"2025-01-17T20:55:03.000000000Z\",\n git_branch: \"alan/tokio-to-140\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:17.019483692Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1494} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:17.019505245Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1494} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:17.01953026Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1494} {"msg":"Crucible 4946fdac-0494-4e95-977c-6d7422c88bff has session id: 8dd832c9-acb0-433b-8125-272d699b344f","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:17.0200438Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1494,"session_id":"8dd832c9-acb0-433b-8125-272d699b344f"} {"msg":"Upstairs opts: Upstairs UUID: 4946fdac-0494-4e95-977c-6d7422c88bff, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:17.020084364Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1494,"session_id":"8dd832c9-acb0-433b-8125-272d699b344f"} {"msg":"Crucible stats registered with UUID: 4946fdac-0494-4e95-977c-6d7422c88bff","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:17.020109328Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1494,"session_id":"8dd832c9-acb0-433b-8125-272d699b344f"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:14:17.020131411Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1494,"session_id":"8dd832c9-acb0-433b-8125-272d699b344f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:17.02057421Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1494,"task":"crutest"} {"msg":"The guest has requested activation with gen:78","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:17.020599845Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1494} {"msg":"4946fdac-0494-4e95-977c-6d7422c88bff active request set","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:17.020661651Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1494,"session_id":"8dd832c9-acb0-433b-8125-272d699b344f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:17.020678901Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1494,"client":"0","":"downstairs","session_id":"8dd832c9-acb0-433b-8125-272d699b344f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:17.020704126Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1494,"client":"0","":"downstairs","session_id":"8dd832c9-acb0-433b-8125-272d699b344f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:17.020718234Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1494,"client":"1","":"downstairs","session_id":"8dd832c9-acb0-433b-8125-272d699b344f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:17.020732192Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1494,"client":"1","":"downstairs","session_id":"8dd832c9-acb0-433b-8125-272d699b344f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:17.020752024Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1494,"client":"2","":"downstairs","session_id":"8dd832c9-acb0-433b-8125-272d699b344f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:17.020777879Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1494,"client":"2","":"downstairs","session_id":"8dd832c9-acb0-433b-8125-272d699b344f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:17.02079635Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1494,"":"io task","client":"2","":"downstairs","session_id":"8dd832c9-acb0-433b-8125-272d699b344f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:17.020811128Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1494,"":"io task","client":"0","":"downstairs","session_id":"8dd832c9-acb0-433b-8125-272d699b344f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:17.020832421Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1494,"":"io task","client":"1","":"downstairs","session_id":"8dd832c9-acb0-433b-8125-272d699b344f"} {"msg":"ds_connection connected from Ok(127.0.0.1:60121)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:17.020856855Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1494,"":"io task","client":"1","":"downstairs","session_id":"8dd832c9-acb0-433b-8125-272d699b344f"} {"msg":"ds_connection connected from Ok(127.0.0.1:33463)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:17.02088276Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1494,"":"io task","client":"2","":"downstairs","session_id":"8dd832c9-acb0-433b-8125-272d699b344f"} {"msg":"ds_connection connected from Ok(127.0.0.1:37615)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:17.020907815Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1494,"":"io task","client":"0","":"downstairs","session_id":"8dd832c9-acb0-433b-8125-272d699b344f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:17.021619831Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1494,"client":"1","":"downstairs","session_id":"8dd832c9-acb0-433b-8125-272d699b344f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:17.021660665Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1494,"client":"0","":"downstairs","session_id":"8dd832c9-acb0-433b-8125-272d699b344f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:17.021689131Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1494,"client":"2","":"downstairs","session_id":"8dd832c9-acb0-433b-8125-272d699b344f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:17.02221643Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1494,"client":"0","":"downstairs","session_id":"8dd832c9-acb0-433b-8125-272d699b344f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:17.022327404Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1494,"client":"1","":"downstairs","session_id":"8dd832c9-acb0-433b-8125-272d699b344f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:17.022347296Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1494,"client":"2","":"downstairs","session_id":"8dd832c9-acb0-433b-8125-272d699b344f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:17.022721255Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1494,"session_id":"8dd832c9-acb0-433b-8125-272d699b344f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:17.022842035Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1494,"session_id":"8dd832c9-acb0-433b-8125-272d699b344f"} {"msg":"[0]R flush_numbers[0..12]: [128, 140, 140, 138, 140, 137, 132, 139, 140, 136, 136, 138]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:17.022972211Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1494,"":"downstairs","":"downstairs","session_id":"8dd832c9-acb0-433b-8125-272d699b344f"} {"msg":"[0]R generation[0..12]: [70, 77, 77, 76, 77, 75, 72, 76, 77, 74, 74, 76]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:17.022991272Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1494,"":"downstairs","":"downstairs","session_id":"8dd832c9-acb0-433b-8125-272d699b344f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:17.023005661Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1494,"":"downstairs","":"downstairs","session_id":"8dd832c9-acb0-433b-8125-272d699b344f"} {"msg":"[1]R flush_numbers[0..12]: [128, 140, 140, 138, 140, 137, 132, 139, 140, 136, 136, 138]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:17.023019789Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1494,"":"downstairs","":"downstairs","session_id":"8dd832c9-acb0-433b-8125-272d699b344f"} {"msg":"[1]R generation[0..12]: [70, 77, 77, 76, 77, 75, 72, 76, 77, 74, 74, 76]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:17.023033167Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1494,"":"downstairs","":"downstairs","session_id":"8dd832c9-acb0-433b-8125-272d699b344f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:17.023046795Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1494,"":"downstairs","":"downstairs","session_id":"8dd832c9-acb0-433b-8125-272d699b344f"} {"msg":"[2]R flush_numbers[0..12]: [128, 140, 140, 138, 140, 137, 132, 139, 140, 136, 136, 138]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:17.023060253Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1494,"":"downstairs","":"downstairs","session_id":"8dd832c9-acb0-433b-8125-272d699b344f"} {"msg":"[2]R generation[0..12]: [70, 77, 77, 76, 77, 75, 72, 76, 77, 74, 74, 76]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:17.023076702Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1494,"":"downstairs","":"downstairs","session_id":"8dd832c9-acb0-433b-8125-272d699b344f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:17.023090601Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1494,"":"downstairs","":"downstairs","session_id":"8dd832c9-acb0-433b-8125-272d699b344f"} {"msg":"Max found gen is 78","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:17.023103428Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1494,"":"downstairs","":"downstairs","session_id":"8dd832c9-acb0-433b-8125-272d699b344f"} {"msg":"Generation requested: 78 >= found:78","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:17.023116296Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1494,"":"downstairs","":"downstairs","session_id":"8dd832c9-acb0-433b-8125-272d699b344f"} {"msg":"Next flush: 141","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:17.023128773Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1494,"":"downstairs",""Wait for a query_work_queue command to finish before sending IO :"downstairs","session_id":"8dd832c9-acb0-433b-8125-272d699b344f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:17.023148164Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1494,"":"downstairs","":"downstairs","session_id":"8dd832c9-acb0-433b-8125-272d699b344f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:17.023161062Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1494,"session_id":"8dd832c9-acb0-433b-8125-272d699b344f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:17.023172869Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1494,"client":"0","":"downstairs","session_id":"8dd832c9-acb0-433b-8125-272d699b344f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:17.023185656Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1494,"client":"1","":"downstairs","session_id":"8dd832c9-acb0-433b-8125-272d699b344f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:17.023198364Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1494,"client":"2","":"downstairs","session_id":"8dd832c9-acb0-433b-8125-272d699b344f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:17.023211171Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1494,"session_id":"8dd832c9-acb0-433b-8125-272d699b344f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:17.023223148Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1494,"session_id":"8dd832c9-acb0-433b-8125-272d699b344f"} {"msg":"4946fdac-0494-4e95-977c-6d7422c88bff is now active with session: 8dd832c9-acb0-433b-8125-272d699b344f","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:17.023235505Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1494,"session_id":"8dd832c9-acb0-433b-8125-272d699b344f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:17.023247082Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1494,"session_id":"8dd832c9-acb0-433b-8125-272d699b344f"} {"msg":"The guest has finished waiting for activation with:78","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:17.023268534Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1494} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 1 len 12288 data: 4 3 3 02/30 Read block 70 len 24576 03/30 Read block 517 len 24576 04/30 Write block 200 len 4096 data: 9 05/30 Write block 34 len 8192 data: 10 12 06/30 Write block 413 len 8192 data: 6 6 07/30 Read block 0 len 12288 08/30 Write block 66 len 24576 data: 5 5 6 6 6 8 09/30 Flush 10/30 Write block 119 len 8192 data: 7 6 11/30 Read block 116 len 4096 12/30 Write block 486 len 24576 data: 4 4 5 7 7 7 13/30 Flush 14/30 Write block 175 len 12288 data: 9 8 8 15/30 Write block 118 len 8192 data: 8 8 16/30 Write block 254 len 16384 data: 4 6 8 8 17/30 Write block 324 len 32768 data: 6 7 8 6 5 5 6 6 18/30 Read block 184 len 4096 19/30 Read block 473 len 32768 20/30 Read block 596 len 4096 21/30 Read block 67 len 24576 22/30 Write block 291 len 32768 data: 7 8 7 6 8 8 7 6 23/30 Read block 563 len 16384 24/30 Read block 10 len 4096 25/30 Read block 227 len 20480 26/30 Read block 545 len 36864 27/30 Read block 337 len 28672 28/30 Write block 579 len 40960 data: 6 6 5 4 5 6 5 5 6 6 29/30 Read block 77 len 32768 30/30 Read block 103 len 16384 ---------------------------------------------------------------- Crucible gen:78 GIO:true work queues: Upstairs:0 downstairs:17 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1013 Acked Write 3 Done Done Done false 1014 Acked Write 2 Done Done Done false 1015 Acked Write 4 Done Done Done false 1016 Acked Write 8 Done Done Done false 1017 Acked Read 1 Done Done Done false 1018 Acked Read 8 Done Done Done false 1019 Acked Read 1 Done Done Done false 1020 Acked Read 6 Done Done Done false 1021 Acked Write 8 Done Done Done false 1022 Acked Read 4 Done Done Done false 1023 Acked Read 1 Done Done Done false 1024 Acked Read 5 Done Done Done false 1025 Acked Read 9 Done Done Done false 1026 Acked Read 7 Done Done Done false 1027 Acked Write 10 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Read 4 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 17 16 16 49 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1012 1012 1012 Downstairs last five completed: 1012 1011 1010 1009 1008 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:14:17.042086733Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1494} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 78 78 78 141 141 141 F F F 1 020-039 78 78 78 141 141 141 F F F 2 040-059 77 77 77 140 140 140 F F F 3 060-079 78 78 78 141 141 141 F F F 4 080-099 77 77 77 140 140 140 F F F 5 100-119 78 78 78 142 142 142 T T T 6 120-139 78 78 78 142 142 142 F F F 7 140-159 76 76 76 139 139 139 F F F 8 160-179 77 77 77 140 140 140 T T T 9 180-199 74 74 74 136 136 136 F F F 10 200-219 78 78 78 141 141 141 F F F 11 220-239 76 76 76 138 138 138 F F F 12 240-259 76 76 76 138 138 138 T T T 13 260-279 75 75 75 137 137 137 F F F 14 280-299 68 68 68 126 126 126 T T T 15 300-319 75 75 75 137 137 137 F F F 16 320-339 75 75 75 137 137 137 T T T 17 340-359 76 76 76 138 138 138 F F F 18 360-379 75 75 75 137 137 137 F F F 19 380-399 70 70 70 128 128 128 F F F 20 400-419 78 78 78 141 141 141 F F F 21 420-439 77 77 77 140 140 140 F F F 22 440-459 68 68 68 126 126 126 F F F 23 460-479 75 75 75 137 137 137 F F F 24 480-499 78 78 78 142 142 142 F F F 25 500-519 77 77 77 140 140 140 F F F 26 520-539 74 74 74 136 136 136 F F F 27 540-559 76 76 76 138 138 138 F F F 28 560-579 74 74 74 136 136 136 T T T 29 580-599 70 70 70 128 128 128 T T T Max gen: 78, Max flush: 142 A Difference in extent metadata is expected here On loop 39 Verifying data now No change after verify Loop: 39 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 78 78 78 141 141 141 F F F 1 020-039 78 78 78 141 141 141 F F F 2 040-059 77 77 77 140 140 140 F F F 3 060-079 78 78 78 141 141 141 F F F 4 080-099 77 77 77 140 140 140 F F F 5 100-119 79 79 79 143 143 143 F F F 6 120-139 78 78 78 142 142 142 F F F 7 140-159 76 76 76 139 139 139 F F F 8 160-179 79 79 79 143 143 143 F F F 9 180-199 74 74 74 136 136 136 F F F 10 200-219 78 78 78 141 141 141 F F F 11 220-239 76 76 76 138 138 138 F F F 12 240-259 79 79 79 143 143 143 F F F 13 260-279 75 75 75 137 137 137 F F F 14 280-299 79 79 79 143 143 143 F F F 15 300-319 75 75 75 137 137 137 F F F 16 320-339 79 79 79 143 143 143 F F F 17 340-359 76 76 76 138 138 138 F F F 18 360-379 75 75 75 137 137 137 F F F 19 380-399 70 70 70 128 128 128 F F F 20 400-419 78 78 78 141 141 141 F F F 21 420-439 77 77 77 140 140 140 F F F 22 440-459 68 68 68 126 126 126 F F F 23 460-479 75 75 75 137 137 137 F F F 24 480-499 78 78 78 142 142 142 F F F 25 500-519 77 77 77 140 140 140 F F F 26 520-539 74 74 74 136 136 136 F F F 27 540-559 76 76 76 138 138 138 F F F 28 560-579 79 79 79 143 143 143 F F F 29 580-599 79 79 79 143 143 143 F F F Max gen: 79, Max flush: 143 Begin loop 40 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:21.293194123Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1502,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:21.339343686Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1502,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:21.339981168Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1502} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"abd5430f1508af0bc008143ea2566277ef714ec2\",\n git_commit_timestamp: \"2025-01-17T20:55:03.000000000Z\",\n git_branch: \"alan/tokio-to-140\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:21.34003423Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1502} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:21.340049919Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1502} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:21.340067679Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1502} {"msg":"Crucible cc1db865-6fda-4c54-9cb1-af0c43c4349a has session id: 74d5e011-77b7-455e-bb7e-795997af62ad","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:21.340614539Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1502,"session_id":"74d5e011-77b7-455e-bb7e-795997af62ad"} {"msg":"Upstairs opts: Upstairs UUID: cc1db865-6fda-4c54-9cb1-af0c43c4349a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:21.34065208Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1502,"session_id":"74d5e011-77b7-455e-bb7e-795997af62ad"} {"msg":"Crucible stats registered with UUID: cc1db865-6fda-4c54-9cb1-af0c43c4349a","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:21.340675524Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1502,"session_id":"74d5e011-77b7-455e-bb7e-795997af62ad"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:14:21.340695846Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1502,"session_id":"74d5e011-77b7-455e-bb7e-795997af62ad"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:21.341144028Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1502,"task":"crutest"} {"msg":"The guest has requested activation with gen:80","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:21.341168162Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1502} {"msg":"cc1db865-6fda-4c54-9cb1-af0c43c4349a active request set","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:21.341225466Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1502,"session_id":"74d5e011-77b7-455e-bb7e-795997af62ad"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:21.341251941Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1502,"client":"0","":"downstairs","session_id":"74d5e011-77b7-455e-bb7e-795997af62ad"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:21.341276456Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1502,"client":"0","":"downstairs","session_id":"74d5e011-77b7-455e-bb7e-795997af62ad"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:21.341293515Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1502,"client":"1","":"downstairs","session_id":"74d5e011-77b7-455e-bb7e-795997af62ad"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:21.34131884Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1502,"client":"1","":"downstairs","session_id":"74d5e011-77b7-455e-bb7e-795997af62ad"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:21.341343995Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1502,"client":"2","":"downstairs","session_id":"74d5e011-77b7-455e-bb7e-795997af62ad"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:21.34137004Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1502,"client":"2","":"downstairs","session_id":"74d5e011-77b7-455e-bb7e-795997af62ad"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:21.341400218Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1502,"":"io task","client":"2","":"downstairs","session_id":"74d5e011-77b7-455e-bb7e-795997af62ad"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:21.341427384Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1502,"":"io task","client":"0","":"downstairs","session_id":"74d5e011-77b7-455e-bb7e-795997af62ad"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:21.341453139Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1502,"":"io task","client":"1","":"downstairs","session_id":"74d5e011-77b7-455e-bb7e-795997af62ad"} {"msg":"ds_connection connected from Ok(127.0.0.1:64382)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:21.341477123Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1502,"":"io task","client":"2","":"downstairs","session_id":"74d5e011-77b7-455e-bb7e-795997af62ad"} {"msg":"ds_connection connected from Ok(127.0.0.1:53400)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:21.341505879Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1502,"":"io task","client":"1","":"downstairs","session_id":"74d5e011-77b7-455e-bb7e-795997af62ad"} {"msg":"ds_connection connected from Ok(127.0.0.1:53021)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:21.341645811Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1502,"":"io task","client":"0","":"downstairs","session_id":"74d5e011-77b7-455e-bb7e-795997af62ad"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:21.342153998Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1502,"client":"0","":"downstairs","session_id":"74d5e011-77b7-455e-bb7e-795997af62ad"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:21.342183105Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1502,"client":"2","":"downstairs","session_id":"74d5e011-77b7-455e-bb7e-795997af62ad"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:21.342837898Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1502,"client":"0","":"downstairs","session_id":"74d5e011-77b7-455e-bb7e-795997af62ad"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:21.342866485Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1502,"client":"2","":"downstairs","session_id":"74d5e011-77b7-455e-bb7e-795997af62ad"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:21.343292594Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1502,"client":"1","":"downstairs","session_id":"74d5e011-77b7-455e-bb7e-795997af62ad"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:21.343449916Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1502,"session_id":"74d5e011-77b7-455e-bb7e-795997af62ad"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:21.343550325Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1502,"session_id":"74d5e011-77b7-455e-bb7e-795997af62ad"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:21.343826536Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1502,"client":"1","":"downstairs","session_id":"74d5e011-77b7-455e-bb7e-795997af62ad"} {"msg":"[0]R flush_numbers[0..12]: [141, 141, 140, 141, 140, 143, 142, 139, 143, 136, 141, 138]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:21.344300693Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1502,"":"downstairs","":"downstairs","session_id":"74d5e011-77b7-455e-bb7e-795997af62ad"} {"msg":"[0]R generation[0..12]: [78, 78, 77, 78, 77, 79, 78, 76, 79, 74, 78, 76]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:21.344333552Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1502,"":"downstairs","":"downstairs","session_id":"74d5e011-77b7-455e-bb7e-795997af62ad"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:21.344352373Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1502,"":"downstairs","":"downstairs","session_id":"74d5e011-77b7-455e-bb7e-795997af62ad"} {"msg":"[1]R flush_numbers[0..12]: [141, 141, 140, 141, 140, 143, 142, 139, 143, 136, 141, 138]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:21.344369723Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1502,"":"downstairs","":"downstairs","session_id":"74d5e011-77b7-455e-bb7e-795997af62ad"} {"msg":"[1]R generation[0..12]: [78, 78, 77, 78, 77, 79, 78, 76, 79, 74, 78, 76]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:21.344386523Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1502,"":"downstairs","":"downstairs","session_id":"74d5e011-77b7-455e-bb7e-795997af62ad"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:21.344403383Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1502,"":"downstairs","":"downstairs","session_id":"74d5e011-77b7-455e-bb7e-795997af62ad"} {"msg":"[2]R flush_numbers[0..12]: [141, 141, 140, 141, 140, 143, 142, 139, 143, 136, 141, 138]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:21.344419953Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1502,"":"downstairs","":"downstairs","session_id":"74d5e011-77b7-455e-bb7e-795997af62ad"} {"msg":"[2]R generation[0..12]: [78, 78, 77, 78, 77, 79, 78, 76, 79, 74, 78, 76]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:21.344436652Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1502,"":"downstairs","":"downstairs","session_id":"74d5e011-77b7-455e-bb7e-795997af62ad"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-17T21:14:21.344453542Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1502,"":"downstairs","":"downstairs","session_id":"74d5e011-77b7-455e-bb7e-795997af62ad"} {"msg":"Max found gen is 80","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:21.34448428Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1502,"":"downstairs","":"downstairs","session_id":"74d5e011-77b7-455e-bb7e-795997af62ad"} {"msg":"Generation requested: 80 >= found:80","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:21.344514758Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1502,"":"downstairs","":"downstairs","session_id":"74d5e011-77b7-455e-bb7e-795997af62ad"} {"msg":"Next flush: 144","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:21.344540193Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1502,"":"downstairs","":"downstairs","session_id":"74d5e011-77b7-455e-bb7e-795997af62ad"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:21.344564838Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1502,"":"downstairs","":"downstairs","session_id":"74d5e011-77b7-455e-bb7e-795997af62ad"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:21.344590642Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1502,"session_id":"74d5e011-77b7-455e-bb7e-795997af62ad"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:21.344614817Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1502,"client":"0","":"downstairs","session_id":"74d5e011-77b7-455e-bb7e-795997af62ad"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:21.344644584Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1502,"client":"1","":"downstairs","session_id":"74d5e011-77b7-455e-bb7e-795997af62ad"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:21.34467155Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1502,"client":"2","":"downstairs","session_id":"74d5e011-77b7-455e-bb7e-795997af62ad"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:21.344698225Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1502,"session_id":"74d5e011-77b7-455e-bb7e-795997af62ad"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2025-01-17T21:14:21.34472347Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1502,"session_id":"74d5e011-77b7-455e-bb7e-795997af62ad"} {"msg":"cc1db865-6fda-4c54-9cb1-af0c43c4349a is now active with session: 74d5e011-77b7-455e-bb7e-795997af62ad","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:21.344751596Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1502,"session_id":"74d5e011-77b7-455e-bb7e-795997af62ad"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:21.34477496Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1502,"session_id":"74d5e011-77b7-455e-bb7e-795997af62ad"} {"msg":"The guest has finished waiting for activation with:80","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:21.3447924Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1502} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 205 len 40960 data: 5 3 4 6 7 5 5 5 5 4 03/30 Read block 516 len 40960 04/30 Write block 434 len 12288 data: 6 5 4 05/30 Write block 510 len 28672 data: 10 8 7 8 9 12 12 06/30 Read block 287 len 36864 07/30 Read block 377 len 16384 08/30 Flush 09/30 Flush 10/30 Write block 437 len 20480 data: 5 4 3 3 4 11/30 Read block 490 len 32768 12/30 Read block 576 len 40960 13/30 Read block 22 len 8192 14/30 Read block 457 len 4096 15/30 Write block 541 len 40960 data: 4 5 7 6 6 8 8 9 8 8 16/30 Write block 281 len 36864 data: 7 7 5 6 6 5 4 4 5 17/30 Read block 520 len 4096 18/30 Flush 19/30 Write block 225 len 32768 data: 6 6 6 5 5 8 6 6 20/30 Write block 200 len 12288 data: 10 8 8 21/30 Read block 0 len 8192 22/30 Read block 41 len 4096 23/30 Flush 24/30 Write block 213 len 32768 data: 6 5 4 5 5 4 5 4 25/30 Flush 26/30 Write block 145 len 8192 data: 5 5 27/30 Write block 565 len 40960 data: 7 10 7 7 5 5 4 3 3 4 28/30 Read block 566 len 32768 29/30 Write block 537 len 20480 data: 4 3 4 4 5 30/30 Write block 210 len 24576 data: 6 6 6 7 6 5 ---------------------------------------------------------------- Crucible gen:80 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 2 Done Done Done false 1026 Acked Write 10 Done Done Done false 1027 Acked Read 8 Done Done Done false 1028 Acked Write 5 Sent Sent Sent false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:14:21.36494754Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1502} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 78 78 78 141 141 141 F F F 1 020-039 78 78 78 141 141 141 F F F 2 040-059 77 77 77 140 140 140 F F F 3 060-079 78 78 78 141 141 141 F F F 4 080-099 77 77 77 140 140 140 F F F 5 100-119 79 79 79 143 143 143 F F F 6 120-139 78 78 78 142 142 142 F F F 7 140-159 76 76 76 139 139 139 T T T 8 160-179 79 79 79 143 143 143 F F F 9 180-199 74 74 74 136 136 136 F F F 10 200-219 80 80 80 149 149 149 T T T 11 220-239 80 80 80 149 149 149 F F F 12 240-259 79 79 79 143 143 143 F F F 13 260-279 75 75 75 137 137 137 F F F 14 280-299 80 80 80 147 147 147 F F F 15 300-319 75 75 75 137 137 137 F F F 16 320-339 79 79 79 143 143 143 F F F 17 340-359 76 76 76 138 138 138 F F F 18 360-379 75 75 75 137 137 137 F F F 19 380-399 70 70 70 128 128 128 F F F 20 400-419 78 78 78 141 141 141 F F F 21 420-439 80 80 80 147 147 147 F F F 22 440-459 80 80 80 147 147 147 F F F 23 460-479 75 75 75 137 137 137 F F F 24 480-499 78 78 78 142 142 142 F F F 25 500-519 80 80 80 145 145 145 F F F 26 520-539 74 74 74 136 136 136 T T T 27 540-559 80 80 80 147 147 147 T T T 28 560-579 79 79 79 143 143 143 T T T 29 580-599 79 79 79 143 143 143 F F F Max gen: 80, Max flush: 149 A Difference in extent metadata is expected here On loop 40 Verifying data now No change after verify Loop: 40 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 78 78 78 141 141 141 F F F 1 020-039 78 78 78 141 141 141 F F F 2 040-059 77 77 77 140 140 140 F F F 3 060-079 78 78 78 141 141 141 F F F 4 080-099 77 77 77 140 140 140 F F F 5 100-119 79 79 79 143 143 143 F F F 6 120-139 78 78 78 142 142 142 F F F 7 140-159 81 81 81 150 150 150 F F F 8 160-179 79 79 79 143 143 143 F F F 9 180-199 74 74 74 136 136 136 F F F 10 200-219 81 81 81 150 150 150 F F F 11 220-239 80 80 80 149 149 149 F F F 12 240-259 79 79 79 143 143 143 F F F 13 260-279 75 75 75 137 137 137 F F F 14 280-299 80 80 80 147 147 147 F F F 15 300-319 75 75 75 137 137 137 F F F 16 320-339 79 79 79 143 143 143 F F F 17 340-359 76 76 76 138 138 138 F F F 18 360-379 75 75 75 137 137 137 F F F 19 380-399 70 70 70 128 128 128 F F F 20 400-419 78 78 78 141 141 141 F F F 21 420-439 80 80 80 147 147 147 F F F 22 440-459 80 80 80 147 147 147 F F F 23 460-479 75 75 75 137 137 137 F F F 24 480-499 78 78 78 142 142 142 F F F 25 500-519 80 80 80 145 145 145 F F F 26 520-539 81 81 81 150 150 150 F F F 27 540-559 81 81 81 150 150 150 F F F 28 560-579 81 81 81 150 150 150 F F F 29 580-599 79 79 79 143 143 143 F F F Max gen: 81, Max flush: 150 Begin loop 41 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:25.604493114Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1510,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:14:25.647628876Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1510,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:25.64806377Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1510,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:25.680019157Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1510,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:25.680662603Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1510} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"abd5430f1508af0bc008143ea2566277ef714ec2\",\n git_commit_timestamp: \"2025-01-17T20:55:03.000000000Z\",\n git_branch: \"alan/tokio-to-140\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:25.680695863Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1510} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:25.68070936Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1510} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:25.680723489Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1510} {"msg":"Crucible 68851d04-1fdf-4e98-8943-31bd7561a5d6 has session id: e8ef85b8-5103-40d0-91ae-3d8b8e698f40","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:25.68130729Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1510,"session_id":"e8ef85b8-5103-40d0-91ae-3d8b8e698f40"} {"msg":"Upstairs opts: Upstairs UUID: 68851d04-1fdf-4e98-8943-31bd7561a5d6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:25.681348354Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1510,"session_id":"e8ef85b8-5103-40d0-91ae-3d8b8e698f40"} {"msg":"Crucible stats registered with UUID: 68851d04-1fdf-4e98-8943-31bd7561a5d6","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:25.68137494Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1510,"session_id":"e8ef85b8-5103-40d0-91ae-3d8b8e698f40"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:14:25.681396052Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1510,"session_id":"e8ef85b8-5103-40d0-91ae-3d8b8e698f40"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:25.681867017Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1510,"task":"crutest"} {"msg":"The guest has requested activation with gen:82","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:25.68188987Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1510} {"msg":"68851d04-1fdf-4e98-8943-31bd7561a5d6 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:25.681956019Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1510,"session_id":"e8ef85b8-5103-40d0-91ae-3d8b8e698f40"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:25.681972899Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1510,"client":"0","":"downstairs","session_id":"e8ef85b8-5103-40d0-91ae-3d8b8e698f40"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:25.682000185Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1510,"client":"0","":"downstairs","session_id":"e8ef85b8-5103-40d0-91ae-3d8b8e698f40"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:25.682014293Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1510,"client":"1","":"downstairs","session_id":"e8ef85b8-5103-40d0-91ae-3d8b8e698f40"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:25.682028201Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1510,"client":"1","":"downstairs","session_id":"e8ef85b8-5103-40d0-91ae-3d8b8e698f40"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:25.682040979Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1510,"client":"2","":"downstairs","session_id":"e8ef85b8-5103-40d0-91ae-3d8b8e698f40"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:25.682054487Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1510,"client":"2","":"downstairs","session_id":"e8ef85b8-5103-40d0-91ae-3d8b8e698f40"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:25.682119815Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1510,"":"io task","client":"2","":"downstairs","session_id":"e8ef85b8-5103-40d0-91ae-3d8b8e698f40"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:25.682156726Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1510,"":"io task","client":"0","":"downstairs","session_id":"e8ef85b8-5103-40d0-91ae-3d8b8e698f40"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:25.682184723Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1510,"":"io task","client":"1","":"downstairs","session_id":"e8ef85b8-5103-40d0-91ae-3d8b8e698f40"} {"msg":"ds_connection connected from Ok(127.0.0.1:59078)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:25.682212029Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1510,"":"io task","client":"0","":"downstairs","session_id":"e8ef85b8-5103-40d0-91ae-3d8b8e698f40"} {"msg":"ds_connection connected from Ok(127.0.0.1:49125)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:25.682239014Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1510,"":"io task","client":"2","":"downstairs","session_id":"e8ef85b8-5103-40d0-91ae-3d8b8e698f40"} {"msg":"ds_connection connected from Ok(127.0.0.1:57529)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:25.682263429Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1510,"":"io task","client":"1","":"downstairs","session_id":"e8ef85b8-5103-40d0-91ae-3d8b8e698f40"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:25.682887203Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1510,"client":"1","":"downstairs","session_id":"e8ef85b8-5103-40d0-91ae-3d8b8e698f40"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:25.682927307Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1510,"client":"2","":"downstairs","session_id":"e8ef85b8-5103-40d0-91ae-3d8b8e698f40"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:25.683192542Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1510,"client":"0","":"downstairs","session_id":"e8ef85b8-5103-40d0-91ae-3d8b8e698f40"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:25.683655633Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1510,"client":"1","":"downstairs","session_id":"e8ef85b8-5103-40d0-91ae-3d8b8e698f40"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:25.683692874Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1510,"client":"2","":"downstairs","session_id":"e8ef85b8-5103-40d0-91ae-3d8b8e698f40"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:25.683724053Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1510,"client":"0","":"downstairs","session_id":"e8ef85b8-5103-40d0-91ae-3d8b8e698f40"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:25.684229758Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1510,"session_id":"e8ef85b8-5103-40d0-91ae-3d8b8e698f40"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:25.684351929Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1510,"session_id":"e8ef85b8-5103-40d0-91ae-3d8b8e698f40"} {"msg":"[0]R flush_numbers[0..12]: [141, 141, 140, 141, 140, 143, 142, 150, 143, 136, 150, 149]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:25.684414276Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1510,"":"downstairs","":"downstairs","session_id":"e8ef85b8-5103-40d0-91ae-3d8b8e698f40"} {"msg":"[0]R generation[0..12]: [78, 78, 77, 78, 77, 79, 78, 81, 79, 74, 81, 80]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:25.684433897Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1510,"":"downstairs","":"downstairs","session_id":"e8ef85b8-5103-40d0-91ae-3d8b8e698f40"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:25.684448506Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1510,"":"downstairs","":"downstairs","session_id":"e8ef85b8-5103-40d0-91ae-3d8b8e698f40"} {"msg":"[1]R flush_numbers[0..12]: [141, 141, 140, 141, 140, 143, 142, 150, 143, 136, 150, 149]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:25.684462594Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1510,"":"downstairs","":"downstairs","session_id":"e8ef85b8-5103-40d0-91ae-3d8b8e698f40"} {"msg":"[1]R generation[0..12]: [78, 78, 77, 78, 77, 79, 78, 81, 79, 74, 81, 80]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:25.684476032Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1510,"":"downstairs","":"downstairs","session_id":"e8ef85b8-5103-40d0-91ae-3d8b8e698f40"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:25.6844899Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1510,"":"downstairs","":"downstairs","session_id":"e8ef85b8-5103-40d0-91ae-3d8b8e698f40"} {"msg":"[2]R flush_numbers[0..12]: [141, 141, 140, 141, 140, 143, 142, 150, 143, 136, 150, 149]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:25.684503118Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1510,"":"downstairs","":"downstairs","session_id":"e8ef85b8-5103-40d0-91ae-3d8b8e698f40"} {"msg":"[2]R generation[0..12]: [78, 78, 77, 78, 77, 79, 78, 81, 79, 74, 81, 80]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:25.684516345Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1510,"":"downstairs","":"downstairs","session_id":"e8ef85b8-5103-40d0-91ae-3d8b8e698f40"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:25.684529913Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1510,"":"downstairs","":"downstairs","session_id":"e8ef85b8-5103-40d0-91ae-3d8b8e698f40"} {"msg":"Max found gen is 82","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:25.684545683Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1510,"":"downstairs","":"downstairs","session_id":"e8ef85b8-5103-40d0-91ae-3d8b8e698f40"} {"msg":"Generation requested: 82 >= found:82","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:25.684559181Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1510,"":"downstairs","":"downstairs","session_id":"e8ef85b8-5103-40d0-91ae-3d8b8e698f40"} {"msg":"Next flush: 151","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:25.684571638Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1510,"":"downstairs","":"downstairs","session_id":"e8ef85b8-5103-40d0-91ae-3d8b8e698f40"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:25.684588838Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1510,"":"downstairs","":"downstairs"Wait for a query_work_queue command to finish before sending IO ,"session_id":"e8ef85b8-5103-40d0-91ae-3d8b8e698f40"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:25.684617685Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1510,"session_id":"e8ef85b8-5103-40d0-91ae-3d8b8e698f40"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:25.684629632Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1510,"client":"0","":"downstairs","session_id":"e8ef85b8-5103-40d0-91ae-3d8b8e698f40"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:25.684642359Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1510,"client":"1","":"downstairs","session_id":"e8ef85b8-5103-40d0-91ae-3d8b8e698f40"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:25.684654877Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1510,"client":"2","":"downstairs","session_id":"e8ef85b8-5103-40d0-91ae-3d8b8e698f40"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:25.684667534Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1510,"session_id":"e8ef85b8-5103-40d0-91ae-3d8b8e698f40"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:25.684679491Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1510,"session_id":"e8ef85b8-5103-40d0-91ae-3d8b8e698f40"} {"msg":"68851d04-1fdf-4e98-8943-31bd7561a5d6 is now active with session: e8ef85b8-5103-40d0-91ae-3d8b8e698f40","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:25.684691868Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1510,"session_id":"e8ef85b8-5103-40d0-91ae-3d8b8e698f40"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:25.684703595Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1510,"session_id":"e8ef85b8-5103-40d0-91ae-3d8b8e698f40"} {"msg":"The guest has finished waiting for activation with:82","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:25.684715452Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1510} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 115 len 28672 data: 10 7 8 9 9 7 5 02/30 Write block 83 len 12288 data: 6 4 4 03/30 Write block 2 len 32768 data: 4 4 5 6 6 6 5 6 04/30 Read block 520 len 28672 05/30 Write block 578 len 36864 data: 6 7 7 6 5 6 7 6 6 06/30 Write block 291 len 40960 data: 8 9 8 7 9 9 8 7 8 6 07/30 Read block 422 len 12288 08/30 Read block 543 len 32768 09/30 Read block 114 len 36864 10/30 Read block 496 len 12288 11/30 Write block 84 len 32768 data: 5 5 3 3 3 4 7 6 12/30 Write block 258 len 40960 data: 8 8 7 7 6 6 6 7 7 7 13/30 Write block 337 len 36864 data: 10 10 10 10 8 8 12 8 7 14/30 Write block 16 len 8192 data: 10 8 15/30 Write block 434 len 20480 data: 7 6 5 6 5 16/30 Write block 213 len 40960 data: 8 7 6 6 6 5 6 5 5 7 17/30 Write block 211 len 12288 data: 7 7 9 18/30 Flush 19/30 Write block 482 len 24576 data: 5 4 5 4 5 5 20/30 Read block 53 len 24576 21/30 Read block 302 len 20480 22/30 Flush 23/30 Write block 112 len 20480 data: 9 9 10 11 8 24/30 Read block 232 len 16384 25/30 Write block 87 len 36864 data: 4 4 5 8 7 8 8 9 9 26/30 Write block 220 len 40960 data: 6 6 8 8 7 7 7 7 6 6 27/30 Write block 300 len 8192 data: 7 7 28/30 Read block 376 len 36864 29/30 Write block 215 len 8192 data: 7 7 30/30 Write block 104 len 36864 data: 8 9 7 6 6 7 7 9 10 ---------------------------------------------------------------- Crucible gen:82 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 5 Done Done Done false 1023 Acked Read 4 Done Done Done false 1024 Acked Write 9 Done Done Done false 1025 Acked Write 10 Done Done Done false 1026 Acked Write 2 Done Done Done false 1027 Acked Read 9 Done Done Done false 1028 Acked Write 2 Done Done Done false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 7 7 7 21 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1021 1021 1021 Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:14:25.704539749Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1510} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 82 82 82 151 151 151 F F F 1 020-039 78 78 78 141 141 141 F F F 2 040-059 77 77 77 140 140 140 F F F 3 060-079 78 78 78 141 141 141 F F F 4 080-099 82 82 82 151 151 151 T T T 5 100-119 82 82 82 151 151 151 T T T 6 120-139 82 82 82 151 151 151 F F F 7 140-159 81 81 81 150 150 150 F F F 8 160-179 79 79 79 143 143 143 F F F 9 180-199 74 74 74 136 136 136 F F F 10 200-219 82 82 82 151 151 151 T T T 11 220-239 82 82 82 151 151 151 T T T 12 240-259 82 82 82 151 151 151 F F F 13 260-279 82 82 82 151 151 151 F F F 14 280-299 82 82 82 151 151 151 F F F 15 300-319 82 82 82 151 151 151 T T T 16 320-339 82 82 82 151 151 151 F F F 17 340-359 82 82 82 151 151 151 F F F 18 360-379 75 75 75 137 137 137 F F F 19 380-399 70 70 70 128 128 128 F F F 20 400-419 78 78 78 141 141 141 F F F 21 420-439 82 82 82 151 151 151 F F F 22 440-459 80 80 80 147 147 147 F F F 23 460-479 75 75 75 137 137 137 F F F 24 480-499 82 82 82 152 152 152 F F F 25 500-519 80 80 80 145 145 145 F F F 26 520-539 81 81 81 150 150 150 F F F 27 540-559 81 81 81 150 150 150 F F F 28 560-579 82 82 82 151 151 151 F F F 29 580-599 82 82 82 151 151 151 F F F Max gen: 82, Max flush: 152 A Difference in extent metadata is expected here On loop 41 Verifying data now No change after verify Loop: 41 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 82 82 82 151 151 151 F F F 1 020-039 78 78 78 141 141 141 F F F 2 040-059 77 77 77 140 140 140 F F F 3 060-079 78 78 78 141 141 141 F F F 4 080-099 83 83 83 153 153 153 F F F 5 100-119 83 83 83 153 153 153 F F F 6 120-139 82 82 82 151 151 151 F F F 7 140-159 81 81 81 150 150 150 F F F 8 160-179 79 79 79 143 143 143 F F F 9 180-199 74 74 74 136 136 136 F F F 10 200-219 83 83 83 153 153 153 F F F 11 220-239 83 83 83 153 153 153 F F F 12 240-259 82 82 82 151 151 151 F F F 13 260-279 82 82 82 151 151 151 F F F 14 280-299 82 82 82 151 151 151 F F F 15 300-319 83 83 83 153 153 153 F F F 16 320-339 82 82 82 151 151 151 F F F 17 340-359 82 82 82 151 151 151 F F F 18 360-379 75 75 75 137 137 137 F F F 19 380-399 70 70 70 128 128 128 F F F 20 400-419 78 78 78 141 141 141 F F F 21 420-439 82 82 82 151 151 151 F F F 22 440-459 80 80 80 147 147 147 F F F 23 460-479 75 75 75 137 137 137 F F F 24 480-499 82 82 82 152 152 152 F F F 25 500-519 80 80 80 145 145 145 F F F 26 520-539 81 81 81 150 150 150 F F F 27 540-559 81 81 81 150 150 150 F F F 28 560-579 82 82 82 151 151 151 F F F 29 580-599 82 82 82 151 151 151 F F F Max gen: 83, Max flush: 153 Begin loop 42 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:29.975799997Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1518,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:30.022196364Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1518,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:30.022932064Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1518} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"abd5430f1508af0bc008143ea2566277ef714ec2\",\n git_commit_timestamp: \"2025-01-17T20:55:03.000000000Z\",\n git_branch: \"alan/tokio-to-140\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:30.022970997Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1518} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:30.022985225Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1518} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:30.023014272Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1518} {"msg":"Crucible 2f476c5d-c642-4037-8d32-7e0d729cd774 has session id: 2cef7608-ad97-4105-988e-4dcc80d27fd1","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:30.023615914Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1518,"session_id":"2cef7608-ad97-4105-988e-4dcc80d27fd1"} {"msg":"Upstairs opts: Upstairs UUID: 2f476c5d-c642-4037-8d32-7e0d729cd774, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:30.023658589Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1518,"session_id":"2cef7608-ad97-4105-988e-4dcc80d27fd1"} {"msg":"Crucible stats registered with UUID: 2f476c5d-c642-4037-8d32-7e0d729cd774","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:30.023692288Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1518,"session_id":"2cef7608-ad97-4105-988e-4dcc80d27fd1"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:14:30.023716533Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1518,"session_id":"2cef7608-ad97-4105-988e-4dcc80d27fd1"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:30.024124251Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1518,"task":"crutest"} {"msg":"The guest has requested activation with gen:84","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:30.024155859Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1518} {"msg":"2f476c5d-c642-4037-8d32-7e0d729cd774 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:30.024205859Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1518,"session_id":"2cef7608-ad97-4105-988e-4dcc80d27fd1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:30.024230073Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1518,"client":"0","":"downstairs","session_id":"2cef7608-ad97-4105-988e-4dcc80d27fd1"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:30.024263673Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1518,"client":"0","":"downstairs","session_id":"2cef7608-ad97-4105-988e-4dcc80d27fd1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:30.024290268Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1518,"client":"1","":"downstairs","session_id":"2cef7608-ad97-4105-988e-4dcc80d27fd1"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:30.024315973Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1518,"client":"1","":"downstairs","session_id":"2cef7608-ad97-4105-988e-4dcc80d27fd1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:30.024339186Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1518,"client":"2","":"downstairs","session_id":"2cef7608-ad97-4105-988e-4dcc80d27fd1"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:30.024363Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1518,"client":"2","":"downstairs","session_id":"2cef7608-ad97-4105-988e-4dcc80d27fd1"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:30.024384773Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1518,"":"io task","client":"0","":"downstairs","session_id":"2cef7608-ad97-4105-988e-4dcc80d27fd1"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:30.024407747Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1518,"":"io task","client":"2","":"downstairs","session_id":"2cef7608-ad97-4105-988e-4dcc80d27fd1"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:30.024432631Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1518,"":"io task","client":"1","":"downstairs","session_id":"2cef7608-ad97-4105-988e-4dcc80d27fd1"} {"msg":"ds_connection connected from Ok(127.0.0.1:58015)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:30.024460968Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1518,"":"io task","client":"2","":"downstairs","session_id":"2cef7608-ad97-4105-988e-4dcc80d27fd1"} {"msg":"ds_connection connected from Ok(127.0.0.1:49540)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:30.024487093Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1518,"":"io task","client":"1","":"downstairs","session_id":"2cef7608-ad97-4105-988e-4dcc80d27fd1"} {"msg":"ds_connection connected from Ok(127.0.0.1:50608)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:30.024744433Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1518,"":"io task","client":"0","":"downstairs","session_id":"2cef7608-ad97-4105-988e-4dcc80d27fd1"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:30.025240493Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1518,"client":"1","":"downstairs","session_id":"2cef7608-ad97-4105-988e-4dcc80d27fd1"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:30.025400627Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1518,"client":"2","":"downstairs","session_id":"2cef7608-ad97-4105-988e-4dcc80d27fd1"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:30.025473429Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1518,"client":"0","":"downstairs","session_id":"2cef7608-ad97-4105-988e-4dcc80d27fd1"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:30.025904251Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1518,"client":"1","":"downstairs","session_id":"2cef7608-ad97-4105-988e-4dcc80d27fd1"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:30.025977574Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1518,"client":"2","":"downstairs","session_id":"2cef7608-ad97-4105-988e-4dcc80d27fd1"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:30.026006241Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1518,"client":"0","":"downstairs","session_id":"2cef7608-ad97-4105-988e-4dcc80d27fd1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:30.026427647Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1518,"session_id":"2cef7608-ad97-4105-988e-4dcc80d27fd1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:30.026549919Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1518,"session_id":"2cef7608-ad97-4105-988e-4dcc80d27fd1"} {"msg":"[0]R flush_numbers[0..12]: [151, 141, 140, 141, 153, 153, 151, 150, 143, 136, 153, 153]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:30.026684237Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1518,"":"downstairs","":"downstairs","session_id":"2cef7608-ad97-4105-988e-4dcc80d27fd1"} {"msg":"[0]R generation[0..12]: [82, 78, 77, 78, 83, 83, 82, 81, 79, 74, 83, 83]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:30.026704109Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1518,"":"downstairs","":"downstairs","session_id":"2cef7608-ad97-4105-988e-4dcc80d27fd1"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:30.026718667Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1518,"":"downstairs","":"downstairs","session_id":"2cef7608-ad97-4105-988e-4dcc80d27fd1"} {"msg":"[1]R flush_numbers[0..12]: [151, 141, 140, 141, 153, 153, 151, 150, 143, 136, 153, 153]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:30.026732285Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1518,"":"downstairs","":"downstairs","session_id":"2cef7608-ad97-4105-988e-4dcc80d27fd1"} {"msg":"[1]R generation[0..12]: [82, 78, 77, 78, 83, 83, 82, 81, 79, 74, 83, 83]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:30.026749085Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1518,"":"downstairs","":"downstairs","session_id":"2cef7608-ad97-4105-988e-4dcc80d27fd1"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:30.026763193Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1518,"":"downstairs","":"downstairs","session_id":"2cef7608-ad97-4105-988e-4dcc80d27fd1"} {"msg":"[2]R flush_numbers[0..12]: [151, 141, 140, 141, 153, 153, 151, 150, 143, 136, 153, 153]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:30.026776891Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1518,"":"downstairs","":"downstairs","session_id":"2cef7608-ad97-4105-988e-4dcc80d27fd1"} {"msg":"[2]R generation[0..12]: [82, 78, 77, 78, 83, 83, 82, 81, 79, 74, 83, 83]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:30.026790199Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1518,"":"downstairs","":"downstairs","session_id":"2cef7608-ad97-4105-988e-4dcc80d27fd1"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:30.026803807Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1518,"":"downstairs","":"downstairs","session_id":"2cef7608-ad97-4105-988e-4dcc80d27fd1"} {"msg":"Max found gen is 84","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:30.026816425Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1518,"":"downstairs","":"downstairs","session_id":"2cef7608-ad97-4105-988e-4dcc80d27fd1"} {"msg":"Generation requested: 84 >= found:84","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:30.026829492Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1518,"":"downstairs","":"downstairs","session_id":"2cef7608-ad97-4105-988e-4dcc80d27fd1"} {"msg":"Next flush: 154","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:30.026842249Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1518,"":"downstairs","":"downstairs","Wait for a query_work_queue command to finish before sending IO session_id":"2cef7608-ad97-4105-988e-4dcc80d27fd1"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:30.02686081Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1518,"":"downstairs","":"downstairs","session_id":"2cef7608-ad97-4105-988e-4dcc80d27fd1"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:30.026873798Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1518,"session_id":"2cef7608-ad97-4105-988e-4dcc80d27fd1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:30.026885395Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1518,"client":"0","":"downstairs","session_id":"2cef7608-ad97-4105-988e-4dcc80d27fd1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:30.026898302Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1518,"client":"1","":"downstairs","session_id":"2cef7608-ad97-4105-988e-4dcc80d27fd1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:30.02691118Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1518,"client":"2","":"downstairs","session_id":"2cef7608-ad97-4105-988e-4dcc80d27fd1"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:30.026924147Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1518,"session_id":"2cef7608-ad97-4105-988e-4dcc80d27fd1"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:30.026937855Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1518,"session_id":"2cef7608-ad97-4105-988e-4dcc80d27fd1"} {"msg":"2f476c5d-c642-4037-8d32-7e0d729cd774 is now active with session: 2cef7608-ad97-4105-988e-4dcc80d27fd1","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:30.026950453Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1518,"session_id":"2cef7608-ad97-4105-988e-4dcc80d27fd1"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:30.02696216Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1518,"session_id":"2cef7608-ad97-4105-988e-4dcc80d27fd1"} {"msg":"The guest has finished waiting for activation with:84","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:30.026974026Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1518} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 519 len 4096 data: 9 02/30 Write block 421 len 32768 data: 6 5 6 6 6 8 9 10 03/30 Write block 116 len 12288 data: 9 9 10 04/30 Read block 468 len 16384 05/30 Write block 413 len 40960 data: 7 7 7 6 6 5 5 5 7 6 06/30 Read block 113 len 8192 07/30 Write block 73 len 16384 data: 6 6 10 10 08/30 Read block 542 len 24576 09/30 Write block 46 len 16384 data: 6 6 6 5 10/30 Write block 188 len 12288 data: 5 5 8 11/30 Read block 502 len 40960 12/30 Write block 173 len 24576 data: 7 7 10 9 9 6 13/30 Read block 363 len 8192 14/30 Read block 417 len 36864 15/30 Flush 16/30 Write block 351 len 28672 data: 6 7 7 8 8 9 9 17/30 Read block 447 len 24576 18/30 Read block 457 len 8192 19/30 Read block 524 len 24576 20/30 Read block 194 len 16384 21/30 Read block 24 len 4096 22/30 Read block 69 len 28672 23/30 Read block 351 len 32768 24/30 Read block 346 len 8192 25/30 Read block 477 len 40960 26/30 Write block 568 len 16384 data: 8 6 6 5 27/30 Read block 41 len 32768 28/30 Read block 199 len 4096 29/30 Write block 542 len 36864 data: 6 8 7 7 9 9 10 9 9 30/30 Write block 503 len 28672 data: 10 11 11 9 11 10 12 ---------------------------------------------------------------- Crucible gen:84 GIO:true work queues: Upstairs:0 downstairs:15 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1015 Acked Write 7 Done Done Done false 1016 Acked Read 6 Done Done Done false 1017 Acked Read 2 Done Done Done false 1018 Acked Read 6 Done Done Done false 1019 Acked Read 4 Done Done Done false 1020 Acked Read 1 Done Done Done false 1021 Acked Read 7 Done Done Done false 1022 Acked Read 8 Done Done Done false 1023 Acked Read 2 Done Done Done false 1024 Acked Read 10 Done Done Done false 1025 Acked Write 4 Done Done Done false 1026 Acked Read 8 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Write 9 Sent Sent Sent false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 13 13 13 39 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1014 1014 1014 Downstairs last five completed: 1014 1013 1012 1011 1010 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:14:30.046273707Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1518} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 82 82 82 151 151 151 F F F 1 020-039 78 78 78 141 141 141 F F F 2 040-059 84 84 84 154 154 154 F F F 3 060-079 84 84 84 154 154 154 F F F 4 080-099 83 83 83 153 153 153 F F F 5 100-119 84 84 84 154 154 154 F F F 6 120-139 82 82 82 151 151 151 F F F 7 140-159 81 81 81 150 150 150 F F F 8 160-179 84 84 84 154 154 154 F F F 9 180-199 84 84 84 154 154 154 F F F 10 200-219 83 83 83 153 153 153 F F F 11 220-239 83 83 83 153 153 153 F F F 12 240-259 82 82 82 151 151 151 F F F 13 260-279 82 82 82 151 151 151 F F F 14 280-299 82 82 82 151 151 151 F F F 15 300-319 83 83 83 153 153 153 F F F 16 320-339 82 82 82 151 151 151 F F F 17 340-359 82 82 82 151 151 151 T T T 18 360-379 75 75 75 137 137 137 F F F 19 380-399 70 70 70 128 128 128 F F F 20 400-419 84 84 84 154 154 154 F F F 21 420-439 84 84 84 154 154 154 F F F 22 440-459 80 80 80 147 147 147 F F F 23 460-479 75 75 75 137 137 137 F F F 24 480-499 82 82 82 152 152 152 F F F 25 500-519 84 84 84 154 154 154 T T T 26 520-539 81 81 81 150 150 150 F F F 27 540-559 81 81 81 150 150 150 T T T 28 560-579 82 82 82 151 151 151 T T T 29 580-599 82 82 82 151 151 151 F F F Max gen: 84, Max flush: 154 A Difference in extent metadata is expected here On loop 42 Verifying data now No change after verify Loop: 42 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 82 82 82 151 151 151 F F F 1 020-039 78 78 78 141 141 141 F F F 2 040-059 84 84 84 154 154 154 F F F 3 060-079 84 84 84 154 154 154 F F F 4 080-099 83 83 83 153 153 153 F F F 5 100-119 84 84 84 154 154 154 F F F 6 120-139 82 82 82 151 151 151 F F F 7 140-159 81 81 81 150 150 150 F F F 8 160-179 84 84 84 154 154 154 F F F 9 180-199 84 84 84 154 154 154 F F F 10 200-219 83 83 83 153 153 153 F F F 11 220-239 83 83 83 153 153 153 F F F 12 240-259 82 82 82 151 151 151 F F F 13 260-279 82 82 82 151 151 151 F F F 14 280-299 82 82 82 151 151 151 F F F 15 300-319 83 83 83 153 153 153 F F F 16 320-339 82 82 82 151 151 151 F F F 17 340-359 85 85 85 155 155 155 F F F 18 360-379 75 75 75 137 137 137 F F F 19 380-399 70 70 70 128 128 128 F F F 20 400-419 84 84 84 154 154 154 F F F 21 420-439 84 84 84 154 154 154 F F F 22 440-459 80 80 80 147 147 147 F F F 23 460-479 75 75 75 137 137 137 F F F 24 480-499 82 82 82 152 152 152 F F F 25 500-519 85 85 85 155 155 155 F F F 26 520-539 81 81 81 150 150 150 F F F 27 540-559 85 85 85 155 155 155 F F F 28 560-579 85 85 85 155 155 155 F F F 29 580-599 82 82 82 151 151 151 F F F Max gen: 85, Max flush: 155 Begin loop 43 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:34.277290475Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1526,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:14:34.319617134Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1526,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:34.32003798Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1526,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:34.351676161Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1526,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:34.352271469Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1526} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"abd5430f1508af0bc008143ea2566277ef714ec2\",\n git_commit_timestamp: \"2025-01-17T20:55:03.000000000Z\",\n git_branch: \"alan/tokio-to-140\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:34.35230722Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1526} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:34.352321278Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1526} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:34.352335167Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1526} {"msg":"Crucible 969282bd-f5d5-4f9a-8d78-ea887ab97c92 has session id: 29a77d39-3beb-4555-9a19-d09a5a611582","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:34.352954268Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1526,"session_id":"29a77d39-3beb-4555-9a19-d09a5a611582"} {"msg":"Upstairs opts: Upstairs UUID: 969282bd-f5d5-4f9a-8d78-ea887ab97c92, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:34.352997614Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1526,"session_id":"29a77d39-3beb-4555-9a19-d09a5a611582"} {"msg":"Crucible stats registered with UUID: 969282bd-f5d5-4f9a-8d78-ea887ab97c92","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:34.353014253Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1526,"session_id":"29a77d39-3beb-4555-9a19-d09a5a611582"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:14:34.353028031Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1526,"session_id":"29a77d39-3beb-4555-9a19-d09a5a611582"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:34.35355567Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1526,"task":"crutest"} {"msg":"The guest has requested activation with gen:86","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:34.353580875Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1526} {"msg":"969282bd-f5d5-4f9a-8d78-ea887ab97c92 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:34.353639709Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1526,"session_id":"29a77d39-3beb-4555-9a19-d09a5a611582"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:34.353665154Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1526,"client":"0","":"downstairs","session_id":"29a77d39-3beb-4555-9a19-d09a5a611582"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:34.353702206Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1526,"client":"0","":"downstairs","session_id":"29a77d39-3beb-4555-9a19-d09a5a611582"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:34.353729802Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1526,"client":"1","":"downstairs","session_id":"29a77d39-3beb-4555-9a19-d09a5a611582"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:34.353759229Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1526,"client":"1","":"downstairs","session_id":"29a77d39-3beb-4555-9a19-d09a5a611582"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:34.353786075Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1526,"client":"2","":"downstairs","session_id":"29a77d39-3beb-4555-9a19-d09a5a611582"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:34.353815452Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1526,"client":"2","":"downstairs","session_id":"29a77d39-3beb-4555-9a19-d09a5a611582"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:34.353841547Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1526,"":"io task","client":"2","":"downstairs","session_id":"29a77d39-3beb-4555-9a19-d09a5a611582"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:34.3538637Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1526,"":"io task","client":"0","":"downstairs","session_id":"29a77d39-3beb-4555-9a19-d09a5a611582"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:34.353882891Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1526,"":"io task","client":"1","":"downstairs","session_id":"29a77d39-3beb-4555-9a19-d09a5a611582"} {"msg":"ds_connection connected from Ok(127.0.0.1:47893)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:34.353905414Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1526,"":"io task","client":"1","":"downstairs","session_id":"29a77d39-3beb-4555-9a19-d09a5a611582"} {"msg":"ds_connection connected from Ok(127.0.0.1:57040)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:34.35393185Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1526,"":"io task","client":"0","":"downstairs","session_id":"29a77d39-3beb-4555-9a19-d09a5a611582"} {"msg":"ds_connection connected from Ok(127.0.0.1:64724)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:34.353957455Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1526,"":"io task","client":"2","":"downstairs","session_id":"29a77d39-3beb-4555-9a19-d09a5a611582"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:34.354619221Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1526,"client":"1","":"downstairs","session_id":"29a77d39-3beb-4555-9a19-d09a5a611582"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:34.35465045Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1526,"client":"2","":"downstairs","session_id":"29a77d39-3beb-4555-9a19-d09a5a611582"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:34.354665789Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1526,"client":"0","":"downstairs","session_id":"29a77d39-3beb-4555-9a19-d09a5a611582"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:34.355195979Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1526,"client":"1","":"downstairs","session_id":"29a77d39-3beb-4555-9a19-d09a5a611582"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:34.355253862Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1526,"client":"0","":"downstairs","session_id":"29a77d39-3beb-4555-9a19-d09a5a611582"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:34.355349058Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1526,"client":"2","":"downstairs","session_id":"29a77d39-3beb-4555-9a19-d09a5a611582"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:34.355789615Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1526,"session_id":"29a77d39-3beb-4555-9a19-d09a5a611582"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:34.355811448Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1526,"session_id":"29a77d39-3beb-4555-9a19-d09a5a611582"} {"msg":"[0]R flush_numbers[0..12]: [151, 141, 154, 154, 153, 154, 151, 150, 154, 154, 153, 153]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:34.355912027Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1526,"":"downstairs","":"downstairs","session_id":"29a77d39-3beb-4555-9a19-d09a5a611582"} {"msg":"[0]R generation[0..12]: [82, 78, 84, 84, 83, 84, 82, 81, 84, 84, 83, 83]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:34.355930318Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1526,"":"downstairs","":"downstairs","session_id":"29a77d39-3beb-4555-9a19-d09a5a611582"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:34.355944256Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1526,"":"downstairs","":"downstairs","session_id":"29a77d39-3beb-4555-9a19-d09a5a611582"} {"msg":"[1]R flush_numbers[0..12]: [151, 141, 154, 154, 153, 154, 151, 150, 154, 154, 153, 153]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:34.355957644Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1526,"":"downstairs","":"downstairs","session_id":"29a77d39-3beb-4555-9a19-d09a5a611582"} {"msg":"[1]R generation[0..12]: [82, 78, 84, 84, 83, 84, 82, 81, 84, 84, 83, 83]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:34.355970932Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1526,"":"downstairs","":"downstairs","session_id":"29a77d39-3beb-4555-9a19-d09a5a611582"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:34.355984429Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1526,"":"downstairs","":"downstairs","session_id":"29a77d39-3beb-4555-9a19-d09a5a611582"} {"msg":"[2]R flush_numbers[0..12]: [151, 141, 154, 154, 153, 154, 151, 150, 154, 154, 153, 153]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:34.355997817Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1526,"":"downstairs","":"downstairs","session_id":"29a77d39-3beb-4555-9a19-d09a5a611582"} {"msg":"[2]R generation[0..12]: [82, 78, 84, 84, 83, 84, 82, 81, 84, 84, 83, 83]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:34.356010835Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1526,"":"downstairs","":"downstairs","session_id":"29a77d39-3beb-4555-9a19-d09a5a611582"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:34.356024313Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1526,"":"downstairs","":"downstairs","session_id":"29a77d39-3beb-4555-9a19-d09a5a611582"} {"msg":"Max found gen is 86","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:34.356039852Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1526,"":"downstairs","":"downstairs","session_id":"29a77d39-3beb-4555-9a19-d09a5a611582"} {"msg":"Generation requested: 86 >= found:86","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:34.356052839Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1526,"":"downstairs","":"downstairs","session_id":"29a77d39-3beb-4555-9a19-d09a5a611582"} {"msg":"Next flush: 156","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:34.356065317Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1526,"":"downstairs","":"downstairs","session_id":"29a77d39-3beb-4555-9a19-d09a5a611582"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-17T21:14:34.356077724Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1526,"":"downstairs","":"downstairs","session_id":"29a77d39-3beb-4555-9a19-d09a5a611582"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:34.356093283Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1526,"session_id":"29a77d39-3beb-4555-9a19-d09a5a611582"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:34.35610507Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1526,"client":"0","":"downstairs","session_id":"29a77d39-3beb-4555-9a19-d09a5a611582"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:34.356118087Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1526,"client":"1","":"downstairs","session_id":"29a77d39-3beb-4555-9a19-d09a5a611582"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:34.356130635Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1526,"client":"2","":"downstairs","session_id":"29a77d39-3beb-4555-9a19-d09a5a611582"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:34.356143392Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1526,"session_id":"29a77d39-3beb-4555-9a19-d09a5a611582"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:34.356155399Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1526,"session_id":"29a77d39-3beb-4555-9a19-d09a5a611582"} {"msg":"969282bd-f5d5-4f9a-8d78-ea887ab97c92 is now active with session: 29a77d39-3beb-4555-9a19-d09a5a611582","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:34.356167796Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1526,"session_id":"29a77d39-3beb-4555-9a19-d09a5a611582"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:34.356179423Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1526,"session_id":"29a77d39-3beb-4555-9a19-d09a5a611582"} {"msg":"The guest has finished waiting for activation with:86","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:34.35619117Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1526} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 152 len 40960 data: 7 7 8 9 9 10 9 9 8 9 02/30 Read block 107 len 36864 03/30 Read block 512 len 24576 04/30 Write block 55 len 12288 data: 10 10 10 05/30 Flush 06/30 Write block 209 len 12288 data: 8 7 8 07/30 Read block 387 len 4096 08/30 Write block 531 len 40960 data: 5 5 4 3 3 4 5 4 5 5 09/30 Read block 222 len 20480 10/30 Read block 246 len 12288 11/30 Read block 201 len 40960 12/30 Write block 288 len 4096 data: 5 13/30 Read block 97 len 28672 14/30 Read block 91 len 36864 15/30 Read block 204 len 20480 16/30 Read block 557 len 24576 17/30 Read block 561 len 28672 18/30 Write block 428 len 28672 data: 11 10 9 9 8 6 8 19/30 Read block 75 len 32768 20/30 Flush 21/30 Flush 22/30 Flush 23/30 Write block 164 len 40960 data: 7 8 8 6 7 6 7 7 7 8 24/30 Flush 25/30 Flush 26/30 Write block 338 len 16384 data: 11 11 11 9 27/30 Write block 301 len 28672 data: 8 5 6 5 6 6 6 28/30 Write block 454 len 20480 data: 6 6 7 8 8 29/30 Write block 446 len 4096 data: 5 30/30 Read block 566 len 32768 ---------------------------------------------------------------- Crucible gen:86 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 4 Done Done Done false 1026 Acked Write 7 Done Done Done false 1027 Acked Write 5 Done Done Done false 1028 Acked Write 1 Done Done Done false 1029 Acked Read 8 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:14:34.376886106Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1526} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 82 82 82 151 151 151 F F F 1 020-039 78 78 78 141 141 141 F F F 2 040-059 86 86 86 156 156 156 F F F 3 060-079 84 84 84 154 154 154 F F F 4 080-099 83 83 83 153 153 153 F F F 5 100-119 84 84 84 154 154 154 F F F 6 120-139 82 82 82 151 151 151 F F F 7 140-159 86 86 86 156 156 156 F F F 8 160-179 86 86 86 160 160 160 F F F 9 180-199 84 84 84 154 154 154 F F F 10 200-219 86 86 86 157 157 157 F F F 11 220-239 83 83 83 153 153 153 F F F 12 240-259 82 82 82 151 151 151 F F F 13 260-279 82 82 82 151 151 151 F F F 14 280-299 86 86 86 157 157 157 F F F 15 300-319 83 83 83 153 153 153 T T T 16 320-339 82 82 82 151 151 151 T T T 17 340-359 85 85 85 155 155 155 T T T 18 360-379 75 75 75 137 137 137 F F F 19 380-399 70 70 70 128 128 128 F F F 20 400-419 84 84 84 154 154 154 F F F 21 420-439 86 86 86 157 157 157 F F F 22 440-459 80 80 80 147 147 147 T T T 23 460-479 75 75 75 137 137 137 F F F 24 480-499 82 82 82 152 152 152 F F F 25 500-519 85 85 85 155 155 155 F F F 26 520-539 86 86 86 157 157 157 F F F 27 540-559 86 86 86 157 157 157 F F F 28 560-579 85 85 85 155 155 155 F F F 29 580-599 82 82 82 151 151 151 F F F Max gen: 86, Max flush: 160 A Difference in extent metadata is expected here On loop 43 Verifying data now No change after verify Loop: 43 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 82 82 82 151 151 151 F F F 1 020-039 78 78 78 141 141 141 F F F 2 040-059 86 86 86 156 156 156 F F F 3 060-079 84 84 84 154 154 154 F F F 4 080-099 83 83 83 153 153 153 F F F 5 100-119 84 84 84 154 154 154 F F F 6 120-139 82 82 82 151 151 151 F F F 7 140-159 86 86 86 156 156 156 F F F 8 160-179 86 86 86 160 160 160 F F F 9 180-199 84 84 84 154 154 154 F F F 10 200-219 86 86 86 157 157 157 F F F 11 220-239 83 83 83 153 153 153 F F F 12 240-259 82 82 82 151 151 151 F F F 13 260-279 82 82 82 151 151 151 F F F 14 280-299 86 86 86 157 157 157 F F F 15 300-319 87 87 87 161 161 161 F F F 16 320-339 87 87 87 161 161 161 F F F 17 340-359 87 87 87 161 161 161 F F F 18 360-379 75 75 75 137 137 137 F F F 19 380-399 70 70 70 128 128 128 F F F 20 400-419 84 84 84 154 154 154 F F F 21 420-439 86 86 86 157 157 157 F F F 22 440-459 87 87 87 161 161 161 F F F 23 460-479 75 75 75 137 137 137 F F F 24 480-499 82 82 82 152 152 152 F F F 25 500-519 85 85 85 155 155 155 F F F 26 520-539 86 86 86 157 157 157 F F F 27 540-559 86 86 86 157 157 157 F F F 28 560-579 85 85 85 155 155 155 F F F 29 580-599 82 82 82 151 151 151 F F F Max gen: 87, Max flush: 161 Begin loop 44 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:38.638775837Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1534,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:14:38.681766764Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1534,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:38.682210924Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1534,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:38.713798346Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1534,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:38.71440438Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1534} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"abd5430f1508af0bc008143ea2566277ef714ec2\",\n git_commit_timestamp: \"2025-01-17T20:55:03.000000000Z\",\n git_branch: \"alan/tokio-to-140\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:38.71443809Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1534} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:38.714451448Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1534} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:38.714465476Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1534} {"msg":"Crucible 3be26146-f77d-43c3-9190-d0f0242807e5 has session id: 9c36d8a0-1cd8-48fc-942e-dc458f145e46","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:38.714991474Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1534,"session_id":"9c36d8a0-1cd8-48fc-942e-dc458f145e46"} {"msg":"Upstairs opts: Upstairs UUID: 3be26146-f77d-43c3-9190-d0f0242807e5, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:38.715029746Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1534,"session_id":"9c36d8a0-1cd8-48fc-942e-dc458f145e46"} {"msg":"Crucible stats registered with UUID: 3be26146-f77d-43c3-9190-d0f0242807e5","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:38.715048987Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1534,"session_id":"9c36d8a0-1cd8-48fc-942e-dc458f145e46"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:14:38.715061905Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1534,"session_id":"9c36d8a0-1cd8-48fc-942e-dc458f145e46"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:38.715569901Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1534,"task":"crutest"} {"msg":"The guest has requested activation with gen:88","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:38.715594296Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1534} {"msg":"3be26146-f77d-43c3-9190-d0f0242807e5 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:38.715658933Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1534,"session_id":"9c36d8a0-1cd8-48fc-942e-dc458f145e46"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:38.715676974Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1534,"client":"0","":"downstairs","session_id":"9c36d8a0-1cd8-48fc-942e-dc458f145e46"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:38.715700498Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1534,"client":"0","":"downstairs","session_id":"9c36d8a0-1cd8-48fc-942e-dc458f145e46"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:38.715714386Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1534,"client":"1","":"downstairs","session_id":"9c36d8a0-1cd8-48fc-942e-dc458f145e46"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:38.715728865Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1534,"client":"1","":"downstairs","session_id":"9c36d8a0-1cd8-48fc-942e-dc458f145e46"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:38.715741892Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1534,"client":"2","":"downstairs","session_id":"9c36d8a0-1cd8-48fc-942e-dc458f145e46"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:38.71575561Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1534,"client":"2","":"downstairs","session_id":"9c36d8a0-1cd8-48fc-942e-dc458f145e46"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:38.715768268Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1534,"":"io task","client":"2","":"downstairs","session_id":"9c36d8a0-1cd8-48fc-942e-dc458f145e46"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:38.715781475Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1534,"":"io task","client":"0","":"downstairs","session_id":"9c36d8a0-1cd8-48fc-942e-dc458f145e46"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:38.715794673Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1534,"":"io task","client":"1","":"downstairs","session_id":"9c36d8a0-1cd8-48fc-942e-dc458f145e46"} {"msg":"ds_connection connected from Ok(127.0.0.1:59242)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:38.715949954Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1534,"":"io task","client":"1","":"downstairs","session_id":"9c36d8a0-1cd8-48fc-942e-dc458f145e46"} {"msg":"ds_connection connected from Ok(127.0.0.1:55367)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:38.715993239Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1534,"":"io task","client":"0","":"downstairs","session_id":"9c36d8a0-1cd8-48fc-942e-dc458f145e46"} {"msg":"ds_connection connected from Ok(127.0.0.1:51728)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:38.716019024Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1534,"":"io task","client":"2","":"downstairs","session_id":"9c36d8a0-1cd8-48fc-942e-dc458f145e46"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:38.71669605Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1534,"client":"1","":"downstairs","session_id":"9c36d8a0-1cd8-48fc-942e-dc458f145e46"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:38.716734422Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1534,"client":"2","":"downstairs","session_id":"9c36d8a0-1cd8-48fc-942e-dc458f145e46"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:38.716787893Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1534,"client":"0","":"downstairs","session_id":"9c36d8a0-1cd8-48fc-942e-dc458f145e46"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:38.717367342Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1534,"client":"1","":"downstairs","session_id":"9c36d8a0-1cd8-48fc-942e-dc458f145e46"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:38.717395809Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1534,"client":"2","":"downstairs","session_id":"9c36d8a0-1cd8-48fc-942e-dc458f145e46"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:38.717420473Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1534,"client":"0","":"downstairs","session_id":"9c36d8a0-1cd8-48fc-942e-dc458f145e46"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:38.717938496Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1534,"session_id":"9c36d8a0-1cd8-48fc-942e-dc458f145e46"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:38.717973726Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1534,"session_id":"9c36d8a0-1cd8-48fc-942e-dc458f145e46"} {"msg":"[0]R flush_numbers[0..12]: [151, 141, 156, 154, 153, 154, 151, 156, 160, 154, 157, 153]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:38.718068672Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1534,"":"downstairs","":"downstairs","session_id":"9c36d8a0-1cd8-48fc-942e-dc458f145e46"} {"msg":"[0]R generation[0..12]: [82, 78, 86, 84, 83, 84, 82, 86, 86, 84, 86, 83]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:38.718095088Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1534,"":"downstairs","":"downstairs","session_id":"9c36d8a0-1cd8-48fc-942e-dc458f145e46"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:38.718111097Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1534,"":"downstairs","":"downstairs","session_id":"9c36d8a0-1cd8-48fc-942e-dc458f145e46"} {"msg":"[1]R flush_numbers[0..12]: [151, 141, 156, 154, 153, 154, 151, 156, 160, 154, 157, 153]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:38.718125085Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1534,"":"downstairs","":"downstairs","session_id":"9c36d8a0-1cd8-48fc-942e-dc458f145e46"} {"msg":"[1]R generation[0..12]: [82, 78, 86, 84, 83, 84, 82, 86, 86, 84, 86, 83]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:38.718141855Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1534,"":"downstairs","":"downstairs","session_id":"9c36d8a0-1cd8-48fc-942e-dc458f145e46"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:38.718155813Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1534,"":"downstairs","":"downstairs","session_id":"9c36d8a0-1cd8-48fc-942e-dc458f145e46"} {"msg":"[2]R flush_numbers[0..12]: [151, 141, 156, 154, 153, 154, 151, 156, 160, 154, 157, 153]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:38.718169091Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1534,"":"downstairs","":"downstairs","session_id":"9c36d8a0-1cd8-48fc-942e-dc458f145e46"} {"msg":"[2]R generation[0..12]: [82, 78, 86, 84, 83, 84, 82, 86, 86, 84, 86, 83]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:38.718182218Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1534,"":"downstairs","":"downstairs","session_id":"9c36d8a0-1cd8-48fc-942e-dc458f145e46"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:38.718195596Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1534,"":"downstairs","":"downstairs","session_id":"9c36d8a0-1cd8-48fc-942e-dc458f145e46"} {"msg":"Max found gen is 88","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:38.718208043Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1534,"":"downstairs","":"downstairs","session_id":"9c36d8a0-1cd8-48fc-942e-dc458f145e46"} {"msg":"Generation requested: 88 >= found:88","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:38.718220771Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1534,"":"downstairs","":"downstairs","session_id":"9c36d8a0-1cd8-48fc-942e-dc458f145e46"} Wait for a query_work_queue command to finish before sending IO {"msg":"Next flush: 162","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:38.718238912Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1534,"":"downstairs","":"downstairs","session_id":"9c36d8a0-1cd8-48fc-942e-dc458f145e46"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:38.718251349Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1534,"":"downstairs","":"downstairs","session_id":"9c36d8a0-1cd8-48fc-942e-dc458f145e46"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:38.718264306Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1534,"session_id":"9c36d8a0-1cd8-48fc-942e-dc458f145e46"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:38.718276063Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1534,"client":"0","":"downstairs","session_id":"9c36d8a0-1cd8-48fc-942e-dc458f145e46"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:38.718289191Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1534,"client":"1","":"downstairs","session_id":"9c36d8a0-1cd8-48fc-942e-dc458f145e46"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:38.718303019Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1534,"client":"2","":"downstairs","session_id":"9c36d8a0-1cd8-48fc-942e-dc458f145e46"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:38.718316107Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1534,"session_id":"9c36d8a0-1cd8-48fc-942e-dc458f145e46"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:38.718328114Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1534,"session_id":"9c36d8a0-1cd8-48fc-942e-dc458f145e46"} {"msg":"3be26146-f77d-43c3-9190-d0f0242807e5 is now active with session: 9c36d8a0-1cd8-48fc-942e-dc458f145e46","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:38.718342592Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1534,"session_id":"9c36d8a0-1cd8-48fc-942e-dc458f145e46"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:38.718354179Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1534,"session_id":"9c36d8a0-1cd8-48fc-942e-dc458f145e46"} {"msg":"The guest has finished waiting for activation with:88","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:38.718365826Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1534} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 199 len 40960 data: 11 11 9 9 6 5 6 4 5 7 02/30 Read block 513 len 12288 03/30 Flush 04/30 Write block 475 len 4096 data: 8 05/30 Write block 192 len 20480 data: 9 11 12 10 9 06/30 Write block 213 len 16384 data: 10 8 8 8 07/30 Write block 299 len 28672 data: 9 8 9 6 7 6 7 08/30 Read block 476 len 32768 09/30 Read block 453 len 8192 10/30 Read block 306 len 4096 11/30 Write block 372 len 4096 data: 11 12/30 Read block 61 len 24576 13/30 Write block 495 len 32768 data: 10 10 10 11 11 12 12 10 14/30 Write block 494 len 4096 data: 8 15/30 Write block 100 len 16384 data: 9 9 9 8 16/30 Read block 423 len 16384 17/30 Read block 237 len 24576 18/30 Read block 430 len 24576 19/30 Read block 400 len 40960 20/30 Flush 21/30 Write block 111 len 32768 data: 10 11 10 11 12 10 10 11 22/30 Read block 284 len 40960 23/30 Read block 158 len 8192 24/30 Write block 291 len 24576 data: 9 10 9 8 10 10 25/30 Write block 196 len 16384 data: 10 7 9 12 26/30 Read block 352 len 28672 27/30 Read block 177 len 24576 28/30 Write block 110 len 20480 data: 8 11 12 11 12 29/30 Read block 154 len 4096 30/30 Read block 570 len 36864 ---------------------------------------------------------------- Crucible gen:88 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1020 Acked Write 8 Done Done Done false 1021 Acked Read 10 Done Done Done false 1022 Acked Read 2 Done Done Done false 1023 Acked Write 6 Done Done Done false 1024 Acked Write 4 Done Done Done false 1025 Acked Read 7 Done Done Done false 1026 Acked Read 6 Done Done Done false 1027 Acked Write 5 Done Done Done false 1028 Acked Read 1 Done Done Done false 1029 Acked Read 9 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 10 9 9 28 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1019 1019 1019 Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:14:38.737486041Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1534} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 82 82 82 151 151 151 F F F 1 020-039 78 78 78 141 141 141 F F F 2 040-059 86 86 86 156 156 156 F F F 3 060-079 84 84 84 154 154 154 F F F 4 080-099 83 83 83 153 153 153 F F F 5 100-119 88 88 88 163 163 163 T T T 6 120-139 82 82 82 151 151 151 F F F 7 140-159 86 86 86 156 156 156 F F F 8 160-179 86 86 86 160 160 160 F F F 9 180-199 88 88 88 163 163 163 T T T 10 200-219 88 88 88 163 163 163 F F F 11 220-239 83 83 83 153 153 153 F F F 12 240-259 82 82 82 151 151 151 F F F 13 260-279 82 82 82 151 151 151 F F F 14 280-299 88 88 88 163 163 163 T T T 15 300-319 88 88 88 163 163 163 F F F 16 320-339 87 87 87 161 161 161 F F F 17 340-359 87 87 87 161 161 161 F F F 18 360-379 88 88 88 163 163 163 F F F 19 380-399 70 70 70 128 128 128 F F F 20 400-419 84 84 84 154 154 154 F F F 21 420-439 86 86 86 157 157 157 F F F 22 440-459 87 87 87 161 161 161 F F F 23 460-479 88 88 88 163 163 163 F F F 24 480-499 88 88 88 163 163 163 F F F 25 500-519 88 88 88 163 163 163 F F F 26 520-539 86 86 86 157 157 157 F F F 27 540-559 86 86 86 157 157 157 F F F 28 560-579 85 85 85 155 155 155 F F F 29 580-599 82 82 82 151 151 151 F F F Max gen: 88, Max flush: 163 A Difference in extent metadata is expected here On loop 44 Verifying data now No change after verify Loop: 44 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 82 82 82 151 151 151 F F F 1 020-039 78 78 78 141 141 141 F F F 2 040-059 86 86 86 156 156 156 F F F 3 060-079 84 84 84 154 154 154 F F F 4 080-099 83 83 83 153 153 153 F F F 5 100-119 89 89 89 164 164 164 F F F 6 120-139 82 82 82 151 151 151 F F F 7 140-159 86 86 86 156 156 156 F F F 8 160-179 86 86 86 160 160 160 F F F 9 180-199 89 89 89 164 164 164 F F F 10 200-219 88 88 88 163 163 163 F F F 11 220-239 83 83 83 153 153 153 F F F 12 240-259 82 82 82 151 151 151 F F F 13 260-279 82 82 82 151 151 151 F F F 14 280-299 89 89 89 164 164 164 F F F 15 300-319 88 88 88 163 163 163 F F F 16 320-339 87 87 87 161 161 161 F F F 17 340-359 87 87 87 161 161 161 F F F 18 360-379 88 88 88 163 163 163 F F F 19 380-399 70 70 70 128 128 128 F F F 20 400-419 84 84 84 154 154 154 F F F 21 420-439 86 86 86 157 157 157 F F F 22 440-459 87 87 87 161 161 161 F F F 23 460-479 88 88 88 163 163 163 F F F 24 480-499 88 88 88 163 163 163 F F F 25 500-519 88 88 88 163 163 163 F F F 26 520-539 86 86 86 157 157 157 F F F 27 540-559 86 86 86 157 157 157 F F F 28 560-579 85 85 85 155 155 155 F F F 29 580-599 82 82 82 151 151 151 F F F Max gen: 89, Max flush: 164 Begin loop 45 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:42.993829831Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1542,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:43.038317213Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1542,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:43.038969524Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1542} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"abd5430f1508af0bc008143ea2566277ef714ec2\",\n git_commit_timestamp: \"2025-01-17T20:55:03.000000000Z\",\n git_branch: \"alan/tokio-to-140\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:43.039017842Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1542} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:43.039040986Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1542} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:43.039056525Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1542} {"msg":"Crucible 387c57b4-49f9-4686-bb37-5fb469c8542f has session id: 34c40c42-9ea0-4126-9e03-dedf687ab2b3","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:43.039499824Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1542,"session_id":"34c40c42-9ea0-4126-9e03-dedf687ab2b3"} {"msg":"Upstairs opts: Upstairs UUID: 387c57b4-49f9-4686-bb37-5fb469c8542f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:43.039540117Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1542,"session_id":"34c40c42-9ea0-4126-9e03-dedf687ab2b3"} {"msg":"Crucible stats registered with UUID: 387c57b4-49f9-4686-bb37-5fb469c8542f","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:43.039563911Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1542,"session_id":"34c40c42-9ea0-4126-9e03-dedf687ab2b3"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:14:43.039584093Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1542,"session_id":"34c40c42-9ea0-4126-9e03-dedf687ab2b3"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:43.040165363Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1542,"task":"crutest"} {"msg":"The guest has requested activation with gen:90","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:43.040190608Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1542} {"msg":"387c57b4-49f9-4686-bb37-5fb469c8542f active request set","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:43.04026412Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1542,"session_id":"34c40c42-9ea0-4126-9e03-dedf687ab2b3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:43.040281511Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1542,"client":"0","":"downstairs","session_id":"34c40c42-9ea0-4126-9e03-dedf687ab2b3"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:43.040305375Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1542,"client":"0","":"downstairs","session_id":"34c40c42-9ea0-4126-9e03-dedf687ab2b3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:43.040319263Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1542,"client":"1","":"downstairs","session_id":"34c40c42-9ea0-4126-9e03-dedf687ab2b3"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:43.040333071Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1542,"client":"1","":"downstairs","session_id":"34c40c42-9ea0-4126-9e03-dedf687ab2b3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:43.040346059Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1542,"client":"2","":"downstairs","session_id":"34c40c42-9ea0-4126-9e03-dedf687ab2b3"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:43.040363559Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1542,"client":"2","":"downstairs","session_id":"34c40c42-9ea0-4126-9e03-dedf687ab2b3"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:43.04038234Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1542,"":"io task","client":"2","":"downstairs","session_id":"34c40c42-9ea0-4126-9e03-dedf687ab2b3"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:43.040408335Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1542,"":"io task","client":"0","":"downstairs","session_id":"34c40c42-9ea0-4126-9e03-dedf687ab2b3"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:43.040433399Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1542,"":"io task","client":"1","":"downstairs","session_id":"34c40c42-9ea0-4126-9e03-dedf687ab2b3"} {"msg":"ds_connection connected from Ok(127.0.0.1:45369)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:43.040459185Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1542,"":"io task","client":"2","":"downstairs","session_id":"34c40c42-9ea0-4126-9e03-dedf687ab2b3"} {"msg":"ds_connection connected from Ok(127.0.0.1:63877)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:43.040692871Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1542,"":"io task","client":"1","":"downstairs","session_id":"34c40c42-9ea0-4126-9e03-dedf687ab2b3"} {"msg":"ds_connection connected from Ok(127.0.0.1:57438)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:43.040730133Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1542,"":"io task","client":"0","":"downstairs","session_id":"34c40c42-9ea0-4126-9e03-dedf687ab2b3"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:43.041212615Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1542,"client":"0","":"downstairs","session_id":"34c40c42-9ea0-4126-9e03-dedf687ab2b3"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:43.041252428Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1542,"client":"1","":"downstairs","session_id":"34c40c42-9ea0-4126-9e03-dedf687ab2b3"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:43.041268538Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1542,"client":"2","":"downstairs","session_id":"34c40c42-9ea0-4126-9e03-dedf687ab2b3"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:43.041898646Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1542,"client":"1","":"downstairs","session_id":"34c40c42-9ea0-4126-9e03-dedf687ab2b3"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:43.042047973Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1542,"client":"0","":"downstairs","session_id":"34c40c42-9ea0-4126-9e03-dedf687ab2b3"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:43.042075979Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1542,"client":"2","":"downstairs","session_id":"34c40c42-9ea0-4126-9e03-dedf687ab2b3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:43.042426314Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1542,"session_id":"34c40c42-9ea0-4126-9e03-dedf687ab2b3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:43.042587969Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1542,"session_id":"34c40c42-9ea0-4126-9e03-dedf687ab2b3"} {"msg":"[0]R flush_numbers[0..12]: [151, 141, 156, 154, 153, 164, 151, 156, 160, 164, 163, 153]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:43.042712271Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1542,"":"downstairs","":"downstairs","session_id":"34c40c42-9ea0-4126-9e03-dedf687ab2b3"} {"msg":"[0]R generation[0..12]: [82, 78, 86, 84, 83, 89, 82, 86, 86, 89, 88, 83]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:43.042732994Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1542,"":"downstairs","":"downstairs","session_id":"34c40c42-9ea0-4126-9e03-dedf687ab2b3"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:43.042747992Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1542,"":"downstairs","":"downstairs","session_id":"34c40c42-9ea0-4126-9e03-dedf687ab2b3"} {"msg":"[1]R flush_numbers[0..12]: [151, 141, 156, 154, 153, 164, 151, 156, 160, 164, 163, 153]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:43.04276185Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1542,"":"downstairs","":"downstairs","session_id":"34c40c42-9ea0-4126-9e03-dedf687ab2b3"} {"msg":"[1]R generation[0..12]: [82, 78, 86, 84, 83, 89, 82, 86, 86, 89, 88, 83]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:43.042775278Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1542,"":"downstairs","":"downstairs","session_id":"34c40c42-9ea0-4126-9e03-dedf687ab2b3"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:43.042788926Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1542,"":"downstairs","":"downstairs","session_id":"34c40c42-9ea0-4126-9e03-dedf687ab2b3"} {"msg":"[2]R flush_numbers[0..12]: [151, 141, 156, 154, 153, 164, 151, 156, 160, 164, 163, 153]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:43.042802244Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1542,"":"downstairs","":"downstairs","session_id":"34c40c42-9ea0-4126-9e03-dedf687ab2b3"} {"msg":"[2]R generation[0..12]: [82, 78, 86, 84, 83, 89, 82, 86, 86, 89, 88, 83]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:43.042815452Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1542,"":"downstairs","":"downstairs","session_id":"34c40c42-9ea0-4126-9e03-dedf687ab2b3"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:43.04282893Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1542,"":"downstairs","":"downstairs","session_id":"34c40c42-9ea0-4126-9e03-dedf687ab2b3"} {"msg":"Max found gen is 90","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:43.042841587Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1542,"":"downstairs","":"downstairs","session_id":"34c40c42-9ea0-4126-9e03-dedf687ab2b3"} {"msg":"Generation requested: 90 >= found:90","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:43.042854374Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1542,"":"downstairs","":"downstairs","session_id":"34c40c42-9ea0-4126-9e03-dedf687ab2b3"} {"msg":"Next flush: 165","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-17T21:14:43.042866922Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1542,"":"downstairs","":"downstairs","session_id":"34c40c42-9ea0-4126-9e03-dedf687ab2b3"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:43.042891826Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1542,"":"downstairs","":"downstairs","session_id":"34c40c42-9ea0-4126-9e03-dedf687ab2b3"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:43.042921053Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1542,"session_id":"34c40c42-9ea0-4126-9e03-dedf687ab2b3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:43.042944757Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1542,"client":"0","":"downstairs","session_id":"34c40c42-9ea0-4126-9e03-dedf687ab2b3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:43.042967551Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1542,"client":"1","":"downstairs","session_id":"34c40c42-9ea0-4126-9e03-dedf687ab2b3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:43.042990784Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1542,"client":"2","":"downstairs","session_id":"34c40c42-9ea0-4126-9e03-dedf687ab2b3"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:43.043015789Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1542,"session_id":"34c40c42-9ea0-4126-9e03-dedf687ab2b3"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:43.043039193Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1542,"session_id":"34c40c42-9ea0-4126-9e03-dedf687ab2b3"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 {"msg":"387c57b4-49f9-4686-bb37-5fb469c8542f is now active with session: 34c40c42-9ea0-4126-9e03-dedf687ab2b3","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:43.043065378Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1542,"session_id":"34c40c42-9ea0-4126-9e03-dedf687ab2b3"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:43.04308579Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1542,"session_id":"34c40c42-9ea0-4126-9e03-dedf687ab2b3"} {"msg":"The guest has finished waiting for activation with:90","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:43.043098847Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1542} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 579 len 28672 data: 8 8 7 6 7 8 7 03/30 Read block 512 len 40960 04/30 Write block 135 len 20480 data: 4 6 6 6 7 05/30 Read block 422 len 24576 06/30 Read block 308 len 24576 07/30 Read block 480 len 40960 08/30 Write block 375 len 40960 data: 12 11 13 14 14 11 9 7 8 8 09/30 Read block 442 len 24576 10/30 Write block 339 len 12288 data: 12 12 10 11/30 Write block 214 len 20480 data: 9 9 9 7 6 12/30 Write block 125 len 40960 data: 4 4 5 5 5 7 5 6 5 5 13/30 Read block 81 len 20480 14/30 Read block 368 len 4096 15/30 Read block 246 len 32768 16/30 Read block 560 len 28672 17/30 Write block 532 len 24576 data: 6 5 4 4 5 6 18/30 Write block 468 len 8192 data: 9 10 19/30 Flush 20/30 Write block 281 len 40960 data: 8 8 6 7 7 6 5 6 6 7 21/30 Write block 352 len 28672 data: 8 8 9 9 10 10 9 22/30 Read block 161 len 16384 23/30 Write block 594 len 12288 data: 4 4 4 24/30 Write block 530 len 24576 data: 4 6 7 6 5 5 25/30 Write block 71 len 16384 data: 9 8 7 7 26/30 Flush 27/30 Write block 357 len 4096 data: 11 28/30 Read block 285 len 28672 29/30 Read block 244 len 12288 30/30 Write block 586 len 32768 data: 7 7 7 7 7 4 6 5 ---------------------------------------------------------------- Crucible gen:90 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 1 Done Done Done false 1027 Acked Read 7 Done Done Done false 1028 Acked Read 3 Done Done Done false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:14:43.063642375Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1542} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 82 82 82 151 151 151 F F F 1 020-039 78 78 78 141 141 141 F F F 2 040-059 86 86 86 156 156 156 F F F 3 060-079 90 90 90 167 167 167 F F F 4 080-099 83 83 83 153 153 153 F F F 5 100-119 89 89 89 164 164 164 F F F 6 120-139 90 90 90 166 166 166 F F F 7 140-159 86 86 86 156 156 156 F F F 8 160-179 86 86 86 160 160 160 F F F 9 180-199 89 89 89 164 164 164 F F F 10 200-219 90 90 90 166 166 166 F F F 11 220-239 83 83 83 153 153 153 F F F 12 240-259 82 82 82 151 151 151 F F F 13 260-279 82 82 82 151 151 151 F F F 14 280-299 90 90 90 167 167 167 F F F 15 300-319 88 88 88 163 163 163 F F F 16 320-339 90 90 90 166 166 166 F F F 17 340-359 90 90 90 167 167 167 T T T 18 360-379 90 90 90 166 166 166 F F F 19 380-399 90 90 90 166 166 166 F F F 20 400-419 84 84 84 154 154 154 F F F 21 420-439 86 86 86 157 157 157 F F F 22 440-459 87 87 87 161 161 161 F F F 23 460-479 90 90 90 166 166 166 F F F 24 480-499 88 88 88 163 163 163 F F F 25 500-519 88 88 88 163 163 163 F F F 26 520-539 90 90 90 167 167 167 F F F 27 540-559 86 86 86 157 157 157 F F F 28 560-579 90 90 90 166 166 166 F F F 29 580-599 90 90 90 167 167 167 T T T Max gen: 90, Max flush: 167 A Difference in extent metadata is expected here On loop 45 Verifying data now No change after verify Loop: 45 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 82 82 82 151 151 151 F F F 1 020-039 78 78 78 141 141 141 F F F 2 040-059 86 86 86 156 156 156 F F F 3 060-079 90 90 90 167 167 167 F F F 4 080-099 83 83 83 153 153 153 F F F 5 100-119 89 89 89 164 164 164 F F F 6 120-139 90 90 90 166 166 166 F F F 7 140-159 86 86 86 156 156 156 F F F 8 160-179 86 86 86 160 160 160 F F F 9 180-199 89 89 89 164 164 164 F F F 10 200-219 90 90 90 166 166 166 F F F 11 220-239 83 83 83 153 153 153 F F F 12 240-259 82 82 82 151 151 151 F F F 13 260-279 82 82 82 151 151 151 F F F 14 280-299 90 90 90 167 167 167 F F F 15 300-319 88 88 88 163 163 163 F F F 16 320-339 90 90 90 166 166 166 F F F 17 340-359 91 91 91 168 168 168 F F F 18 360-379 90 90 90 166 166 166 F F F 19 380-399 90 90 90 166 166 166 F F F 20 400-419 84 84 84 154 154 154 F F F 21 420-439 86 86 86 157 157 157 F F F 22 440-459 87 87 87 161 161 161 F F F 23 460-479 90 90 90 166 166 166 F F F 24 480-499 88 88 88 163 163 163 F F F 25 500-519 88 88 88 163 163 163 F F F 26 520-539 90 90 90 167 167 167 F F F 27 540-559 86 86 86 157 157 157 F F F 28 560-579 90 90 90 166 166 166 F F F 29 580-599 91 91 91 168 168 168 F F F Max gen: 91, Max flush: 168 Begin loop 46 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:47.289519448Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1550,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:47.335436175Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1550,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:47.336116372Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1550} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"abd5430f1508af0bc008143ea2566277ef714ec2\",\n git_commit_timestamp: \"2025-01-17T20:55:03.000000000Z\",\n git_branch: \"alan/tokio-to-140\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:47.336156556Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1550} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:47.336171094Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1550} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:47.336185262Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1550} {"msg":"Crucible 14e6285c-6269-46ec-b69d-efcc6a1de669 has session id: e9f18386-27af-47dc-a78c-16c0d4063566","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:47.336661031Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1550,"session_id":"e9f18386-27af-47dc-a78c-16c0d4063566"} {"msg":"Upstairs opts: Upstairs UUID: 14e6285c-6269-46ec-b69d-efcc6a1de669, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:47.336701054Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1550,"session_id":"e9f18386-27af-47dc-a78c-16c0d4063566"} {"msg":"Crucible stats registered with UUID: 14e6285c-6269-46ec-b69d-efcc6a1de669","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:47.336725288Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1550,"session_id":"e9f18386-27af-47dc-a78c-16c0d4063566"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:14:47.33674657Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1550,"session_id":"e9f18386-27af-47dc-a78c-16c0d4063566"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:47.337275219Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1550,"task":"crutest"} {"msg":"The guest has requested activation with gen:92","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:47.337309139Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1550} {"msg":"14e6285c-6269-46ec-b69d-efcc6a1de669 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:47.337397321Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1550,"session_id":"e9f18386-27af-47dc-a78c-16c0d4063566"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:47.337414851Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1550,"client":"0","":"downstairs","session_id":"e9f18386-27af-47dc-a78c-16c0d4063566"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:47.337442067Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1550,"client":"0","":"downstairs","session_id":"e9f18386-27af-47dc-a78c-16c0d4063566"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:47.33746371Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1550,"client":"1","":"downstairs","session_id":"e9f18386-27af-47dc-a78c-16c0d4063566"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:47.337478658Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1550,"client":"1","":"downstairs","session_id":"e9f18386-27af-47dc-a78c-16c0d4063566"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:47.337492917Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1550,"client":"2","":"downstairs","session_id":"e9f18386-27af-47dc-a78c-16c0d4063566"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:47.33751598Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1550,"client":"2","":"downstairs","session_id":"e9f18386-27af-47dc-a78c-16c0d4063566"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:47.337538874Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1550,"":"io task","client":"2","":"downstairs","session_id":"e9f18386-27af-47dc-a78c-16c0d4063566"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:47.337613427Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1550,"":"io task","client":"0","":"downstairs","session_id":"e9f18386-27af-47dc-a78c-16c0d4063566"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:47.337647057Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1550,"":"io task","client":"1","":"downstairs","session_id":"e9f18386-27af-47dc-a78c-16c0d4063566"} {"msg":"ds_connection connected from Ok(127.0.0.1:37394)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:47.337671771Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1550,"":"io task","client":"2","":"downstairs","session_id":"e9f18386-27af-47dc-a78c-16c0d4063566"} {"msg":"ds_connection connected from Ok(127.0.0.1:58155)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:47.337696706Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1550,"":"io task","client":"0","":"downstairs","session_id":"e9f18386-27af-47dc-a78c-16c0d4063566"} {"msg":"ds_connection connected from Ok(127.0.0.1:33318)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:47.337726113Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1550,"":"io task","client":"1","":"downstairs","session_id":"e9f18386-27af-47dc-a78c-16c0d4063566"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:47.338364446Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1550,"client":"0","":"downstairs","session_id":"e9f18386-27af-47dc-a78c-16c0d4063566"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:47.338494032Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1550,"client":"2","":"downstairs","session_id":"e9f18386-27af-47dc-a78c-16c0d4063566"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:47.338650903Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1550,"client":"1","":"downstairs","session_id":"e9f18386-27af-47dc-a78c-16c0d4063566"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:47.339075872Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1550,"client":"0","":"downstairs","session_id":"e9f18386-27af-47dc-a78c-16c0d4063566"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:47.339115565Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1550,"client":"2","":"downstairs","session_id":"e9f18386-27af-47dc-a78c-16c0d4063566"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:47.339384042Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1550,"client":"1","":"downstairs","session_id":"e9f18386-27af-47dc-a78c-16c0d4063566"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:47.339597377Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1550,"session_id":"e9f18386-27af-47dc-a78c-16c0d4063566"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:47.339685628Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1550,"session_id":"e9f18386-27af-47dc-a78c-16c0d4063566"} {"msg":"[0]R flush_numbers[0..12]: [151, 141, 156, 167, 153, 164, 166, 156, 160, 164, 166, 153]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:47.339934674Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1550,"":"downstairs","":"downstairs","session_id":"e9f18386-27af-47dc-a78c-16c0d4063566"} {"msg":"[0]R generation[0..12]: [82, 78, 86, 90, 83, 89, 90, 86, 86, 89, 90, 83]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:47.339953525Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1550,"":"downstairs","":"downstairs","session_id":"e9f18386-27af-47dc-a78c-16c0d4063566"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:47.339967783Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1550,"":"downstairs","":"downstairs","session_id":"e9f18386-27af-47dc-a78c-16c0d4063566"} {"msg":"[1]R flush_numbers[0..12]: [151, 141, 156, 167, 153, 164, 166, 156, 160, 164, 166, 153]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:47.339981471Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1550,"":"downstairs","":"downstairs","session_id":"e9f18386-27af-47dc-a78c-16c0d4063566"} {"msg":"[1]R generation[0..12]: [82, 78, 86, 90, 83, 89, 90, 86, 86, 89, 90, 83]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:47.340007487Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1550,"":"downstairs","":"downstairs","session_id":"e9f18386-27af-47dc-a78c-16c0d4063566"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:47.340023266Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1550,"":"downstairs","":"downstairs","session_id":"e9f18386-27af-47dc-a78c-16c0d4063566"} {"msg":"[2]R flush_numbers[0..12]: [151, 141, 156, 167, 153, 164, 166, 156, 160, 164, 166, 153]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:47.340036613Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1550,"":"downstairs","":"downstairs","session_id":"e9f18386-27af-47dc-a78c-16c0d4063566"} {"msg":"[2]R generation[0..12]: [82, 78, 86, 90, 83, 89, 90, 86, 86, 89, 90, 83]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:47.340049651Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1550,"":"downstairs","":"downstairs","session_id":"e9f18386-27af-47dc-a78c-16c0d4063566"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:47.340063009Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1550,"":"downstairs","":"downstairs","session_id":"e9f18386-27af-47dc-a78c-16c0d4063566"} {"msg":"Max found gen is 92","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:47.340079478Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1550,"":"downstairs","":"downstairs","session_id":"e9f18386-27af-47dc-a78c-16c0d4063566"} {"msg":"Generation requested: 92 >= found:92","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:47.340092936Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1550,"":"downstairs","":"downstairs","session_id":"e9f18386-27af-47dc-a78c-16c0d4063566"} Wait for a query_work_queue command to finish before sending IO {"msg":"Next flush: 169","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:47.340113418Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1550,"":"downstairs","":"downstairs","session_id":"e9f18386-27af-47dc-a78c-16c0d4063566"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:47.340137222Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1550,"":"downstairs","":"downstairs","session_id":"e9f18386-27af-47dc-a78c-16c0d4063566"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:47.340162557Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1550,"session_id":"e9f18386-27af-47dc-a78c-16c0d4063566"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:47.340186261Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1550,"client":"0","":"downstairs","session_id":"e9f18386-27af-47dc-a78c-16c0d4063566"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:47.340212947Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1550,"client":"1","":"downstairs","session_id":"e9f18386-27af-47dc-a78c-16c0d4063566"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:47.340238902Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1550,"client":"2","":"downstairs","session_id":"e9f18386-27af-47dc-a78c-16c0d4063566"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:47.340265527Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1550,"session_id":"e9f18386-27af-47dc-a78c-16c0d4063566"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:47.340281957Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1550,"session_id":"e9f18386-27af-47dc-a78c-16c0d4063566"} {"msg":"14e6285c-6269-46ec-b69d-efcc6a1de669 is now active with session: e9f18386-27af-47dc-a78c-16c0d4063566","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:47.340294804Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1550,"session_id":"e9f18386-27af-47dc-a78c-16c0d4063566"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:47.340306521Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1550,"session_id":"e9f18386-27af-47dc-a78c-16c0d4063566"} {"msg":"The guest has finished waiting for activation with:92","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:47.340318418Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1550} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 500 len 16384 data: 13 13 11 11 03/30 Read block 587 len 4096 04/30 Read block 359 len 16384 05/30 Write block 99 len 28672 data: 8 10 10 10 9 9 10 06/30 Read block 548 len 28672 07/30 Read block 440 len 8192 08/30 Read block 76 len 36864 09/30 Write block 337 len 28672 data: 11 12 13 13 11 9 13 10/30 Read block 386 len 32768 11/30 Write block 334 len 4096 data: 9 12/30 Write block 363 len 32768 data: 5 4 4 4 3 3 4 7 13/30 Write block 7 len 8192 data: 7 6 14/30 Write block 540 len 28672 data: 6 6 7 9 8 8 10 15/30 Read block 534 len 4096 16/30 Write block 529 len 28672 data: 5 5 7 8 7 6 6 17/30 Read block 424 len 8192 18/30 Write block 92 len 28672 data: 9 9 10 10 9 6 9 19/30 Flush 20/30 Write block 95 len 20480 data: 11 10 7 10 9 21/30 Flush 22/30 Write block 569 len 24576 data: 7 7 6 4 4 5 23/30 Write block 348 len 12288 data: 8 7 9 24/30 Read block 240 len 8192 25/30 Read block 496 len 20480 26/30 Read block 328 len 24576 27/30 Write block 81 len 24576 data: 6 6 7 6 6 4 28/30 Write block 301 len 4096 data: 10 29/30 Write block 86 len 32768 data: 5 5 5 6 9 8 10 10 30/30 Read block 428 len 32768 ---------------------------------------------------------------- Crucible gen:92 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 6 Done Done Done false 1022 Acked Write 3 Done Done Done false 1023 Acked Read 2 Done Done Done false 1024 Acked Read 5 Done Done Done false 1025 Acked Read 6 Done Done Done false 1026 Acked Write 6 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Write 8 Done Sent Done false 1029 Acked Read 8 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 2 0 3 Done 8 7 9 24 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1020 1020 1020 Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:14:47.359532028Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1550} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 92 92 92 170 170 170 F F F 1 020-039 78 78 78 141 141 141 F F F 2 040-059 86 86 86 156 156 156 F F F 3 060-079 90 90 90 167 167 167 F F F 4 080-099 92 92 92 171 171 171 T T T 5 100-119 92 92 92 170 170 170 F F F 6 120-139 90 90 90 166 166 166 F F F 7 140-159 86 86 86 156 156 156 F F F 8 160-179 86 86 86 160 160 160 F F F 9 180-199 89 89 89 164 164 164 F F F 10 200-219 90 90 90 166 166 166 F F F 11 220-239 83 83 83 153 153 153 F F F 12 240-259 82 82 82 151 151 151 F F F 13 260-279 82 82 82 151 151 151 F F F 14 280-299 90 90 90 167 167 167 F F F 15 300-319 88 88 88 163 163 163 T T T 16 320-339 92 92 92 170 170 170 F F F 17 340-359 92 92 92 170 170 170 T T T 18 360-379 92 92 92 170 170 170 F F F 19 380-399 90 90 90 166 166 166 F F F 20 400-419 84 84 84 154 154 154 F F F 21 420-439 86 86 86 157 157 157 F F F 22 440-459 87 87 87 161 161 161 F F F 23 460-479 90 90 90 166 166 166 F F F 24 480-499 88 88 88 163 163 163 F F F 25 500-519 92 92 92 170 170 170 F F F 26 520-539 92 92 92 170 170 170 F F F 27 540-559 92 92 92 170 170 170 F F F 28 560-579 90 90 90 166 166 166 T T T 29 580-599 91 91 91 168 168 168 F F F Max gen: 92, Max flush: 171 A Difference in extent metadata is expected here On loop 46 Verifying data now No change after verify Loop: 46 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 92 92 92 170 170 170 F F F 1 020-039 78 78 78 141 141 141 F F F 2 040-059 86 86 86 156 156 156 F F F 3 060-079 90 90 90 167 167 167 F F F 4 080-099 93 93 93 172 172 172 F F F 5 100-119 92 92 92 170 170 170 F F F 6 120-139 90 90 90 166 166 166 F F F 7 140-159 86 86 86 156 156 156 F F F 8 160-179 86 86 86 160 160 160 F F F 9 180-199 89 89 89 164 164 164 F F F 10 200-219 90 90 90 166 166 166 F F F 11 220-239 83 83 83 153 153 153 F F F 12 240-259 82 82 82 151 151 151 F F F 13 260-279 82 82 82 151 151 151 F F F 14 280-299 90 90 90 167 167 167 F F F 15 300-319 93 93 93 172 172 172 F F F 16 320-339 92 92 92 170 170 170 F F F 17 340-359 93 93 93 172 172 172 F F F 18 360-379 92 92 92 170 170 170 F F F 19 380-399 90 90 90 166 166 166 F F F 20 400-419 84 84 84 154 154 154 F F F 21 420-439 86 86 86 157 157 157 F F F 22 440-459 87 87 87 161 161 161 F F F 23 460-479 90 90 90 166 166 166 F F F 24 480-499 88 88 88 163 163 163 F F F 25 500-519 92 92 92 170 170 170 F F F 26 520-539 92 92 92 170 170 170 F F F 27 540-559 92 92 92 170 170 170 F F F 28 560-579 93 93 93 172 172 172 F F F 29 580-599 91 91 91 168 168 168 F F F Max gen: 93, Max flush: 172 Begin loop 47 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:51.593092281Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1558,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:51.637083193Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1558,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:51.637706487Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1558} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"abd5430f1508af0bc008143ea2566277ef714ec2\",\n git_commit_timestamp: \"2025-01-17T20:55:03.000000000Z\",\n git_branch: \"alan/tokio-to-140\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:51.637747861Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1558} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:51.63776298Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1558} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:51.637777169Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1558} {"msg":"Crucible e7421964-3819-488f-acb2-4eb44c59945b has session id: e1209098-1e0a-46b3-9d08-7691b1d99be8","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:51.638368414Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1558,"session_id":"e1209098-1e0a-46b3-9d08-7691b1d99be8"} {"msg":"Upstairs opts: Upstairs UUID: e7421964-3819-488f-acb2-4eb44c59945b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:51.638407307Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1558,"session_id":"e1209098-1e0a-46b3-9d08-7691b1d99be8"} {"msg":"Crucible stats registered with UUID: e7421964-3819-488f-acb2-4eb44c59945b","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:51.638431921Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1558,"session_id":"e1209098-1e0a-46b3-9d08-7691b1d99be8"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:14:51.638445489Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1558,"session_id":"e1209098-1e0a-46b3-9d08-7691b1d99be8"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:51.638939588Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1558,"task":"crutest"} {"msg":"The guest has requested activation with gen:94","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:51.638972377Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1558} {"msg":"e7421964-3819-488f-acb2-4eb44c59945b active request set","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:51.639026799Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1558,"session_id":"e1209098-1e0a-46b3-9d08-7691b1d99be8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:51.63904546Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1558,"client":"0","":"downstairs","session_id":"e1209098-1e0a-46b3-9d08-7691b1d99be8"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:51.639078449Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1558,"client":"0","":"downstairs","session_id":"e1209098-1e0a-46b3-9d08-7691b1d99be8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:51.639106466Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1558,"client":"1","":"downstairs","session_id":"e1209098-1e0a-46b3-9d08-7691b1d99be8"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:51.639127818Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1558,"client":"1","":"downstairs","session_id":"e1209098-1e0a-46b3-9d08-7691b1d99be8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:51.639188463Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1558,"client":"2","":"downstairs","session_id":"e1209098-1e0a-46b3-9d08-7691b1d99be8"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:51.639215559Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1558,"client":"2","":"downstairs","session_id":"e1209098-1e0a-46b3-9d08-7691b1d99be8"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:51.639240024Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1558,"":"io task","client":"2","":"downstairs","session_id":"e1209098-1e0a-46b3-9d08-7691b1d99be8"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:51.639263838Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1558,"":"io task","client":"0","":"downstairs","session_id":"e1209098-1e0a-46b3-9d08-7691b1d99be8"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:51.639287191Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1558,"":"io task","client":"1","":"downstairs","session_id":"e1209098-1e0a-46b3-9d08-7691b1d99be8"} {"msg":"ds_connection connected from Ok(127.0.0.1:61614)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:51.639313396Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1558,"":"io task","client":"2","":"downstairs","session_id":"e1209098-1e0a-46b3-9d08-7691b1d99be8"} {"msg":"ds_connection connected from Ok(127.0.0.1:46372)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:51.639343814Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1558,"":"io task","client":"1","":"downstairs","session_id":"e1209098-1e0a-46b3-9d08-7691b1d99be8"} {"msg":"ds_connection connected from Ok(127.0.0.1:62126)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:51.639417857Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1558,"":"io task","client":"0","":"downstairs","session_id":"e1209098-1e0a-46b3-9d08-7691b1d99be8"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:51.640064826Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1558,"client":"2","":"downstairs","session_id":"e1209098-1e0a-46b3-9d08-7691b1d99be8"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:51.640093382Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1558,"client":"0","":"downstairs","session_id":"e1209098-1e0a-46b3-9d08-7691b1d99be8"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:51.640666637Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1558,"client":"0","":"downstairs","session_id":"e1209098-1e0a-46b3-9d08-7691b1d99be8"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:51.640794722Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1558,"client":"2","":"downstairs","session_id":"e1209098-1e0a-46b3-9d08-7691b1d99be8"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:51.64104674Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1558,"client":"1","":"downstairs","session_id":"e1209098-1e0a-46b3-9d08-7691b1d99be8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:51.641200089Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1558,"session_id":"e1209098-1e0a-46b3-9d08-7691b1d99be8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:51.641318608Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1558,"session_id":"e1209098-1e0a-46b3-9d08-7691b1d99be8"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:51.641639306Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1558,"client":"1","":"downstairs","session_id":"e1209098-1e0a-46b3-9d08-7691b1d99be8"} {"msg":"[0]R flush_numbers[0..12]: [170, 141, 156, 167, 172, 170, 166, 156, 160, 164, 166, 153]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:51.642158719Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1558,"":"downstairs","":"downstairs","session_id":"e1209098-1e0a-46b3-9d08-7691b1d99be8"} {"msg":"[0]R generation[0..12]: [92, 78, 86, 90, 93, 92, 90, 86, 86, 89, 90, 83]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:51.642178151Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1558,"":"downstairs","":"downstairs","session_id":"e1209098-1e0a-46b3-9d08-7691b1d99be8"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:51.642192269Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1558,"":"downstairs","":"downstairs","session_id":"e1209098-1e0a-46b3-9d08-7691b1d99be8"} {"msg":"[1]R flush_numbers[0..12]: [170, 141, 156, 167, 172, 170, 166, 156, 160, 164, 166, 153]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:51.642205727Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1558,"":"downstairs","":"downstairs","session_id":"e1209098-1e0a-46b3-9d08-7691b1d99be8"} {"msg":"[1]R generation[0..12]: [92, 78, 86, 90, 93, 92, 90, 86, 86, 89, 90, 83]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:51.642221967Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1558,"":"downstairs","":"downstairs","session_id":"e1209098-1e0a-46b3-9d08-7691b1d99be8"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:51.642235795Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1558,"":"downstairs","":"downstairs","session_id":"e1209098-1e0a-46b3-9d08-7691b1d99be8"} {"msg":"[2]R flush_numbers[0..12]: [170, 141, 156, 167, 172, 170, 166, 156, 160, 164, 166, 153]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:51.642249062Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1558,"":"downstairs","":"downstairs","session_id":"e1209098-1e0a-46b3-9d08-7691b1d99be8"} {"msg":"[2]R generation[0..12]: [92, 78, 86, 90, 93, 92, 90, 86, 86, 89, 90, 83]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:51.64226253Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1558,"":"downstairs","":"downstairs","session_id":"e1209098-1e0a-46b3-9d08-7691b1d99be8"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:51.642276338Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1558,"":"downstairs","":"downstairs","session_id":"e1209098-1e0a-46b3-9d08-7691b1d99be8"} {"msg":"Max found gen is 94","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:51.642289156Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1558,"":"downstairs","":"downstairs","session_id":"e1209098-1e0a-46b3-9d08-7691b1d99be8"} {"msg":"Generation requested: 94 >= found:94","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:51.642301923Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1558,"":"downstairs","":"downstairs","session_id":"e1209098-1e0a-46b3-9d08-7691b1d99be8"} {"msg":"Next flush: 173","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:51.64231423Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1558,"":"downstairs","":"downstairs","session_id":"e1209098-1e0a-46b3-9d08-7691b1d99be8"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-17T21:14:51.642326888Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1558,"":"downstairs","":"downstairs","session_id":"e1209098-1e0a-46b3-9d08-7691b1d99be8"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:51.642342627Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1558,"session_id":"e1209098-1e0a-46b3-9d08-7691b1d99be8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:51.642354264Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1558,"client":"0","":"downstairs","session_id":"e1209098-1e0a-46b3-9d08-7691b1d99be8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:51.642367081Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1558,"client":"1","":"downstairs","session_id":"e1209098-1e0a-46b3-9d08-7691b1d99be8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:51.642379679Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1558,"client":"2","":"downstairs","session_id":"e1209098-1e0a-46b3-9d08-7691b1d99be8"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:51.642392386Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1558,"session_id":"e1209098-1e0a-46b3-9d08-7691b1d99be8"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:51.642404203Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1558,"session_id":"e1209098-1e0a-46b3-9d08-7691b1d99be8"} {"msg":"e7421964-3819-488f-acb2-4eb44c59945b is now active with session: e1209098-1e0a-46b3-9d08-7691b1d99be8","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:51.642419322Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1558,"session_id":"e1209098-1e0a-46b3-9d08-7691b1d99be8"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:51.642431049Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1558,"session_id":"e1209098-1e0a-46b3-9d08-7691b1d99be8"} {"msg":"The guest has finished waiting for activation with:94","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:51.642442785Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1558} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 537 len 28672 data: 7 5 6 7 7 8 10 02/30 Flush 03/30 Write block 167 len 24576 data: 7 8 7 8 8 8 04/30 Read block 581 len 16384 05/30 Read block 237 len 4096 06/30 Write block 288 len 12288 data: 7 7 8 07/30 Read block 230 len 40960 08/30 Write block 226 len 16384 data: 8 8 7 7 09/30 Read block 137 len 16384 10/30 Read block 198 len 12288 11/30 Read block 100 len 36864 12/30 Read block 385 len 16384 13/30 Read block 115 len 40960 14/30 Write block 458 len 28672 data: 9 9 9 10 10 9 9 15/30 Write block 291 len 12288 data: 10 11 10 16/30 Read block 482 len 16384 17/30 Write block 300 len 36864 data: 9 11 7 8 7 8 7 7 5 18/30 Write block 0 len 24576 data: 3 5 5 5 6 7 19/30 Read block 388 len 20480 20/30 Write block 533 len 36864 data: 8 7 7 6 8 6 7 8 8 21/30 Flush 22/30 Write block 303 len 28672 data: 9 8 9 8 8 6 5 23/30 Read block 175 len 40960 24/30 Write block 88 len 28672 data: 6 7 10 9 11 11 11 25/30 Write block 569 len 32768 data: 8 8 7 5 5 6 5 6 26/30 Read block 93 len 24576 27/30 Read block 346 len 16384 28/30 Read block 344 len 20480 29/30 Read block 482 len 8192 30/30 Read block 396 len 20480 ---------------------------------------------------------------- Crucible gen:94 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 7 Done Done Done false 1022 Acked Read 10 Done Done Done false 1023 Acked Write 7 Done Done Done false 1024 Acked Write 8 Done Done Done false 1025 Acked Read 6 Done Done Done false 1026 Acked Read 4 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Read 5 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 0 1 Done 9 8 9 26 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1020 1020 1020 Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:14:51.663930785Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1558} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 94 94 94 174 174 174 F F F 1 020-039 78 78 78 141 141 141 F F F 2 040-059 86 86 86 156 156 156 F F F 3 060-079 90 90 90 167 167 167 F F F 4 080-099 93 93 93 172 172 172 T T T 5 100-119 92 92 92 170 170 170 F F F 6 120-139 90 90 90 166 166 166 F F F 7 140-159 86 86 86 156 156 156 F F F 8 160-179 94 94 94 174 174 174 F F F 9 180-199 89 89 89 164 164 164 F F F 10 200-219 90 90 90 166 166 166 F F F 11 220-239 94 94 94 174 174 174 F F F 12 240-259 82 82 82 151 151 151 F F F 13 260-279 82 82 82 151 151 151 F F F 14 280-299 94 94 94 174 174 174 F F F 15 300-319 94 94 94 174 174 174 T T T 16 320-339 92 92 92 170 170 170 F F F 17 340-359 93 93 93 172 172 172 F F F 18 360-379 92 92 92 170 170 170 F F F 19 380-399 90 90 90 166 166 166 F F F 20 400-419 84 84 84 154 154 154 F F F 21 420-439 86 86 86 157 157 157 F F F 22 440-459 94 94 94 174 174 174 F F F 23 460-479 94 94 94 174 174 174 F F F 24 480-499 88 88 88 163 163 163 F F F 25 500-519 92 92 92 170 170 170 F F F 26 520-539 94 94 94 174 174 174 F F F 27 540-559 94 94 94 174 174 174 F F F 28 560-579 93 93 93 172 172 172 T T T 29 580-599 91 91 91 168 168 168 F F F Max gen: 94, Max flush: 174 A Difference in extent metadata is expected here On loop 47 Verifying data now No change after verify Loop: 47 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 94 94 94 174 174 174 F F F 1 020-039 78 78 78 141 141 141 F F F 2 040-059 86 86 86 156 156 156 F F F 3 060-079 90 90 90 167 167 167 F F F 4 080-099 95 95 95 175 175 175 F F F 5 100-119 92 92 92 170 170 170 F F F 6 120-139 90 90 90 166 166 166 F F F 7 140-159 86 86 86 156 156 156 F F F 8 160-179 94 94 94 174 174 174 F F F 9 180-199 89 89 89 164 164 164 F F F 10 200-219 90 90 90 166 166 166 F F F 11 220-239 94 94 94 174 174 174 F F F 12 240-259 82 82 82 151 151 151 F F F 13 260-279 82 82 82 151 151 151 F F F 14 280-299 94 94 94 174 174 174 F F F 15 300-319 95 95 95 175 175 175 F F F 16 320-339 92 92 92 170 170 170 F F F 17 340-359 93 93 93 172 172 172 F F F 18 360-379 92 92 92 170 170 170 F F F 19 380-399 90 90 90 166 166 166 F F F 20 400-419 84 84 84 154 154 154 F F F 21 420-439 86 86 86 157 157 157 F F F 22 440-459 94 94 94 174 174 174 F F F 23 460-479 94 94 94 174 174 174 F F F 24 480-499 88 88 88 163 163 163 F F F 25 500-519 92 92 92 170 170 170 F F F 26 520-539 94 94 94 174 174 174 F F F 27 540-559 94 94 94 174 174 174 F F F 28 560-579 95 95 95 175 175 175 F F F 29 580-599 91 91 91 168 168 168 F F F Max gen: 95, Max flush: 175 Begin loop 48 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:55.891722978Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1566,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:14:55.935111638Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1566,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:55.935550274Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1566,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:55.967722167Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1566,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:55.968315454Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1566} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"abd5430f1508af0bc008143ea2566277ef714ec2\",\n git_commit_timestamp: \"2025-01-17T20:55:03.000000000Z\",\n git_branch: \"alan/tokio-to-140\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:55.968348424Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1566} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:55.968361761Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1566} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:55.96837644Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1566} {"msg":"Crucible 87eb98fe-8cac-4d28-9994-e386f725994e has session id: 77e74234-8e41-465e-b805-809cb9945815","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:55.968910292Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1566,"session_id":"77e74234-8e41-465e-b805-809cb9945815"} {"msg":"Upstairs opts: Upstairs UUID: 87eb98fe-8cac-4d28-9994-e386f725994e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:55.968938158Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1566,"session_id":"77e74234-8e41-465e-b805-809cb9945815"} {"msg":"Crucible stats registered with UUID: 87eb98fe-8cac-4d28-9994-e386f725994e","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:55.968952106Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1566,"session_id":"77e74234-8e41-465e-b805-809cb9945815"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:14:55.968964624Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1566,"session_id":"77e74234-8e41-465e-b805-809cb9945815"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:55.969377415Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1566,"task":"crutest"} {"msg":"The guest has requested activation with gen:96","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:55.969401579Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1566} {"msg":"87eb98fe-8cac-4d28-9994-e386f725994e active request set","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:55.969451108Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1566,"session_id":"77e74234-8e41-465e-b805-809cb9945815"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:55.969476853Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1566,"client":"0","":"downstairs","session_id":"77e74234-8e41-465e-b805-809cb9945815"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:55.969502388Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1566,"client":"0","":"downstairs","session_id":"77e74234-8e41-465e-b805-809cb9945815"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:55.969517217Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1566,"client":"1","":"downstairs","session_id":"77e74234-8e41-465e-b805-809cb9945815"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:55.969547184Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1566,"client":"1","":"downstairs","session_id":"77e74234-8e41-465e-b805-809cb9945815"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:55.96957401Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1566,"client":"2","":"downstairs","session_id":"77e74234-8e41-465e-b805-809cb9945815"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:55.969603487Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1566,"client":"2","":"downstairs","session_id":"77e74234-8e41-465e-b805-809cb9945815"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:55.969629742Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1566,"":"io task","client":"2","":"downstairs","session_id":"77e74234-8e41-465e-b805-809cb9945815"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:55.969653607Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1566,"":"io task","client":"0","":"downstairs","session_id":"77e74234-8e41-465e-b805-809cb9945815"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:55.96967747Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1566,"":"io task","client":"1","":"downstairs","session_id":"77e74234-8e41-465e-b805-809cb9945815"} {"msg":"ds_connection connected from Ok(127.0.0.1:38280)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:55.969705237Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1566,"":"io task","client":"2","":"downstairs","session_id":"77e74234-8e41-465e-b805-809cb9945815"} {"msg":"ds_connection connected from Ok(127.0.0.1:38697)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:55.969730421Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1566,"":"io task","client":"1","":"downstairs","session_id":"77e74234-8e41-465e-b805-809cb9945815"} {"msg":"ds_connection connected from Ok(127.0.0.1:33014)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:55.969764381Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1566,"":"io task","client":"0","":"downstairs","session_id":"77e74234-8e41-465e-b805-809cb9945815"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:55.970496989Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1566,"client":"2","":"downstairs","session_id":"77e74234-8e41-465e-b805-809cb9945815"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:55.97053136Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1566,"client":"0","":"downstairs","session_id":"77e74234-8e41-465e-b805-809cb9945815"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:55.970546758Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1566,"client":"1","":"downstairs","session_id":"77e74234-8e41-465e-b805-809cb9945815"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:55.971169342Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1566,"client":"1","":"downstairs","session_id":"77e74234-8e41-465e-b805-809cb9945815"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:55.971301439Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1566,"client":"0","":"downstairs","session_id":"77e74234-8e41-465e-b805-809cb9945815"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:55.971329266Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1566,"client":"2","":"downstairs","session_id":"77e74234-8e41-465e-b805-809cb9945815"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:55.971654746Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1566,"session_id":"77e74234-8e41-465e-b805-809cb9945815"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:55.971925184Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1566,"session_id":"77e74234-8e41-465e-b805-809cb9945815"} {"msg":"[0]R flush_numbers[0..12]: [174, 141, 156, 167, 175, 170, 166, 156, 174, 164, 166, 174]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:55.972006342Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1566,"":"downstairs","":"downstairs","session_id":"77e74234-8e41-465e-b805-809cb9945815"} {"msg":"[0]R generation[0..12]: [94, 78, 86, 90, 95, 92, 90, 86, 94, 89, 90, 94]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:55.972027184Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1566,"":"downstairs","":"downstairs","session_id":"77e74234-8e41-465e-b805-809cb9945815"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:55.972042252Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1566,"":"downstairs","":"downstairs","session_id":"77e74234-8e41-465e-b805-809cb9945815"} {"msg":"[1]R flush_numbers[0..12]: [174, 141, 156, 167, 175, 170, 166, 156, 174, 164, 166, 174]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:55.972056101Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1566,"":"downstairs","":"downstairs","session_id":"77e74234-8e41-465e-b805-809cb9945815"} {"msg":"[1]R generation[0..12]: [94, 78, 86, 90, 95, 92, 90, 86, 94, 89, 90, 94]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:55.972069739Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1566,"":"downstairs","":"downstairs","session_id":"77e74234-8e41-465e-b805-809cb9945815"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:55.972083296Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1566,"":"downstairs","":"downstairs","session_id":"77e74234-8e41-465e-b805-809cb9945815"} {"msg":"[2]R flush_numbers[0..12]: [174, 141, 156, 167, 175, 170, 166, 156, 174, 164, 166, 174]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:55.972096895Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1566,"":"downstairs","":"downstairs","session_id":"77e74234-8e41-465e-b805-809cb9945815"} {"msg":"[2]R generation[0..12]: [94, 78, 86, 90, 95, 92, 90, 86, 94, 89, 90, 94]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:55.972110813Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1566,"":"downstairs","":"downstairs","session_id":"77e74234-8e41-465e-b805-809cb9945815"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:55.972124231Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1566,"":"downstairs","":"downstairs","session_id":"77e74234-8e41-465e-b805-809cb9945815"} {"msg":"Max found gen is 96","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:55.972136768Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1566,"":"downstairs","":"downstairs","session_id":"77e74234-8e41-465e-b805-809cb9945815"} {"msg":"Generation requested: 96 >= found:96","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:55.972149645Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1566,"":"downstairs","":"downstairs","session_id":"77e74234-8e41-465e-b805-809cb9945815"} {"msg":"Next flush: 176","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:55.972165455Z","hostname":"Wait for a query_work_queue command to finish before sending IO w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1566,"":"downstairs","":"downstairs","session_id":"77e74234-8e41-465e-b805-809cb9945815"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:55.972184906Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1566,"":"downstairs","":"downstairs","session_id":"77e74234-8e41-465e-b805-809cb9945815"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:55.972197914Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1566,"session_id":"77e74234-8e41-465e-b805-809cb9945815"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:55.97220964Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1566,"client":"0","":"downstairs","session_id":"77e74234-8e41-465e-b805-809cb9945815"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:55.972222948Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1566,"client":"1","":"downstairs","session_id":"77e74234-8e41-465e-b805-809cb9945815"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:55.972236036Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1566,"client":"2","":"downstairs","session_id":"77e74234-8e41-465e-b805-809cb9945815"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:55.972248753Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1566,"session_id":"77e74234-8e41-465e-b805-809cb9945815"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:55.97226064Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1566,"session_id":"77e74234-8e41-465e-b805-809cb9945815"} {"msg":"87eb98fe-8cac-4d28-9994-e386f725994e is now active with session: 77e74234-8e41-465e-b805-809cb9945815","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:55.972272987Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1566,"session_id":"77e74234-8e41-465e-b805-809cb9945815"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:55.972284534Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1566,"session_id":"77e74234-8e41-465e-b805-809cb9945815"} {"msg":"The guest has finished waiting for activation with:96","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:14:55.972296451Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1566} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 98 len 36864 data: 11 10 11 11 11 10 10 11 8 02/30 Read block 410 len 24576 03/30 Read block 107 len 4096 04/30 Read block 144 len 40960 05/30 Write block 6 len 36864 data: 7 8 7 7 9 11 10 11 10 06/30 Write block 275 len 32768 data: 6 7 6 5 4 6 9 9 07/30 Write block 2 len 40960 data: 6 6 7 8 8 9 8 8 10 12 08/30 Read block 453 len 12288 09/30 Write block 513 len 20480 data: 9 10 13 13 9 10/30 Write block 465 len 40960 data: 11 9 11 10 11 9 8 9 8 7 11/30 Write block 352 len 20480 data: 9 9 10 10 11 12/30 Read block 65 len 40960 13/30 Write block 390 len 12288 data: 7 6 6 14/30 Write block 598 len 4096 data: 4 15/30 Read block 214 len 20480 16/30 Write block 277 len 4096 data: 7 17/30 Read block 122 len 8192 18/30 Read block 274 len 16384 19/30 Write block 426 len 28672 data: 9 10 12 11 10 10 9 20/30 Flush 21/30 Write block 117 len 16384 data: 11 12 10 8 22/30 Flush 23/30 Write block 15 len 36864 data: 8 11 9 9 8 6 7 6 5 24/30 Read block 569 len 16384 25/30 Read block 551 len 36864 26/30 Write block 266 len 20480 data: 8 8 7 8 9 27/30 Write block 12 len 24576 data: 11 12 11 9 12 10 28/30 Write block 200 len 20480 data: 12 10 10 7 6 29/30 Read block 170 len 32768 30/30 Read block 195 len 32768 ---------------------------------------------------------------- Crucible gen:96 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 9 Done Done Done false 1023 Acked Read 4 Done Done Done false 1024 Acked Read 9 Done Done Done false 1025 Acked Write 5 Done Done Done false 1026 Acked Write 6 Done Done Done false 1027 Acked Write 5 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Read 8 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 0 1 Done 7 8 8 23 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1021 1021 1021 Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:14:55.992848403Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1566} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 96 96 96 176 176 176 T T T 1 020-039 78 78 78 141 141 141 T T T 2 040-059 86 86 86 156 156 156 F F F 3 060-079 90 90 90 167 167 167 F F F 4 080-099 96 96 96 176 176 176 F F F 5 100-119 96 96 96 177 177 177 F F F 6 120-139 96 96 96 177 177 177 F F F 7 140-159 86 86 86 156 156 156 F F F 8 160-179 94 94 94 174 174 174 F F F 9 180-199 89 89 89 164 164 164 F F F 10 200-219 90 90 90 166 166 166 T T T 11 220-239 94 94 94 174 174 174 F F F 12 240-259 82 82 82 151 151 151 F F F 13 260-279 96 96 96 176 176 176 T T T 14 280-299 96 96 96 176 176 176 F F F 15 300-319 95 95 95 175 175 175 F F F 16 320-339 92 92 92 170 170 170 F F F 17 340-359 96 96 96 176 176 176 F F F 18 360-379 92 92 92 170 170 170 F F F 19 380-399 96 96 96 176 176 176 F F F 20 400-419 84 84 84 154 154 154 F F F 21 420-439 96 96 96 176 176 176 F F F 22 440-459 94 94 94 174 174 174 F F F 23 460-479 96 96 96 176 176 176 F F F 24 480-499 88 88 88 163 163 163 F F F 25 500-519 96 96 96 176 176 176 F F F 26 520-539 94 94 94 174 174 174 F F F 27 540-559 94 94 94 174 174 174 F F F 28 560-579 95 95 95 175 175 175 F F F 29 580-599 96 96 96 176 176 176 F F F Max gen: 96, Max flush: 177 A Difference in extent metadata is expected here On loop 48 Verifying data now No change after verify Loop: 48 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 97 97 97 178 178 178 F F F 1 020-039 97 97 97 178 178 178 F F F 2 040-059 86 86 86 156 156 156 F F F 3 060-079 90 90 90 167 167 167 F F F 4 080-099 96 96 96 176 176 176 F F F 5 100-119 96 96 96 177 177 177 F F F 6 120-139 96 96 96 177 177 177 F F F 7 140-159 86 86 86 156 156 156 F F F 8 160-179 94 94 94 174 174 174 F F F 9 180-199 89 89 89 164 164 164 F F F 10 200-219 97 97 97 178 178 178 F F F 11 220-239 94 94 94 174 174 174 F F F 12 240-259 82 82 82 151 151 151 F F F 13 260-279 97 97 97 178 178 178 F F F 14 280-299 96 96 96 176 176 176 F F F 15 300-319 95 95 95 175 175 175 F F F 16 320-339 92 92 92 170 170 170 F F F 17 340-359 96 96 96 176 176 176 F F F 18 360-379 92 92 92 170 170 170 F F F 19 380-399 96 96 96 176 176 176 F F F 20 400-419 84 84 84 154 154 154 F F F 21 420-439 96 96 96 176 176 176 F F F 22 440-459 94 94 94 174 174 174 F F F 23 460-479 96 96 96 176 176 176 F F F 24 480-499 88 88 88 163 163 163 F F F 25 500-519 96 96 96 176 176 176 F F F 26 520-539 94 94 94 174 174 174 F F F 27 540-559 94 94 94 174 174 174 F F F 28 560-579 95 95 95 175 175 175 F F F 29 580-599 96 96 96 176 176 176 F F F Max gen: 97, Max flush: 178 Begin loop 49 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:00.259388733Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1574,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:15:00.304512146Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1574,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:00.304939156Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1574,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:00.337441773Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1574,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:00.338099857Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1574} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"abd5430f1508af0bc008143ea2566277ef714ec2\",\n git_commit_timestamp: \"2025-01-17T20:55:03.000000000Z\",\n git_branch: \"alan/tokio-to-140\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:00.33813983Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1574} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:00.33815611Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1574} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:00.3381736Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1574} {"msg":"Crucible cd2f1c1f-db4f-46f7-8b28-1a7b4193920a has session id: 289e245e-02a6-4d50-a60e-c99b055e5e14","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:00.338769458Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1574,"session_id":"289e245e-02a6-4d50-a60e-c99b055e5e14"} {"msg":"Upstairs opts: Upstairs UUID: cd2f1c1f-db4f-46f7-8b28-1a7b4193920a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:00.338800507Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1574,"session_id":"289e245e-02a6-4d50-a60e-c99b055e5e14"} {"msg":"Crucible stats registered with UUID: cd2f1c1f-db4f-46f7-8b28-1a7b4193920a","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:00.338819078Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1574,"session_id":"289e245e-02a6-4d50-a60e-c99b055e5e14"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:15:00.33884048Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1574,"session_id":"289e245e-02a6-4d50-a60e-c99b055e5e14"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:00.339301399Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1574,"task":"crutest"} {"msg":"The guest has requested activation with gen:98","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:00.339333358Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1574} {"msg":"cd2f1c1f-db4f-46f7-8b28-1a7b4193920a active request set","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:00.339365507Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1574,"session_id":"289e245e-02a6-4d50-a60e-c99b055e5e14"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:00.339381396Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1574,"client":"0","":"downstairs","session_id":"289e245e-02a6-4d50-a60e-c99b055e5e14"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:00.339409612Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1574,"client":"0","":"downstairs","session_id":"289e245e-02a6-4d50-a60e-c99b055e5e14"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:00.339426432Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1574,"client":"1","":"downstairs","session_id":"289e245e-02a6-4d50-a60e-c99b055e5e14"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:00.339443723Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1574,"client":"1","":"downstairs","session_id":"289e245e-02a6-4d50-a60e-c99b055e5e14"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:00.339459952Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1574,"client":"2","":"downstairs","session_id":"289e245e-02a6-4d50-a60e-c99b055e5e14"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:00.339477002Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1574,"client":"2","":"downstairs","session_id":"289e245e-02a6-4d50-a60e-c99b055e5e14"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:00.339497854Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1574,"":"io task","client":"2","":"downstairs","session_id":"289e245e-02a6-4d50-a60e-c99b055e5e14"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:00.339530713Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1574,"":"io task","client":"0","":"downstairs","session_id":"289e245e-02a6-4d50-a60e-c99b055e5e14"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:00.33955911Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1574,"":"io task","client":"1","":"downstairs","session_id":"289e245e-02a6-4d50-a60e-c99b055e5e14"} {"msg":"ds_connection connected from Ok(127.0.0.1:38291)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:00.339577501Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1574,"":"io task","client":"2","":"downstairs","session_id":"289e245e-02a6-4d50-a60e-c99b055e5e14"} {"msg":"ds_connection connected from Ok(127.0.0.1:44552)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:00.339594411Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1574,"":"io task","client":"0","":"downstairs","session_id":"289e245e-02a6-4d50-a60e-c99b055e5e14"} {"msg":"ds_connection connected from Ok(127.0.0.1:53842)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:00.339745309Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1574,"":"io task","client":"1","":"downstairs","session_id":"289e245e-02a6-4d50-a60e-c99b055e5e14"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:00.340318814Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1574,"client":"1","":"downstairs","session_id":"289e245e-02a6-4d50-a60e-c99b055e5e14"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:00.340358187Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1574,"client":"2","":"downstairs","session_id":"289e245e-02a6-4d50-a60e-c99b055e5e14"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:00.340440875Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1574,"client":"0","":"downstairs","session_id":"289e245e-02a6-4d50-a60e-c99b055e5e14"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:00.340981501Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1574,"client":"1","":"downstairs","session_id":"289e245e-02a6-4d50-a60e-c99b055e5e14"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:00.341023766Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1574,"client":"2","":"downstairs","session_id":"289e245e-02a6-4d50-a60e-c99b055e5e14"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:00.341057816Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1574,"client":"0","":"downstairs","session_id":"289e245e-02a6-4d50-a60e-c99b055e5e14"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:00.341551655Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1574,"session_id":"289e245e-02a6-4d50-a60e-c99b055e5e14"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:00.341712699Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1574,"session_id":"289e245e-02a6-4d50-a60e-c99b055e5e14"} {"msg":"[0]R flush_numbers[0..12]: [178, 178, 156, 167, 176, 177, 177, 156, 174, 164, 178, 174]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:00.341808125Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1574,"":"downstairs","":"downstairs","session_id":"289e245e-02a6-4d50-a60e-c99b055e5e14"} {"msg":"[0]R generation[0..12]: [97, 97, 86, 90, 96, 96, 96, 86, 94, 89, 97, 94]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:00.341827486Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1574,"":"downstairs","":"downstairs","session_id":"289e245e-02a6-4d50-a60e-c99b055e5e14"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:00.341841774Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1574,"":"downstairs","":"downstairs","session_id":"289e245e-02a6-4d50-a60e-c99b055e5e14"} {"msg":"[1]R flush_numbers[0..12]: [178, 178, 156, 167, 176, 177, 177, 156, 174, 164, 178, 174]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:00.341855462Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1574,"":"downstairs","":"downstairs","session_id":"289e245e-02a6-4d50-a60e-c99b055e5e14"} {"msg":"[1]R generation[0..12]: [97, 97, 86, 90, 96, 96, 96, 86, 94, 89, 97, 94]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:00.34186903Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1574,"":"downstairs","":"downstairs","session_id":"289e245e-02a6-4d50-a60e-c99b055e5e14"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:00.341886Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1574,"":"downstairs","":"downstairs","session_id":"289e245e-02a6-4d50-a60e-c99b055e5e14"} {"msg":"[2]R flush_numbers[0..12]: [178, 178, 156, 167, 176, 177, 177, 156, 174, 164, 178, 174]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:00.341899748Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1574,"":"downstairs","":"downstairs","session_id":"289e245e-02a6-4d50-a60e-c99b055e5e14"} {"msg":"[2]R generation[0..12]: [97, 97, 86, 90, 96, 96, 96, 86, 94, 89, 97, 94]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:00.341912956Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1574,"":"downstairs","":"downstairs","session_id":"289e245e-02a6-4d50-a60e-c99b055e5e14"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:00.341926383Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1574,"":"downstairs","":"downstairs","session_id":"289e245e-02a6-4d50-a60e-c99b055e5e14"} {"msg":"Max found gen is 98","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:00.341938911Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1574,"":"downstairs","":"downstairs","session_id":"289e245e-02a6-4d50-a60e-c99b055e5e14"} {"msg":"Generation requested: 98 >= found:98","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:00.341951898Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1574,"":"downstairs","":"downstairs","session_id":"289e245e-02a6-4d50-a60e-c99b055e5e14"} {"msg":"Next flush: 179","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:00.341964286Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1574,"":"downstairs","":"downstairs","session_id":"289e245e-02a6-4d50-a60e-c99b055e5e14"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:00.341976783Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1574,"":"downstairs","":"downstairs","session_id":"289e245e-02a6-4d50-a60e-c99b055e5e14"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-17T21:15:00.341989591Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1574,"session_id":"289e245e-02a6-4d50-a60e-c99b055e5e14"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:00.34200617Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1574,"client":"0","":"downstairs","session_id":"289e245e-02a6-4d50-a60e-c99b055e5e14"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:00.342019018Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1574,"client":"1","":"downstairs","session_id":"289e245e-02a6-4d50-a60e-c99b055e5e14"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:00.342031745Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1574,"client":"2","":"downstairs","session_id":"289e245e-02a6-4d50-a60e-c99b055e5e14"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:00.342045703Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1574,"session_id":"289e245e-02a6-4d50-a60e-c99b055e5e14"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:00.34205785Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1574,"session_id":"289e245e-02a6-4d50-a60e-c99b055e5e14"} {"msg":"cd2f1c1f-db4f-46f7-8b28-1a7b4193920a is now active with session: 289e245e-02a6-4d50-a60e-c99b055e5e14","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:00.342070218Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1574,"session_id":"289e245e-02a6-4d50-a60e-c99b055e5e14"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:00.342083586Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1574,"session_id":"289e245e-02a6-4d50-a60e-c99b055e5e14"} {"msg":"The guest has finished waiting for activation with:98","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:00.342095502Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1574} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 252 len 16384 data: 6 5 5 7 02/30 Write block 422 len 12288 data: 7 7 7 03/30 Write block 271 len 32768 data: 8 6 6 6 7 8 8 6 04/30 Write block 327 len 36864 data: 7 6 6 7 7 6 5 10 10 05/30 Read block 185 len 24576 06/30 Flush 07/30 Write block 357 len 20480 data: 12 10 11 8 7 08/30 Write block 505 len 8192 data: 12 10 09/30 Read block 120 len 16384 10/30 Write block 52 len 16384 data: 12 12 12 11 11/30 Read block 141 len 4096 12/30 Write block 136 len 24576 data: 7 7 7 8 7 6 13/30 Read block 125 len 32768 14/30 Flush 15/30 Write block 0 len 24576 data: 4 6 7 7 8 9 16/30 Write block 441 len 16384 data: 5 4 6 7 17/30 Read block 44 len 16384 18/30 Read block 227 len 28672 19/30 Flush 20/30 Write block 142 len 36864 data: 7 7 6 6 6 7 6 6 7 21/30 Read block 61 len 4096 22/30 Read block 404 len 16384 23/30 Read block 347 len 36864 24/30 Read block 507 len 36864 25/30 Read block 471 len 12288 26/30 Read block 218 len 40960 27/30 Read block 439 len 32768 28/30 Read block 546 len 8192 29/30 Read block 548 len 36864 30/30 Write block 323 len 16384 data: 8 7 8 9 ---------------------------------------------------------------- Crucible gen:98 GIO:true work queues: Upstairs:0 downstairs:11 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1019 Acked Write 9 Done Done Done false 1020 Acked Read 1 Done Done Done false 1021 Acked Read 4 Done Done Done false 1022 Acked Read 9 Done Done Done false 1023 Acked Read 9 Done Done Done false 1024 Acked Read 3 Done Done Done false 1025 Acked Read 10 Done Done Done false 1026 Acked Read 8 Done Done Done false 1027 Acked Read 2 Done Done Done false 1028 Acked Read 9 Done Done Done false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 10 10 10 30 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1018 1018 1018 Downstairs last five completed: 1018 1017 1016 1015 1014 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:15:00.36174979Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1574} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 98 98 98 181 181 181 F F F 1 020-039 97 97 97 178 178 178 F F F 2 040-059 98 98 98 180 180 180 F F F 3 060-079 90 90 90 167 167 167 F F F 4 080-099 96 96 96 176 176 176 F F F 5 100-119 96 96 96 177 177 177 F F F 6 120-139 98 98 98 180 180 180 F F F 7 140-159 98 98 98 180 180 180 T T T 8 160-179 94 94 94 174 174 174 F F F 9 180-199 89 89 89 164 164 164 F F F 10 200-219 97 97 97 178 178 178 F F F 11 220-239 94 94 94 174 174 174 F F F 12 240-259 98 98 98 179 179 179 F F F 13 260-279 98 98 98 179 179 179 F F F 14 280-299 96 96 96 176 176 176 F F F 15 300-319 95 95 95 175 175 175 F F F 16 320-339 98 98 98 179 179 179 T T T 17 340-359 98 98 98 180 180 180 F F F 18 360-379 98 98 98 180 180 180 F F F 19 380-399 96 96 96 176 176 176 F F F 20 400-419 84 84 84 154 154 154 F F F 21 420-439 98 98 98 179 179 179 F F F 22 440-459 98 98 98 181 181 181 F F F 23 460-479 96 96 96 176 176 176 F F F 24 480-499 88 88 88 163 163 163 F F F 25 500-519 98 98 98 180 180 180 F F F 26 520-539 94 94 94 174 174 174 F F F 27 540-559 94 94 94 174 174 174 F F F 28 560-579 95 95 95 175 175 175 F F F 29 580-599 96 96 96 176 176 176 F F F Max gen: 98, Max flush: 181 A Difference in extent metadata is expected here On loop 49 Verifying data now No change after verify Loop: 49 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 98 98 98 181 181 181 F F F 1 020-039 97 97 97 178 178 178 F F F 2 040-059 98 98 98 180 180 180 F F F 3 060-079 90 90 90 167 167 167 F F F 4 080-099 96 96 96 176 176 176 F F F 5 100-119 96 96 96 177 177 177 F F F 6 120-139 98 98 98 180 180 180 F F F 7 140-159 99 99 99 182 182 182 F F F 8 160-179 94 94 94 174 174 174 F F F 9 180-199 89 89 89 164 164 164 F F F 10 200-219 97 97 97 178 178 178 F F F 11 220-239 94 94 94 174 174 174 F F F 12 240-259 98 98 98 179 179 179 F F F 13 260-279 98 98 98 179 179 179 F F F 14 280-299 96 96 96 176 176 176 F F F 15 300-319 95 95 95 175 175 175 F F F 16 320-339 99 99 99 182 182 182 F F F 17 340-359 98 98 98 180 180 180 F F F 18 360-379 98 98 98 180 180 180 F F F 19 380-399 96 96 96 176 176 176 F F F 20 400-419 84 84 84 154 154 154 F F F 21 420-439 98 98 98 179 179 179 F F F 22 440-459 98 98 98 181 181 181 F F F 23 460-479 96 96 96 176 176 176 F F F 24 480-499 88 88 88 163 163 163 F F F 25 500-519 98 98 98 180 180 180 F F F 26 520-539 94 94 94 174 174 174 F F F 27 540-559 94 94 94 174 174 174 F F F 28 560-579 95 95 95 175 175 175 F F F 29 580-599 96 96 96 176 176 176 F F F Max gen: 99, Max flush: 182 Begin loop 50 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:04.612460778Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1582,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:04.65633225Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1582,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:04.656974435Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1582} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"abd5430f1508af0bc008143ea2566277ef714ec2\",\n git_commit_timestamp: \"2025-01-17T20:55:03.000000000Z\",\n git_branch: \"alan/tokio-to-140\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:04.657018571Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1582} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:04.657040114Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1582} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:04.657059435Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1582} {"msg":"Crucible e8eaa9eb-7b39-483c-8c2f-8a87c32b9064 has session id: 4e67104b-e428-400d-b32d-ff60fc5f3ea0","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:04.657540896Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1582,"session_id":"4e67104b-e428-400d-b32d-ff60fc5f3ea0"} {"msg":"Upstairs opts: Upstairs UUID: e8eaa9eb-7b39-483c-8c2f-8a87c32b9064, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:04.657584602Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1582,"session_id":"4e67104b-e428-400d-b32d-ff60fc5f3ea0"} {"msg":"Crucible stats registered with UUID: e8eaa9eb-7b39-483c-8c2f-8a87c32b9064","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:04.657602012Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1582,"session_id":"4e67104b-e428-400d-b32d-ff60fc5f3ea0"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:15:04.65761528Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1582,"session_id":"4e67104b-e428-400d-b32d-ff60fc5f3ea0"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:04.658072907Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1582,"task":"crutest"} {"msg":"The guest has requested activation with gen:100","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:04.658097392Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1582} {"msg":"e8eaa9eb-7b39-483c-8c2f-8a87c32b9064 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:04.658167943Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1582,"session_id":"4e67104b-e428-400d-b32d-ff60fc5f3ea0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:04.658185613Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1582,"client":"0","":"downstairs","session_id":"4e67104b-e428-400d-b32d-ff60fc5f3ea0"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:04.658209477Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1582,"client":"0","":"downstairs","session_id":"4e67104b-e428-400d-b32d-ff60fc5f3ea0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:04.658224246Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1582,"client":"1","":"downstairs","session_id":"4e67104b-e428-400d-b32d-ff60fc5f3ea0"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:04.658238254Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1582,"client":"1","":"downstairs","session_id":"4e67104b-e428-400d-b32d-ff60fc5f3ea0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:04.658258896Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1582,"client":"2","":"downstairs","session_id":"4e67104b-e428-400d-b32d-ff60fc5f3ea0"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:04.658292696Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1582,"client":"2","":"downstairs","session_id":"4e67104b-e428-400d-b32d-ff60fc5f3ea0"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:04.658319191Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1582,"":"io task","client":"2","":"downstairs","session_id":"4e67104b-e428-400d-b32d-ff60fc5f3ea0"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:04.658344806Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1582,"":"io task","client":"0","":"downstairs","session_id":"4e67104b-e428-400d-b32d-ff60fc5f3ea0"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:04.658370181Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1582,"":"io task","client":"1","":"downstairs","session_id":"4e67104b-e428-400d-b32d-ff60fc5f3ea0"} {"msg":"ds_connection connected from Ok(127.0.0.1:44229)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:04.658395255Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1582,"":"io task","client":"2","":"downstairs","session_id":"4e67104b-e428-400d-b32d-ff60fc5f3ea0"} {"msg":"ds_connection connected from Ok(127.0.0.1:44302)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:04.65841984Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1582,"":"io task","client":"1","":"downstairs","session_id":"4e67104b-e428-400d-b32d-ff60fc5f3ea0"} {"msg":"ds_connection connected from Ok(127.0.0.1:33975)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:04.658442913Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1582,"":"io task","client":"0","":"downstairs","session_id":"4e67104b-e428-400d-b32d-ff60fc5f3ea0"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:04.659144734Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1582,"client":"1","":"downstairs","session_id":"4e67104b-e428-400d-b32d-ff60fc5f3ea0"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:04.659270007Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1582,"client":"0","":"downstairs","session_id":"4e67104b-e428-400d-b32d-ff60fc5f3ea0"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:04.659663417Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1582,"client":"1","":"downstairs","session_id":"4e67104b-e428-400d-b32d-ff60fc5f3ea0"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:04.659796925Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1582,"client":"0","":"downstairs","session_id":"4e67104b-e428-400d-b32d-ff60fc5f3ea0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:04.660264828Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1582,"session_id":"4e67104b-e428-400d-b32d-ff60fc5f3ea0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:04.660372962Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1582,"session_id":"4e67104b-e428-400d-b32d-ff60fc5f3ea0"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:04.660457851Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1582,"client":"2","":"downstairs","session_id":"4e67104b-e428-400d-b32d-ff60fc5f3ea0"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:04.660950909Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1582,"client":"2","":"downstairs","session_id":"4e67104b-e428-400d-b32d-ff60fc5f3ea0"} {"msg":"[0]R flush_numbers[0..12]: [181, 178, 180, 167, 176, 177, 180, 182, 174, 164, 178, 174]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:04.661435483Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1582,"":"downstairs","":"downstairs","session_id":"4e67104b-e428-400d-b32d-ff60fc5f3ea0"} {"msg":"[0]R generation[0..12]: [98, 97, 98, 90, 96, 96, 98, 99, 94, 89, 97, 94]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:04.661458026Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1582,"":"downstairs","":"downstairs","session_id":"4e67104b-e428-400d-b32d-ff60fc5f3ea0"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:04.661472684Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1582,"":"downstairs","":"downstairs","session_id":"4e67104b-e428-400d-b32d-ff60fc5f3ea0"} {"msg":"[1]R flush_numbers[0..12]: [181, 178, 180, 167, 176, 177, 180, 182, 174, 164, 178, 174]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:04.661486612Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1582,"":"downstairs","":"downstairs","session_id":"4e67104b-e428-400d-b32d-ff60fc5f3ea0"} {"msg":"[1]R generation[0..12]: [98, 97, 98, 90, 96, 96, 98, 99, 94, 89, 97, 94]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:04.66149992Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1582,"":"downstairs","":"downstairs","session_id":"4e67104b-e428-400d-b32d-ff60fc5f3ea0"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:04.661513688Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1582,"":"downstairs","":"downstairs","session_id":"4e67104b-e428-400d-b32d-ff60fc5f3ea0"} {"msg":"[2]R flush_numbers[0..12]: [181, 178, 180, 167, 176, 177, 180, 182, 174, 164, 178, 174]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:04.661527186Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1582,"":"downstairs","":"downstairs","session_id":"4e67104b-e428-400d-b32d-ff60fc5f3ea0"} {"msg":"[2]R generation[0..12]: [98, 97, 98, 90, 96, 96, 98, 99, 94, 89, 97, 94]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:04.661540724Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1582,"":"downstairs","":"downstairs","session_id":"4e67104b-e428-400d-b32d-ff60fc5f3ea0"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:04.661554422Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1582,"":"downstairs","":"downstairs","session_id":"4e67104b-e428-400d-b32d-ff60fc5f3ea0"} {"msg":"Max found gen is 100","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:04.661566979Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1582,"":"downstairs","":"downstairs","session_id":"4e67104b-e428-400d-b32d-ff60fc5f3ea0"} {"msg":"Generation requested: 100 >= found:100","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:04.661579937Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1582,"":"downstairs","":"downstairs","session_id":"4e67104b-e428-400d-b32d-ff60fc5f3ea0"} Wait for a query_work_queue command to finish before sending IO {"msg":"Next flush: 183","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:04.661598598Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1582,"":"downstairs","":"downstairs","session_id":"4e67104b-e428-400d-b32d-ff60fc5f3ea0"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:04.661611635Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1582,"":"downstairs","":"downstairs","session_id":"4e67104b-e428-400d-b32d-ff60fc5f3ea0"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:04.661624383Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1582,"session_id":"4e67104b-e428-400d-b32d-ff60fc5f3ea0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:04.661638581Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1582,"client":"0","":"downstairs","session_id":"4e67104b-e428-400d-b32d-ff60fc5f3ea0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:04.661651669Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1582,"client":"1","":"downstairs","session_id":"4e67104b-e428-400d-b32d-ff60fc5f3ea0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:04.661664406Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1582,"client":"2","":"downstairs","session_id":"4e67104b-e428-400d-b32d-ff60fc5f3ea0"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:04.661676994Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1582,"session_id":"4e67104b-e428-400d-b32d-ff60fc5f3ea0"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:04.661688931Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1582,"session_id":"4e67104b-e428-400d-b32d-ff60fc5f3ea0"} {"msg":"e8eaa9eb-7b39-483c-8c2f-8a87c32b9064 is now active with session: 4e67104b-e428-400d-b32d-ff60fc5f3ea0","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:04.661701348Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1582,"session_id":"4e67104b-e428-400d-b32d-ff60fc5f3ea0"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:04.661712774Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1582,"session_id":"4e67104b-e428-400d-b32d-ff60fc5f3ea0"} {"msg":"The guest has finished waiting for activation with:100","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:04.661724641Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1582} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 562 len 8192 data: 11 10 02/30 Write block 581 len 28672 data: 8 7 8 9 8 8 8 03/30 Read block 161 len 32768 04/30 Write block 560 len 20480 data: 9 10 12 11 8 05/30 Read block 561 len 24576 06/30 Read block 288 len 8192 07/30 Write block 237 len 4096 data: 8 08/30 Read block 356 len 4096 09/30 Flush 10/30 Flush 11/30 Write block 241 len 36864 data: 8 8 8 8 7 7 8 9 7 12/30 Read block 91 len 28672 13/30 Read block 202 len 16384 14/30 Flush 15/30 Write block 472 len 36864 data: 10 9 8 9 8 6 5 5 6 16/30 Read block 257 len 4096 17/30 Write block 229 len 36864 data: 8 9 7 7 8 9 10 9 9 18/30 Read block 184 len 12288 19/30 Flush 20/30 Write block 180 len 16384 data: 7 7 6 5 21/30 Flush 22/30 Write block 24 len 28672 data: 5 6 8 8 8 6 9 23/30 Read block 102 len 32768 24/30 Write block 165 len 28672 data: 9 9 8 9 8 9 9 25/30 Write block 89 len 16384 data: 8 11 10 12 26/30 Read block 268 len 16384 27/30 Read block 556 len 4096 28/30 Read block 574 len 24576 29/30 Read block 219 len 28672 30/30 Write block 422 len 40960 data: 8 8 8 7 10 11 13 12 11 11 ---------------------------------------------------------------- Crucible gen:100 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 7 Done Done Done false 1022 Acked Read 8 Done Done Done false 1023 Acked Write 7 Done Done Done false 1024 Acked Write 4 Done Done Done false 1025 Acked Read 4 Done Done Done false 1026 Acked Read 1 Done Done Done false 1027 Acked Read 6 Done Done Done false 1028 Acked Read 7 Done Done Done false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 8 8 8 24 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1020 1020 1020 Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:15:04.681491735Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1582} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 98 98 98 181 181 181 F F F 1 020-039 97 97 97 178 178 178 T T T 2 040-059 98 98 98 180 180 180 F F F 3 060-079 90 90 90 167 167 167 F F F 4 080-099 96 96 96 176 176 176 T T T 5 100-119 96 96 96 177 177 177 F F F 6 120-139 98 98 98 180 180 180 F F F 7 140-159 99 99 99 182 182 182 F F F 8 160-179 94 94 94 174 174 174 T T T 9 180-199 100 100 100 187 187 187 F F F 10 200-219 97 97 97 178 178 178 F F F 11 220-239 100 100 100 186 186 186 F F F 12 240-259 100 100 100 185 185 185 F F F 13 260-279 98 98 98 179 179 179 F F F 14 280-299 96 96 96 176 176 176 F F F 15 300-319 95 95 95 175 175 175 F F F 16 320-339 99 99 99 182 182 182 F F F 17 340-359 98 98 98 180 180 180 F F F 18 360-379 98 98 98 180 180 180 F F F 19 380-399 96 96 96 176 176 176 F F F 20 400-419 84 84 84 154 154 154 F F F 21 420-439 98 98 98 179 179 179 T T T 22 440-459 98 98 98 181 181 181 F F F 23 460-479 100 100 100 186 186 186 F F F 24 480-499 100 100 100 186 186 186 F F F 25 500-519 98 98 98 180 180 180 F F F 26 520-539 94 94 94 174 174 174 F F F 27 540-559 94 94 94 174 174 174 F F F 28 560-579 100 100 100 183 183 183 F F F 29 580-599 100 100 100 183 183 183 F F F Max gen: 100, Max flush: 187 A Difference in extent metadata is expected here On loop 50 Verifying data now No change after verify Loop: 50 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 98 98 98 181 181 181 F F F 1 020-039 101 101 101 188 188 188 F F F 2 040-059 98 98 98 180 180 180 F F F 3 060-079 90 90 90 167 167 167 F F F 4 080-099 101 101 101 188 188 188 F F F 5 100-119 96 96 96 177 177 177 F F F 6 120-139 98 98 98 180 180 180 F F F 7 140-159 99 99 99 182 182 182 F F F 8 160-179 101 101 101 188 188 188 F F F 9 180-199 100 100 100 187 187 187 F F F 10 200-219 97 97 97 178 178 178 F F F 11 220-239 100 100 100 186 186 186 F F F 12 240-259 100 100 100 185 185 185 F F F 13 260-279 98 98 98 179 179 179 F F F 14 280-299 96 96 96 176 176 176 F F F 15 300-319 95 95 95 175 175 175 F F F 16 320-339 99 99 99 182 182 182 F F F 17 340-359 98 98 98 180 180 180 F F F 18 360-379 98 98 98 180 180 180 F F F 19 380-399 96 96 96 176 176 176 F F F 20 400-419 84 84 84 154 154 154 F F F 21 420-439 101 101 101 188 188 188 F F F 22 440-459 98 98 98 181 181 181 F F F 23 460-479 100 100 100 186 186 186 F F F 24 480-499 100 100 100 186 186 186 F F F 25 500-519 98 98 98 180 180 180 F F F 26 520-539 94 94 94 174 174 174 F F F 27 540-559 94 94 94 174 174 174 F F F 28 560-579 100 100 100 183 183 183 F F F 29 580-599 100 100 100 183 183 183 F F F Max gen: 101, Max flush: 188 Begin loop 51 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:08.912684295Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1590,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:08.957302533Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1590,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:08.957927228Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1590} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"abd5430f1508af0bc008143ea2566277ef714ec2\",\n git_commit_timestamp: \"2025-01-17T20:55:03.000000000Z\",\n git_branch: \"alan/tokio-to-140\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:08.957968342Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1590} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:08.957982791Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1590} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:08.958000371Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1590} {"msg":"Crucible ad0ad434-7096-431a-adce-f2ca9f2e0589 has session id: 660ce876-457b-4105-b27c-9c2132e8bbfa","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:08.958553965Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1590,"session_id":"660ce876-457b-4105-b27c-9c2132e8bbfa"} {"msg":"Upstairs opts: Upstairs UUID: ad0ad434-7096-431a-adce-f2ca9f2e0589, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:08.958607936Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1590,"session_id":"660ce876-457b-4105-b27c-9c2132e8bbfa"} {"msg":"Crucible stats registered with UUID: ad0ad434-7096-431a-adce-f2ca9f2e0589","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:08.95863205Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1590,"session_id":"660ce876-457b-4105-b27c-9c2132e8bbfa"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:15:08.958650521Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1590,"session_id":"660ce876-457b-4105-b27c-9c2132e8bbfa"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:08.959072548Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1590,"task":"crutest"} {"msg":"The guest has requested activation with gen:102","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:08.959096531Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1590} {"msg":"ad0ad434-7096-431a-adce-f2ca9f2e0589 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:08.959182282Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1590,"session_id":"660ce876-457b-4105-b27c-9c2132e8bbfa"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:08.959203674Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1590,"client":"0","":"downstairs","session_id":"660ce876-457b-4105-b27c-9c2132e8bbfa"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:08.959226978Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1590,"client":"0","":"downstairs","session_id":"660ce876-457b-4105-b27c-9c2132e8bbfa"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:08.959243247Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1590,"client":"1","":"downstairs","session_id":"660ce876-457b-4105-b27c-9c2132e8bbfa"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:08.959268062Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1590,"client":"1","":"downstairs","session_id":"660ce876-457b-4105-b27c-9c2132e8bbfa"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:08.959293306Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1590,"client":"2","":"downstairs","session_id":"660ce876-457b-4105-b27c-9c2132e8bbfa"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:08.959319242Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1590,"client":"2","":"downstairs","session_id":"660ce876-457b-4105-b27c-9c2132e8bbfa"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:08.959344917Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1590,"":"io task","client":"2","":"downstairs","session_id":"660ce876-457b-4105-b27c-9c2132e8bbfa"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:08.959370392Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1590,"":"io task","client":"0","":"downstairs","session_id":"660ce876-457b-4105-b27c-9c2132e8bbfa"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:08.959393605Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1590,"":"io task","client":"1","":"downstairs","session_id":"660ce876-457b-4105-b27c-9c2132e8bbfa"} {"msg":"ds_connection connected from Ok(127.0.0.1:41042)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:08.959417209Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1590,"":"io task","client":"2","":"downstairs","session_id":"660ce876-457b-4105-b27c-9c2132e8bbfa"} {"msg":"ds_connection connected from Ok(127.0.0.1:47257)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:08.959443605Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1590,"":"io task","client":"1","":"downstairs","session_id":"660ce876-457b-4105-b27c-9c2132e8bbfa"} {"msg":"ds_connection connected from Ok(127.0.0.1:52434)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:08.959469199Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1590,"":"io task","client":"0","":"downstairs","session_id":"660ce876-457b-4105-b27c-9c2132e8bbfa"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:08.96013754Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1590,"client":"2","":"downstairs","session_id":"660ce876-457b-4105-b27c-9c2132e8bbfa"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:08.960166827Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1590,"client":"0","":"downstairs","session_id":"660ce876-457b-4105-b27c-9c2132e8bbfa"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:08.960305838Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1590,"client":"1","":"downstairs","session_id":"660ce876-457b-4105-b27c-9c2132e8bbfa"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:08.960807531Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1590,"client":"0","":"downstairs","session_id":"660ce876-457b-4105-b27c-9c2132e8bbfa"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:08.960835878Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1590,"client":"2","":"downstairs","session_id":"660ce876-457b-4105-b27c-9c2132e8bbfa"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:08.960850847Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1590,"client":"1","":"downstairs","session_id":"660ce876-457b-4105-b27c-9c2132e8bbfa"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:08.961330307Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1590,"session_id":"660ce876-457b-4105-b27c-9c2132e8bbfa"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:08.961429805Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1590,"session_id":"660ce876-457b-4105-b27c-9c2132e8bbfa"} {"msg":"[0]R flush_numbers[0..12]: [181, 188, 180, 167, 188, 177, 180, 182, 188, 187, 178, 186]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:08.961462674Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1590,"":"downstairs","":"downstairs","session_id":"660ce876-457b-4105-b27c-9c2132e8bbfa"} {"msg":"[0]R generation[0..12]: [98, 101, 98, 90, 101, 96, 98, 99, 101, 100, 97, 100]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:08.961478554Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1590,"":"downstairs","":"downstairs","session_id":"660ce876-457b-4105-b27c-9c2132e8bbfa"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:08.961492602Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1590,"":"downstairs","":"downstairs","session_id":"660ce876-457b-4105-b27c-9c2132e8bbfa"} {"msg":"[1]R flush_numbers[0..12]: [181, 188, 180, 167, 188, 177, 180, 182, 188, 187, 178, 186]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:08.96150613Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1590,"":"downstairs","":"downstairs","session_id":"660ce876-457b-4105-b27c-9c2132e8bbfa"} {"msg":"[1]R generation[0..12]: [98, 101, 98, 90, 101, 96, 98, 99, 101, 100, 97, 100]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:08.961519758Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1590,"":"downstairs","":"downstairs","session_id":"660ce876-457b-4105-b27c-9c2132e8bbfa"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:08.961533316Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1590,"":"downstairs","":"downstairs","session_id":"660ce876-457b-4105-b27c-9c2132e8bbfa"} {"msg":"[2]R flush_numbers[0..12]: [181, 188, 180, 167, 188, 177, 180, 182, 188, 187, 178, 186]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:08.961546543Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1590,"":"downstairs","":"downstairs","session_id":"660ce876-457b-4105-b27c-9c2132e8bbfa"} {"msg":"[2]R generation[0..12]: [98, 101, 98, 90, 101, 96, 98, 99, 101, 100, 97, 100]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:08.961559611Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1590,"":"downstairs","":"downstairs","session_id":"660ce876-457b-4105-b27c-9c2132e8bbfa"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:08.961573469Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1590,"":"downstairs","":"downstairs","session_id":"660ce876-457b-4105-b27c-9c2132e8bbfa"} {"msg":"Max found gen is 102","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:08.961589348Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1590,"":"downstairs","":"downstairs","session_id":"660ce876-457b-4105-b27c-9c2132e8bbfa"} {"msg":"Generation requested: 102 >= found:102","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:08.961602616Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1590,"":"downstairs","":"downstairs","session_id":"660ce876-457b-4105-b27c-9c2132e8bbfa"} {"msg":"Next flush: 189","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:08.961615214Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1590,"":"downstairs","":"downstairs","session_id":"660ce876-457b-4105-b27c-9c2132e8bbfa"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-17T21:15:08.961627681Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1590,"":"downstairs","":"downstairs","session_id":"660ce876-457b-4105-b27c-9c2132e8bbfa"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:08.96164377Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1590,"session_id":"660ce876-457b-4105-b27c-9c2132e8bbfa"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:08.961655447Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1590,"client":"0","":"downstairs","session_id":"660ce876-457b-4105-b27c-9c2132e8bbfa"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:08.961668585Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1590,"client":"1","":"downstairs","session_id":"660ce876-457b-4105-b27c-9c2132e8bbfa"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:08.961681513Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1590,"client":"2","":"downstairs","session_id":"660ce876-457b-4105-b27c-9c2132e8bbfa"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:08.96169462Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1590,"session_id":"660ce876-457b-4105-b27c-9c2132e8bbfa"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:08.961706497Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1590,"session_id":"660ce876-457b-4105-b27c-9c2132e8bbfa"} {"msg":"ad0ad434-7096-431a-adce-f2ca9f2e0589 is now active with session: 660ce876-457b-4105-b27c-9c2132e8bbfa","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:08.961719014Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1590,"session_id":"660ce876-457b-4105-b27c-9c2132e8bbfa"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:08.961730681Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1590,"session_id":"660ce876-457b-4105-b27c-9c2132e8bbfa"} {"msg":"The guest has finished waiting for activation with:102","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:08.961742618Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1590} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 329 len 4096 data: 7 02/30 Flush 03/30 Write block 512 len 32768 data: 8 10 11 14 14 10 10 10 04/30 Write block 286 len 8192 data: 7 6 05/30 Read block 181 len 12288 06/30 Read block 249 len 20480 07/30 Write block 419 len 12288 data: 6 6 8 08/30 Write block 139 len 16384 data: 9 8 7 8 09/30 Read block 558 len 32768 10/30 Read block 455 len 32768 11/30 Read block 503 len 12288 12/30 Read block 103 len 4096 13/30 Read block 307 len 4096 14/30 Write block 269 len 28672 data: 9 10 9 7 7 7 8 15/30 Write block 498 len 4096 data: 12 16/30 Read block 94 len 40960 17/30 Write block 84 len 20480 data: 7 7 6 6 7 18/30 Read block 85 len 12288 19/30 Read block 250 len 20480 20/30 Flush 21/30 Write block 362 len 28672 data: 6 6 5 5 5 4 4 22/30 Read block 313 len 40960 23/30 Read block 222 len 4096 24/30 Read block 326 len 4096 25/30 Write block 562 len 8192 data: 13 12 26/30 Read block 53 len 36864 27/30 Write block 27 len 4096 data: 9 28/30 Read block 308 len 4096 29/30 Write block 562 len 20480 data: 14 13 9 8 11 30/30 Read block 102 len 20480 ---------------------------------------------------------------- Crucible gen:102 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1020 Acked Write 7 Done Done Done false 1021 Acked Read 10 Done Done Done false 1022 Acked Read 1 Done Done Done false 1023 Acked Read 1 Done Done Done false 1024 Acked Write 2 Done Done Done false 1025 Acked Read 9 Done Done Done false 1026 Acked Write 1 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Write 5 Done Done Done false 1029 Acked Read 5 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 1 2 Done 9 10 9 28 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1019 1019 1019 Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:15:08.980286276Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1590} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 98 98 98 181 181 181 F F F 1 020-039 101 101 101 188 188 188 T T T 2 040-059 98 98 98 180 180 180 F F F 3 060-079 90 90 90 167 167 167 F F F 4 080-099 102 102 102 190 190 190 F F F 5 100-119 96 96 96 177 177 177 F F F 6 120-139 102 102 102 190 190 190 F F F 7 140-159 102 102 102 190 190 190 F F F 8 160-179 101 101 101 188 188 188 F F F 9 180-199 100 100 100 187 187 187 F F F 10 200-219 97 97 97 178 178 178 F F F 11 220-239 100 100 100 186 186 186 F F F 12 240-259 100 100 100 185 185 185 F F F 13 260-279 102 102 102 190 190 190 F F F 14 280-299 102 102 102 190 190 190 F F F 15 300-319 95 95 95 175 175 175 F F F 16 320-339 102 102 102 189 189 189 F F F 17 340-359 98 98 98 180 180 180 F F F 18 360-379 98 98 98 180 180 180 T T T 19 380-399 96 96 96 176 176 176 F F F 20 400-419 102 102 102 190 190 190 F F F 21 420-439 102 102 102 190 190 190 F F F 22 440-459 98 98 98 181 181 181 F F F 23 460-479 100 100 100 186 186 186 F F F 24 480-499 102 102 102 190 190 190 F F F 25 500-519 102 102 102 190 190 190 F F F 26 520-539 94 94 94 174 174 174 F F F 27 540-559 94 94 94 174 174 174 F F F 28 560-579 100 100 100 183 183 183 T T T 29 580-599 100 100 100 183 183 183 F F F Max gen: 102, Max flush: 190 A Difference in extent metadata is expected here On loop 51 Verifying data now No change after verify Loop: 51 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 98 98 98 181 181 181 F F F 1 020-039 103 103 103 191 191 191 F F F 2 040-059 98 98 98 180 180 180 F F F 3 060-079 90 90 90 167 167 167 F F F 4 080-099 102 102 102 190 190 190 F F F 5 100-119 96 96 96 177 177 177 F F F 6 120-139 102 102 102 190 190 190 F F F 7 140-159 102 102 102 190 190 190 F F F 8 160-179 101 101 101 188 188 188 F F F 9 180-199 100 100 100 187 187 187 F F F 10 200-219 97 97 97 178 178 178 F F F 11 220-239 100 100 100 186 186 186 F F F 12 240-259 100 100 100 185 185 185 F F F 13 260-279 102 102 102 190 190 190 F F F 14 280-299 102 102 102 190 190 190 F F F 15 300-319 95 95 95 175 175 175 F F F 16 320-339 102 102 102 189 189 189 F F F 17 340-359 98 98 98 180 180 180 F F F 18 360-379 103 103 103 191 191 191 F F F 19 380-399 96 96 96 176 176 176 F F F 20 400-419 102 102 102 190 190 190 F F F 21 420-439 102 102 102 190 190 190 F F F 22 440-459 98 98 98 181 181 181 F F F 23 460-479 100 100 100 186 186 186 F F F 24 480-499 102 102 102 190 190 190 F F F 25 500-519 102 102 102 190 190 190 F F F 26 520-539 94 94 94 174 174 174 F F F 27 540-559 94 94 94 174 174 174 F F F 28 560-579 103 103 103 191 191 191 F F F 29 580-599 100 100 100 183 183 183 F F F Max gen: 103, Max flush: 191 Begin loop 52 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:13.211576944Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1598,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:13.256354025Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1598,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:13.25703041Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1598} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"abd5430f1508af0bc008143ea2566277ef714ec2\",\n git_commit_timestamp: \"2025-01-17T20:55:03.000000000Z\",\n git_branch: \"alan/tokio-to-140\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:13.257072835Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1598} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:13.257087604Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1598} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:13.257102913Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1598} {"msg":"Crucible 001b205f-8950-4a11-b52c-81a1a3624a19 has session id: 392291f3-7b73-49f3-8a28-5ecc9c6d6098","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:13.257659128Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1598,"session_id":"392291f3-7b73-49f3-8a28-5ecc9c6d6098"} {"msg":"Upstairs opts: Upstairs UUID: 001b205f-8950-4a11-b52c-81a1a3624a19, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:13.257699261Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1598,"session_id":"392291f3-7b73-49f3-8a28-5ecc9c6d6098"} {"msg":"Crucible stats registered with UUID: 001b205f-8950-4a11-b52c-81a1a3624a19","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:13.257729409Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1598,"session_id":"392291f3-7b73-49f3-8a28-5ecc9c6d6098"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:15:13.257751362Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1598,"session_id":"392291f3-7b73-49f3-8a28-5ecc9c6d6098"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:13.258144061Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1598,"task":"crutest"} {"msg":"The guest has requested activation with gen:104","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:13.258168456Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1598} {"msg":"001b205f-8950-4a11-b52c-81a1a3624a19 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:13.258232153Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1598,"session_id":"392291f3-7b73-49f3-8a28-5ecc9c6d6098"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:13.258250263Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1598,"client":"0","":"downstairs","session_id":"392291f3-7b73-49f3-8a28-5ecc9c6d6098"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:13.258273077Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1598,"client":"0","":"downstairs","session_id":"392291f3-7b73-49f3-8a28-5ecc9c6d6098"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:13.258287395Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1598,"client":"1","":"downstairs","session_id":"392291f3-7b73-49f3-8a28-5ecc9c6d6098"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:13.25831207Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1598,"client":"1","":"downstairs","session_id":"392291f3-7b73-49f3-8a28-5ecc9c6d6098"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:13.258337194Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1598,"client":"2","":"downstairs","session_id":"392291f3-7b73-49f3-8a28-5ecc9c6d6098"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:13.25836342Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1598,"client":"2","":"downstairs","session_id":"392291f3-7b73-49f3-8a28-5ecc9c6d6098"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:13.258388274Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1598,"":"io task","client":"2","":"downstairs","session_id":"392291f3-7b73-49f3-8a28-5ecc9c6d6098"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:13.258414219Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1598,"":"io task","client":"0","":"downstairs","session_id":"392291f3-7b73-49f3-8a28-5ecc9c6d6098"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:13.258438994Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1598,"":"io task","client":"1","":"downstairs","session_id":"392291f3-7b73-49f3-8a28-5ecc9c6d6098"} {"msg":"ds_connection connected from Ok(127.0.0.1:45519)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:13.258465089Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1598,"":"io task","client":"2","":"downstairs","session_id":"392291f3-7b73-49f3-8a28-5ecc9c6d6098"} {"msg":"ds_connection connected from Ok(127.0.0.1:58799)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:13.258650648Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1598,"":"io task","client":"1","":"downstairs","session_id":"392291f3-7b73-49f3-8a28-5ecc9c6d6098"} {"msg":"ds_connection connected from Ok(127.0.0.1:50627)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:13.25868872Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1598,"":"io task","client":"0","":"downstairs","session_id":"392291f3-7b73-49f3-8a28-5ecc9c6d6098"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:13.259196997Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1598,"client":"2","":"downstairs","session_id":"392291f3-7b73-49f3-8a28-5ecc9c6d6098"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:13.25923672Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1598,"client":"0","":"downstairs","session_id":"392291f3-7b73-49f3-8a28-5ecc9c6d6098"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:13.259326492Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1598,"client":"1","":"downstairs","session_id":"392291f3-7b73-49f3-8a28-5ecc9c6d6098"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:13.259825454Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1598,"client":"2","":"downstairs","session_id":"392291f3-7b73-49f3-8a28-5ecc9c6d6098"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:13.259866408Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1598,"client":"1","":"downstairs","session_id":"392291f3-7b73-49f3-8a28-5ecc9c6d6098"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:13.259933497Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1598,"client":"0","":"downstairs","session_id":"392291f3-7b73-49f3-8a28-5ecc9c6d6098"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:13.260423694Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1598,"session_id":"392291f3-7b73-49f3-8a28-5ecc9c6d6098"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:13.260459835Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1598,"session_id":"392291f3-7b73-49f3-8a28-5ecc9c6d6098"} {"msg":"[0]R flush_numbers[0..12]: [181, 191, 180, 167, 190, 177, 190, 190, 188, 187, 178, 186]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:13.260516198Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1598,"":"downstairs","":"downstairs","session_id":"392291f3-7b73-49f3-8a28-5ecc9c6d6098"} {"msg":"[0]R generation[0..12]: [98, 103, 98, 90, 102, 96, 102, 102, 101, 100, 97, 100]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:13.260542533Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1598,"":"downstairs","":"downstairs","session_id":"392291f3-7b73-49f3-8a28-5ecc9c6d6098"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:13.260557302Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1598,"":"downstairs","":"downstairs","session_id":"392291f3-7b73-49f3-8a28-5ecc9c6d6098"} {"msg":"[1]R flush_numbers[0..12]: [181, 191, 180, 167, 190, 177, 190, 190, 188, 187, 178, 186]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:13.26057122Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1598,"":"downstairs","":"downstairs","session_id":"392291f3-7b73-49f3-8a28-5ecc9c6d6098"} {"msg":"[1]R generation[0..12]: [98, 103, 98, 90, 102, 96, 102, 102, 101, 100, 97, 100]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:13.26058906Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1598,"":"downstairs","":"downstairs","session_id":"392291f3-7b73-49f3-8a28-5ecc9c6d6098"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:13.260603449Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1598,"":"downstairs","":"downstairs","session_id":"392291f3-7b73-49f3-8a28-5ecc9c6d6098"} {"msg":"[2]R flush_numbers[0..12]: [181, 191, 180, 167, 190, 177, 190, 190, 188, 187, 178, 186]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:13.260617127Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1598,"":"downstairs","":"downstairs","session_id":"392291f3-7b73-49f3-8a28-5ecc9c6d6098"} {"msg":"[2]R generation[0..12]: [98, 103, 98, 90, 102, 96, 102, 102, 101, 100, 97, 100]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:13.260630425Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1598,"":"downstairs","":"downstairs","session_id":"392291f3-7b73-49f3-8a28-5ecc9c6d6098"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:13.260643902Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1598,"":"downstairs","":"downstairs","session_id":"392291f3-7b73-49f3-8a28-5ecc9c6d6098"} {"msg":"Max found gen is 104","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:13.26065632Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1598,"":"downstairs","":"downstairs","session_id":"392291f3-7b73-49f3-8a28-5ecc9c6d6098"} {"msg":"Generation requested: 104 >= found:104","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:13.260669027Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1598,"":"downstairs","":"downstairs","session_id":"392291f3-7b73-49f3-8a28-5ecc9c6d6098"} {"msg":"Next flush: 192","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:13.260681595Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1598,"":"downstairs","":"downstairs","session_id":"392291f3-7b73-49f3-8a28-5ecc9c6d6098"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:13.260694042Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1598,"":"downstairs","":"downstairs","session_id":"392291f3-7b73-49f3-8a28-5ecc9c6d6098"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:13.260706609Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1598,"session_id":"392291f3-7b73-49f3-8a28-5ecc9c6d6098"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-17T21:15:13.260718106Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1598,"client":"0","":"downstairs","session_id":"392291f3-7b73-49f3-8a28-5ecc9c6d6098"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:13.260740039Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1598,"client":"1","":"downstairs","session_id":"392291f3-7b73-49f3-8a28-5ecc9c6d6098"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:13.260752746Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1598,"client":"2","":"downstairs","session_id":"392291f3-7b73-49f3-8a28-5ecc9c6d6098"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:13.260765584Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1598,"session_id":"392291f3-7b73-49f3-8a28-5ecc9c6d6098"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:13.2607775Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1598,"session_id":"392291f3-7b73-49f3-8a28-5ecc9c6d6098"} {"msg":"001b205f-8950-4a11-b52c-81a1a3624a19 is now active with session: 392291f3-7b73-49f3-8a28-5ecc9c6d6098","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:13.260791509Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1598,"session_id":"392291f3-7b73-49f3-8a28-5ecc9c6d6098"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:13.260803416Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1598,"session_id":"392291f3-7b73-49f3-8a28-5ecc9c6d6098"} {"msg":"The guest has finished waiting for activation with:104","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:13.260815583Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1598} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 302 len 4096 data: 8 02/30 Write block 317 len 36864 data: 6 6 7 8 7 7 9 8 9 03/30 Write block 41 len 16384 data: 6 5 4 5 04/30 Write block 292 len 32768 data: 12 11 9 11 11 9 8 10 05/30 Flush 06/30 Write block 573 len 20480 data: 6 7 6 7 7 07/30 Write block 199 len 16384 data: 13 13 11 11 08/30 Write block 581 len 24576 data: 9 8 9 10 9 9 09/30 Write block 288 len 16384 data: 8 8 9 11 10/30 Read block 334 len 4096 11/30 Write block 29 len 20480 data: 7 10 8 11 11 12/30 Read block 103 len 24576 13/30 Write block 375 len 24576 data: 13 12 14 15 15 12 14/30 Flush 15/30 Write block 277 len 12288 data: 9 7 5 16/30 Write block 461 len 16384 data: 11 11 10 10 17/30 Read block 284 len 8192 18/30 Write block 250 len 36864 data: 7 6 7 6 6 8 9 9 9 19/30 Write block 71 len 4096 data: 10 20/30 Read block 214 len 24576 21/30 Read block 541 len 32768 22/30 Read block 337 len 8192 23/30 Read block 453 len 20480 24/30 Write block 560 len 8192 data: 10 11 25/30 Write block 570 len 20480 data: 9 8 6 7 8 26/30 Write block 421 len 12288 data: 9 9 9 27/30 Write block 45 len 20480 data: 5 7 7 7 6 28/30 Read block 57 len 12288 29/30 Write block 571 len 36864 data: 9 7 8 9 7 8 8 7 9 30/30 Write block 489 len 40960 data: 8 8 8 6 7 9 11 11 11 13 ---------------------------------------------------------------- Crucible gen:104 GIO:true work queues: Upstairs:0 downstairs:16 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1014 Acked Write 3 Done Done Done false 1015 Acked Write 4 Done Done Done false 1016 Acked Read 2 Done Done Done false 1017 Acked Write 9 Done Done Done false 1018 Acked Write 1 Done Done Done false 1019 Acked Read 6 Done Done Done false 1020 Acked Read 8 Done Done Done false 1021 Acked Read 2 Done Done Done false 1022 Acked Read 5 Done Done Done false 1023 Acked Write 2 Done Done Done false 1024 Acked Write 5 Done Done Done false 1025 Acked Write 3 Done Done Done false 1026 Acked Write 5 Done Done Done false 1027 Acked Read 3 Done Done Done false 1028 Acked Write 9 Done Done Done false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 15 15 15 45 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1013 1013 1013 Downstairs last five completed: 1013 1012 1011 1010 1009 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:15:13.278317222Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1598} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 98 98 98 181 181 181 F F F 1 020-039 104 104 104 193 193 193 F F F 2 040-059 104 104 104 192 192 192 T T T 3 060-079 90 90 90 167 167 167 T T T 4 080-099 102 102 102 190 190 190 F F F 5 100-119 96 96 96 177 177 177 F F F 6 120-139 102 102 102 190 190 190 F F F 7 140-159 102 102 102 190 190 190 F F F 8 160-179 101 101 101 188 188 188 F F F 9 180-199 104 104 104 193 193 193 F F F 10 200-219 104 104 104 193 193 193 F F F 11 220-239 100 100 100 186 186 186 F F F 12 240-259 100 100 100 185 185 185 T T T 13 260-279 102 102 102 190 190 190 T T T 14 280-299 104 104 104 193 193 193 F F F 15 300-319 104 104 104 192 192 192 F F F 16 320-339 104 104 104 192 192 192 F F F 17 340-359 98 98 98 180 180 180 F F F 18 360-379 104 104 104 193 193 193 F F F 19 380-399 104 104 104 193 193 193 F F F 20 400-419 102 102 102 190 190 190 F F F 21 420-439 102 102 102 190 190 190 T T T 22 440-459 98 98 98 181 181 181 F F F 23 460-479 100 100 100 186 186 186 T T T 24 480-499 102 102 102 190 190 190 T T T 25 500-519 102 102 102 190 190 190 F F F 26 520-539 94 94 94 174 174 174 F F F 27 540-559 94 94 94 174 174 174 F F F 28 560-579 104 104 104 193 193 193 T T T 29 580-599 104 104 104 193 193 193 F F F Max gen: 104, Max flush: 193 A Difference in extent metadata is expected here On loop 52 Verifying data now No change after verify Loop: 52 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 98 98 98 181 181 181 F F F 1 020-039 104 104 104 193 193 193 F F F 2 040-059 105 105 105 194 194 194 F F F 3 060-079 105 105 105 194 194 194 F F F 4 080-099 102 102 102 190 190 190 F F F 5 100-119 96 96 96 177 177 177 F F F 6 120-139 102 102 102 190 190 190 F F F 7 140-159 102 102 102 190 190 190 F F F 8 160-179 101 101 101 188 188 188 F F F 9 180-199 104 104 104 193 193 193 F F F 10 200-219 104 104 104 193 193 193 F F F 11 220-239 100 100 100 186 186 186 F F F 12 240-259 105 105 105 194 194 194 F F F 13 260-279 105 105 105 194 194 194 F F F 14 280-299 104 104 104 193 193 193 F F F 15 300-319 104 104 104 192 192 192 F F F 16 320-339 104 104 104 192 192 192 F F F 17 340-359 98 98 98 180 180 180 F F F 18 360-379 104 104 104 193 193 193 F F F 19 380-399 104 104 104 193 193 193 F F F 20 400-419 102 102 102 190 190 190 F F F 21 420-439 105 105 105 194 194 194 F F F 22 440-459 98 98 98 181 181 181 F F F 23 460-479 105 105 105 194 194 194 F F F 24 480-499 105 105 105 194 194 194 F F F 25 500-519 102 102 102 190 190 190 F F F 26 520-539 94 94 94 174 174 174 F F F 27 540-559 94 94 94 174 174 174 F F F 28 560-579 105 105 105 194 194 194 F F F 29 580-599 104 104 104 193 193 193 F F F Max gen: 105, Max flush: 194 Begin loop 53 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:17.536116804Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1606,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:15:17.579642493Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1606,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:17.580075366Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1606,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:17.611960842Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1606,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:17.612565456Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1606} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"abd5430f1508af0bc008143ea2566277ef714ec2\",\n git_commit_timestamp: \"2025-01-17T20:55:03.000000000Z\",\n git_branch: \"alan/tokio-to-140\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:17.612604549Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1606} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:17.612620898Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1606} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:17.612638058Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1606} {"msg":"Crucible dd882cb4-5458-4b30-b1f2-8c44e4bd0da0 has session id: f1c9301a-fb07-40a3-a8da-c7e298588fdd","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:17.613228153Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1606,"session_id":"f1c9301a-fb07-40a3-a8da-c7e298588fdd"} {"msg":"Upstairs opts: Upstairs UUID: dd882cb4-5458-4b30-b1f2-8c44e4bd0da0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:17.613275501Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1606,"session_id":"f1c9301a-fb07-40a3-a8da-c7e298588fdd"} {"msg":"Crucible stats registered with UUID: dd882cb4-5458-4b30-b1f2-8c44e4bd0da0","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:17.613305499Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1606,"session_id":"f1c9301a-fb07-40a3-a8da-c7e298588fdd"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:15:17.613331624Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1606,"session_id":"f1c9301a-fb07-40a3-a8da-c7e298588fdd"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:17.613751329Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1606,"task":"crutest"} {"msg":"The guest has requested activation with gen:106","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:17.613779436Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1606} {"msg":"dd882cb4-5458-4b30-b1f2-8c44e4bd0da0 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:17.613811234Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1606,"session_id":"f1c9301a-fb07-40a3-a8da-c7e298588fdd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:17.613828114Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1606,"client":"0","":"downstairs","session_id":"f1c9301a-fb07-40a3-a8da-c7e298588fdd"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:17.613859552Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1606,"client":"0","":"downstairs","session_id":"f1c9301a-fb07-40a3-a8da-c7e298588fdd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:17.613876752Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1606,"client":"1","":"downstairs","session_id":"f1c9301a-fb07-40a3-a8da-c7e298588fdd"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:17.613899426Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1606,"client":"1","":"downstairs","session_id":"f1c9301a-fb07-40a3-a8da-c7e298588fdd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:17.613929483Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1606,"client":"2","":"downstairs","session_id":"f1c9301a-fb07-40a3-a8da-c7e298588fdd"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:17.613963413Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1606,"client":"2","":"downstairs","session_id":"f1c9301a-fb07-40a3-a8da-c7e298588fdd"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:17.61399213Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1606,"":"io task","client":"2","":"downstairs","session_id":"f1c9301a-fb07-40a3-a8da-c7e298588fdd"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:17.614017505Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1606,"":"io task","client":"0","":"downstairs","session_id":"f1c9301a-fb07-40a3-a8da-c7e298588fdd"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:17.614041258Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1606,"":"io task","client":"1","":"downstairs","session_id":"f1c9301a-fb07-40a3-a8da-c7e298588fdd"} {"msg":"ds_connection connected from Ok(127.0.0.1:55101)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:17.614064772Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1606,"":"io task","client":"2","":"downstairs","session_id":"f1c9301a-fb07-40a3-a8da-c7e298588fdd"} {"msg":"ds_connection connected from Ok(127.0.0.1:41372)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:17.614091378Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1606,"":"io task","client":"1","":"downstairs","session_id":"f1c9301a-fb07-40a3-a8da-c7e298588fdd"} {"msg":"ds_connection connected from Ok(127.0.0.1:57022)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:17.614116562Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1606,"":"io task","client":"0","":"downstairs","session_id":"f1c9301a-fb07-40a3-a8da-c7e298588fdd"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:17.614913358Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1606,"client":"2","":"downstairs","session_id":"f1c9301a-fb07-40a3-a8da-c7e298588fdd"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:17.614941405Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1606,"client":"1","":"downstairs","session_id":"f1c9301a-fb07-40a3-a8da-c7e298588fdd"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:17.615030477Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1606,"client":"0","":"downstairs","session_id":"f1c9301a-fb07-40a3-a8da-c7e298588fdd"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:17.615594766Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1606,"client":"1","":"downstairs","session_id":"f1c9301a-fb07-40a3-a8da-c7e298588fdd"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:17.615633209Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1606,"client":"0","":"downstairs","session_id":"f1c9301a-fb07-40a3-a8da-c7e298588fdd"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:17.615663707Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1606,"client":"2","":"downstairs","session_id":"f1c9301a-fb07-40a3-a8da-c7e298588fdd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:17.616208765Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1606,"session_id":"f1c9301a-fb07-40a3-a8da-c7e298588fdd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:17.61623442Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1606,"session_id":"f1c9301a-fb07-40a3-a8da-c7e298588fdd"} {"msg":"[0]R flush_numbers[0..12]: [181, 193, 194, 194, 190, 177, 190, 190, 188, 193, 193, 186]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:17.616276255Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1606,"":"downstairs","":"downstairs","session_id":"f1c9301a-fb07-40a3-a8da-c7e298588fdd"} {"msg":"[0]R generation[0..12]: [98, 104, 105, 105, 102, 96, 102, 102, 101, 104, 104, 100]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:17.616294045Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1606,"":"downstairs","":"downstairs","session_id":"f1c9301a-fb07-40a3-a8da-c7e298588fdd"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:17.616308243Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1606,"":"downstairs","":"downstairs","session_id":"f1c9301a-fb07-40a3-a8da-c7e298588fdd"} {"msg":"[1]R flush_numbers[0..12]: [181, 193, 194, 194, 190, 177, 190, 190, 188, 193, 193, 186]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:17.616321691Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1606,"":"downstairs","":"downstairs","session_id":"f1c9301a-fb07-40a3-a8da-c7e298588fdd"} {"msg":"[1]R generation[0..12]: [98, 104, 105, 105, 102, 96, 102, 102, 101, 104, 104, 100]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:17.616335379Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1606,"":"downstairs","":"downstairs","session_id":"f1c9301a-fb07-40a3-a8da-c7e298588fdd"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:17.616349218Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1606,"":"downstairs","":"downstairs","session_id":"f1c9301a-fb07-40a3-a8da-c7e298588fdd"} {"msg":"[2]R flush_numbers[0..12]: [181, 193, 194, 194, 190, 177, 190, 190, 188, 193, 193, 186]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:17.616362695Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1606,"":"downstairs","":"downstairs","session_id":"f1c9301a-fb07-40a3-a8da-c7e298588fdd"} {"msg":"[2]R generation[0..12]: [98, 104, 105, 105, 102, 96, 102, 102, 101, 104, 104, 100]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:17.616376123Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1606,"":"downstairs","":"downstairs","session_id":"f1c9301a-fb07-40a3-a8da-c7e298588fdd"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:17.616389501Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1606,"":"downstairs","":"downstairs","session_id":"f1c9301a-fb07-40a3-a8da-c7e298588fdd"} {"msg":"Max found gen is 106","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:17.616406521Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1606,"":"downstairs","":"downstairs","session_id":"f1c9301a-fb07-40a3-a8da-c7e298588fdd"} {"msg":"Generation requested: 106 >= found:106","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:17.616419999Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1606,"":"downstairs","":"downstairs","session_id":"f1c9301a-fb07-40a3-a8da-c7e298588fdd"} {"msg":"Next flush: 195","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:17.616432506Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1606,"":"downstairs","":"downstairs","session_id":"f1c9301a-fb07-40a3-a8da-c7e298588fdd"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:17.616444963Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1606,"":"downstairs","":"downstairs","session_id":"f1c9301a-fb07-40a3-a8da-c7e298588fdd"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:17.616457721Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1606,"session_id":"f1c9301a-fb07-40a3-a8da-c7e298588fdd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:17.616469217Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1606,"client":"0","":"downstairsWait for a query_work_queue command to finish before sending IO ","session_id":"f1c9301a-fb07-40a3-a8da-c7e298588fdd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:17.616488929Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1606,"client":"1","":"downstairs","session_id":"f1c9301a-fb07-40a3-a8da-c7e298588fdd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:17.616501656Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1606,"client":"2","":"downstairs","session_id":"f1c9301a-fb07-40a3-a8da-c7e298588fdd"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:17.616514364Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1606,"session_id":"f1c9301a-fb07-40a3-a8da-c7e298588fdd"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:17.616526211Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1606,"session_id":"f1c9301a-fb07-40a3-a8da-c7e298588fdd"} {"msg":"dd882cb4-5458-4b30-b1f2-8c44e4bd0da0 is now active with session: f1c9301a-fb07-40a3-a8da-c7e298588fdd","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:17.616538648Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1606,"session_id":"f1c9301a-fb07-40a3-a8da-c7e298588fdd"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:17.616550295Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1606,"session_id":"f1c9301a-fb07-40a3-a8da-c7e298588fdd"} {"msg":"The guest has finished waiting for activation with:106","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:17.616562032Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1606} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 257 len 16384 data: 10 10 9 8 02/30 Write block 558 len 24576 data: 8 7 11 12 15 14 03/30 Read block 531 len 28672 04/30 Read block 341 len 20480 05/30 Write block 498 len 24576 data: 14 12 14 14 12 12 06/30 Read block 58 len 20480 07/30 Read block 251 len 8192 08/30 Write block 474 len 32768 data: 9 10 9 7 6 6 7 6 09/30 Read block 148 len 32768 10/30 Write block 237 len 4096 data: 10 11/30 Write block 234 len 20480 data: 10 11 10 11 6 12/30 Read block 209 len 8192 13/30 Write block 337 len 20480 data: 12 13 14 14 12 14/30 Read block 428 len 12288 15/30 Write block 136 len 40960 data: 8 8 8 10 9 8 9 8 7 7 16/30 Read block 277 len 16384 17/30 Write block 557 len 20480 data: 7 9 8 12 13 18/30 Write block 449 len 8192 data: 6 5 19/30 Write block 341 len 8192 data: 13 10 20/30 Read block 404 len 28672 21/30 Read block 369 len 28672 22/30 Read block 65 len 40960 23/30 Write block 139 len 8192 data: 11 10 24/30 Flush 25/30 Flush 26/30 Write block 177 len 8192 data: 10 7 27/30 Read block 121 len 24576 28/30 Write block 79 len 24576 data: 7 6 7 7 8 8 29/30 Write block 514 len 32768 data: 12 15 15 11 11 11 9 10 30/30 Write block 49 len 32768 data: 7 7 9 13 13 13 12 11 ---------------------------------------------------------------- Crucible gen:106 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 2 Done Done Done false 1026 Acked Read 6 Done Done Done false 1027 Acked Write 6 Done Done Done false 1028 Acked Write 8 Sent Sent Sent false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:15:17.635605973Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1606} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 98 98 98 181 181 181 F F F 1 020-039 104 104 104 193 193 193 F F F 2 040-059 105 105 105 194 194 194 T T T 3 060-079 105 105 105 194 194 194 T T T 4 080-099 102 102 102 190 190 190 T T T 5 100-119 96 96 96 177 177 177 F F F 6 120-139 106 106 106 195 195 195 F F F 7 140-159 106 106 106 195 195 195 F F F 8 160-179 101 101 101 188 188 188 T T T 9 180-199 104 104 104 193 193 193 F F F 10 200-219 104 104 104 193 193 193 F F F 11 220-239 106 106 106 195 195 195 F F F 12 240-259 106 106 106 195 195 195 F F F 13 260-279 106 106 106 195 195 195 F F F 14 280-299 104 104 104 193 193 193 F F F 15 300-319 104 104 104 192 192 192 F F F 16 320-339 106 106 106 195 195 195 F F F 17 340-359 106 106 106 195 195 195 F F F 18 360-379 104 104 104 193 193 193 F F F 19 380-399 104 104 104 193 193 193 F F F 20 400-419 102 102 102 190 190 190 F F F 21 420-439 105 105 105 194 194 194 F F F 22 440-459 106 106 106 195 195 195 F F F 23 460-479 106 106 106 195 195 195 F F F 24 480-499 106 106 106 195 195 195 F F F 25 500-519 106 106 106 195 195 195 T T T 26 520-539 94 94 94 174 174 174 T T T 27 540-559 106 106 106 195 195 195 F F F 28 560-579 106 106 106 195 195 195 F F F 29 580-599 104 104 104 193 193 193 F F F Max gen: 106, Max flush: 195 A Difference in extent metadata is expected here On loop 53 Verifying data now No change after verify Loop: 53 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 98 98 98 181 181 181 F F F 1 020-039 104 104 104 193 193 193 F F F 2 040-059 107 107 107 196 196 196 F F F 3 060-079 107 107 107 196 196 196 F F F 4 080-099 107 107 107 196 196 196 F F F 5 100-119 96 96 96 177 177 177 F F F 6 120-139 106 106 106 195 195 195 F F F 7 140-159 106 106 106 195 195 195 F F F 8 160-179 107 107 107 196 196 196 F F F 9 180-199 104 104 104 193 193 193 F F F 10 200-219 104 104 104 193 193 193 F F F 11 220-239 106 106 106 195 195 195 F F F 12 240-259 106 106 106 195 195 195 F F F 13 260-279 106 106 106 195 195 195 F F F 14 280-299 104 104 104 193 193 193 F F F 15 300-319 104 104 104 192 192 192 F F F 16 320-339 106 106 106 195 195 195 F F F 17 340-359 106 106 106 195 195 195 F F F 18 360-379 104 104 104 193 193 193 F F F 19 380-399 104 104 104 193 193 193 F F F 20 400-419 102 102 102 190 190 190 F F F 21 420-439 105 105 105 194 194 194 F F F 22 440-459 106 106 106 195 195 195 F F F 23 460-479 106 106 106 195 195 195 F F F 24 480-499 106 106 106 195 195 195 F F F 25 500-519 107 107 107 196 196 196 F F F 26 520-539 107 107 107 196 196 196 F F F 27 540-559 106 106 106 195 195 195 F F F 28 560-579 106 106 106 195 195 195 F F F 29 580-599 104 104 104 193 193 193 F F F Max gen: 107, Max flush: 196 Begin loop 54 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:21.910941813Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1614,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:21.956209171Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1614,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:21.956945711Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1614} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"abd5430f1508af0bc008143ea2566277ef714ec2\",\n git_commit_timestamp: \"2025-01-17T20:55:03.000000000Z\",\n git_branch: \"alan/tokio-to-140\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:21.956985804Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1614} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:21.956999933Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1614} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:21.957014281Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1614} {"msg":"Crucible 9fd6cb40-57bb-42da-b118-a96092a45f8f has session id: 2d5c91bb-897e-436f-937c-245ae9798b74","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:21.957555477Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1614,"session_id":"2d5c91bb-897e-436f-937c-245ae9798b74"} {"msg":"Upstairs opts: Upstairs UUID: 9fd6cb40-57bb-42da-b118-a96092a45f8f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:21.957596792Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1614,"session_id":"2d5c91bb-897e-436f-937c-245ae9798b74"} {"msg":"Crucible stats registered with UUID: 9fd6cb40-57bb-42da-b118-a96092a45f8f","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:21.957619305Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1614,"session_id":"2d5c91bb-897e-436f-937c-245ae9798b74"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:15:21.957633583Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1614,"session_id":"2d5c91bb-897e-436f-937c-245ae9798b74"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:21.958109211Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1614,"task":"crutest"} {"msg":"The guest has requested activation with gen:108","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:21.958135387Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1614} {"msg":"9fd6cb40-57bb-42da-b118-a96092a45f8f active request set","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:21.958194851Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1614,"session_id":"2d5c91bb-897e-436f-937c-245ae9798b74"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:21.958213032Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1614,"client":"0","":"downstairs","session_id":"2d5c91bb-897e-436f-937c-245ae9798b74"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:21.958237086Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1614,"client":"0","":"downstairs","session_id":"2d5c91bb-897e-436f-937c-245ae9798b74"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:21.958250844Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1614,"client":"1","":"downstairs","session_id":"2d5c91bb-897e-436f-937c-245ae9798b74"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:21.958273777Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1614,"client":"1","":"downstairs","session_id":"2d5c91bb-897e-436f-937c-245ae9798b74"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:21.958300533Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1614,"client":"2","":"downstairs","session_id":"2d5c91bb-897e-436f-937c-245ae9798b74"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:21.958351983Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1614,"client":"2","":"downstairs","session_id":"2d5c91bb-897e-436f-937c-245ae9798b74"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:21.958501661Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1614,"":"io task","client":"2","":"downstairs","session_id":"2d5c91bb-897e-436f-937c-245ae9798b74"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:21.958602669Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1614,"":"io task","client":"0","":"downstairs","session_id":"2d5c91bb-897e-436f-937c-245ae9798b74"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:21.958630196Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1614,"":"io task","client":"1","":"downstairs","session_id":"2d5c91bb-897e-436f-937c-245ae9798b74"} {"msg":"ds_connection connected from Ok(127.0.0.1:43706)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:21.958658102Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1614,"":"io task","client":"2","":"downstairs","session_id":"2d5c91bb-897e-436f-937c-245ae9798b74"} {"msg":"ds_connection connected from Ok(127.0.0.1:42143)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:21.958683587Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1614,"":"io task","client":"0","":"downstairs","session_id":"2d5c91bb-897e-436f-937c-245ae9798b74"} {"msg":"ds_connection connected from Ok(127.0.0.1:58657)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:21.958709302Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1614,"":"io task","client":"1","":"downstairs","session_id":"2d5c91bb-897e-436f-937c-245ae9798b74"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:21.95916776Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1614,"client":"2","":"downstairs","session_id":"2d5c91bb-897e-436f-937c-245ae9798b74"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:21.959206152Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1614,"client":"0","":"downstairs","session_id":"2d5c91bb-897e-436f-937c-245ae9798b74"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:21.959222692Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1614,"client":"1","":"downstairs","session_id":"2d5c91bb-897e-436f-937c-245ae9798b74"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:21.959808685Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1614,"client":"2","":"downstairs","session_id":"2d5c91bb-897e-436f-937c-245ae9798b74"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:21.959845196Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1614,"client":"1","":"downstairs","session_id":"2d5c91bb-897e-436f-937c-245ae9798b74"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:21.959973851Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1614,"client":"0","":"downstairs","session_id":"2d5c91bb-897e-436f-937c-245ae9798b74"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:21.960405513Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1614,"session_id":"2d5c91bb-897e-436f-937c-245ae9798b74"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:21.960441234Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1614,"session_id":"2d5c91bb-897e-436f-937c-245ae9798b74"} {"msg":"[0]R flush_numbers[0..12]: [181, 193, 196, 196, 196, 177, 195, 195, 196, 193, 193, 195]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:21.960503491Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1614,"":"downstairs","":"downstairs","session_id":"2d5c91bb-897e-436f-937c-245ae9798b74"} {"msg":"[0]R generation[0..12]: [98, 104, 107, 107, 107, 96, 106, 106, 107, 104, 104, 106]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:21.960530667Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1614,"":"downstairs","":"downstairs","session_id":"2d5c91bb-897e-436f-937c-245ae9798b74"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:21.960545845Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1614,"":"downstairs","":"downstairs","session_id":"2d5c91bb-897e-436f-937c-245ae9798b74"} {"msg":"[1]R flush_numbers[0..12]: [181, 193, 196, 196, 196, 177, 195, 195, 196, 193, 193, 195]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:21.960560674Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1614,"":"downstairs","":"downstairs","session_id":"2d5c91bb-897e-436f-937c-245ae9798b74"} {"msg":"[1]R generation[0..12]: [98, 104, 107, 107, 107, 96, 106, 106, 107, 104, 104, 106]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:21.960574482Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1614,"":"downstairs","":"downstairs","session_id":"2d5c91bb-897e-436f-937c-245ae9798b74"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:21.960591332Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1614,"":"downstairs","":"downstairs","session_id":"2d5c91bb-897e-436f-937c-245ae9798b74"} {"msg":"[2]R flush_numbers[0..12]: [181, 193, 196, 196, 196, 177, 195, 195, 196, 193, 193, 195]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:21.96060495Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1614,"":"downstairs","":"downstairs","session_id":"2d5c91bb-897e-436f-937c-245ae9798b74"} {"msg":"[2]R generation[0..12]: [98, 104, 107, 107, 107, 96, 106, 106, 107, 104, 104, 106]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:21.960618388Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1614,"":"downstairs","":"downstairs","session_id":"2d5c91bb-897e-436f-937c-245ae9798b74"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:21.960631946Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1614,"":"downstairs","":"downstairs","session_id":"2d5c91bb-897e-436f-937c-245ae9798b74"} {"msg":"Max found gen is 108","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:21.960644533Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1614,"":"downstairs","":"downstairs","session_id":"2d5c91bb-897e-436f-937c-245ae9798b74"} {"msg":"Generation requested: 108 >= found:108","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:21.960657561Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1614,"":"downstairs","":"downstairs","session_id":"2d5c91bb-897e-436f-937c-245ae9798b74"} Wait for a query_work_queue command to finish before sending IO {"msg":"Next flush: 197","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:21.960675751Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1614,"":"downstairs","":"downstairs","session_id":"2d5c91bb-897e-436f-937c-245ae9798b74"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:21.96069036Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1614,"":"downstairs","":"downstairs","session_id":"2d5c91bb-897e-436f-937c-245ae9798b74"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:21.960712973Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1614,"session_id":"2d5c91bb-897e-436f-937c-245ae9798b74"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:21.960735896Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1614,"client":"0","":"downstairs","session_id":"2d5c91bb-897e-436f-937c-245ae9798b74"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:21.960761531Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1614,"client":"1","":"downstairs","session_id":"2d5c91bb-897e-436f-937c-245ae9798b74"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:21.96077714Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1614,"client":"2","":"downstairs","session_id":"2d5c91bb-897e-436f-937c-245ae9798b74"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:21.960790518Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1614,"session_id":"2d5c91bb-897e-436f-937c-245ae9798b74"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:21.960811551Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1614,"session_id":"2d5c91bb-897e-436f-937c-245ae9798b74"} {"msg":"9fd6cb40-57bb-42da-b118-a96092a45f8f is now active with session: 2d5c91bb-897e-436f-937c-245ae9798b74","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:21.960834124Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1614,"session_id":"2d5c91bb-897e-436f-937c-245ae9798b74"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:21.960860289Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1614,"session_id":"2d5c91bb-897e-436f-937c-245ae9798b74"} {"msg":"The guest has finished waiting for activation with:108","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:21.960878099Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1614} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 390 len 40960 data: 8 7 7 6 7 6 8 9 9 8 02/30 Read block 447 len 40960 03/30 Read block 100 len 16384 04/30 Write block 191 len 16384 data: 8 10 12 13 05/30 Read block 163 len 32768 06/30 Read block 406 len 12288 07/30 Read block 382 len 4096 08/30 Read block 452 len 4096 09/30 Read block 232 len 8192 10/30 Read block 522 len 16384 11/30 Write block 171 len 24576 data: 10 9 9 8 11 10 12/30 Read block 532 len 40960 13/30 Write block 231 len 4096 data: 8 14/30 Read block 155 len 24576 15/30 Read block 207 len 24576 16/30 Read block 582 len 12288 17/30 Read block 42 len 32768 18/30 Write block 22 len 24576 data: 7 6 6 7 9 10 19/30 Read block 103 len 16384 20/30 Write block 247 len 4096 data: 9 21/30 Write block 490 len 32768 data: 9 9 7 8 10 12 12 12 22/30 Read block 404 len 40960 23/30 Write block 134 len 8192 data: 6 5 24/30 Write block 408 len 20480 data: 4 5 5 4 6 25/30 Read block 464 len 12288 26/30 Read block 103 len 32768 27/30 Read block 51 len 32768 28/30 Read block 178 len 12288 29/30 Read block 5 len 36864 30/30 Read block 482 len 4096 ---------------------------------------------------------------- Crucible gen:108 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 10 Done Done Done false 1001 Acked Read 10 Done Done Done false 1002 Acked Read 4 Done Done Done false 1003 Acked Write 4 Done Done Done false 1004 Acked Read 8 Done Done Done false 1005 Acked Read 3 Done Done Done false 1006 Acked Read 1 Done Done Done false 1007 Acked Read 1 Done Done Done false 1008 Acked Read 2 Done Done Done false 1009 Acked Read 4 Done Done Done false 1010 Acked Write 6 Done Done Done false 1011 Acked Read 10 Done Done Done false 1012 Acked Write 1 Done Done Done false 1013 Acked Read 6 Done Done Done false 1014 Acked Read 6 Done Done Done false 1015 Acked Read 3 Done Done Done false 1016 Acked Read 8 Done Done Done false 1017 Acked Write 6 Done Done Done false 1018 Acked Read 4 Done Done Done false 1019 Acked Write 1 Done Done Done false 1020 Acked Write 8 Done Done Done false 1021 Acked Read 10 Done Done Done false 1022 Acked Write 2 Done Done Done false 1023 Acked Write 5 Done Done Done false 1024 Acked Read 3 Done Done Done false 1025 Acked Read 8 Done Done Done false 1026 Acked Read 8 Done Done Done false 1027 Acked Read 3 Done Done Done false 1028 Acked Read 9 Done Done Done false 1029 Acked Read 1 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 30 30 30 90 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 0 0 0 Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:15:21.980732634Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1614} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 98 98 98 181 181 181 F F F 1 020-039 104 104 104 193 193 193 T T T 2 040-059 107 107 107 196 196 196 F F F 3 060-079 107 107 107 196 196 196 F F F 4 080-099 107 107 107 196 196 196 F F F 5 100-119 96 96 96 177 177 177 F F F 6 120-139 106 106 106 195 195 195 T T T 7 140-159 106 106 106 195 195 195 F F F 8 160-179 107 107 107 196 196 196 T T T 9 180-199 104 104 104 193 193 193 T T T 10 200-219 104 104 104 193 193 193 F F F 11 220-239 106 106 106 195 195 195 T T T 12 240-259 106 106 106 195 195 195 T T T 13 260-279 106 106 106 195 195 195 F F F 14 280-299 104 104 104 193 193 193 F F F 15 300-319 104 104 104 192 192 192 F F F 16 320-339 106 106 106 195 195 195 F F F 17 340-359 106 106 106 195 195 195 F F F 18 360-379 104 104 104 193 193 193 F F F 19 380-399 104 104 104 193 193 193 T T T 20 400-419 102 102 102 190 190 190 T T T 21 420-439 105 105 105 194 194 194 F F F 22 440-459 106 106 106 195 195 195 F F F 23 460-479 106 106 106 195 195 195 F F F 24 480-499 106 106 106 195 195 195 T T T 25 500-519 107 107 107 196 196 196 F F F 26 520-539 107 107 107 196 196 196 F F F 27 540-559 106 106 106 195 195 195 F F F 28 560-579 106 106 106 195 195 195 F F F 29 580-599 104 104 104 193 193 193 F F F Max gen: 107, Max flush: 196 A Difference in extent metadata is expected here On loop 54 Verifying data now No change after verify Loop: 54 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 98 98 98 181 181 181 F F F 1 020-039 108 108 108 197 197 197 F F F 2 040-059 107 107 107 196 196 196 F F F 3 060-079 107 107 107 196 196 196 F F F 4 080-099 107 107 107 196 196 196 F F F 5 100-119 96 96 96 177 177 177 F F F 6 120-139 108 108 108 197 197 197 F F F 7 140-159 106 106 106 195 195 195 F F F 8 160-179 108 108 108 197 197 197 F F F 9 180-199 108 108 108 197 197 197 F F F 10 200-219 104 104 104 193 193 193 F F F 11 220-239 108 108 108 197 197 197 F F F 12 240-259 108 108 108 197 197 197 F F F 13 260-279 106 106 106 195 195 195 F F F 14 280-299 104 104 104 193 193 193 F F F 15 300-319 104 104 104 192 192 192 F F F 16 320-339 106 106 106 195 195 195 F F F 17 340-359 106 106 106 195 195 195 F F F 18 360-379 104 104 104 193 193 193 F F F 19 380-399 108 108 108 197 197 197 F F F 20 400-419 108 108 108 197 197 197 F F F 21 420-439 105 105 105 194 194 194 F F F 22 440-459 106 106 106 195 195 195 F F F 23 460-479 106 106 106 195 195 195 F F F 24 480-499 108 108 108 197 197 197 F F F 25 500-519 107 107 107 196 196 196 F F F 26 520-539 107 107 107 196 196 196 F F F 27 540-559 106 106 106 195 195 195 F F F 28 560-579 106 106 106 195 195 195 F F F 29 580-599 104 104 104 193 193 193 F F F Max gen: 108, Max flush: 197 Begin loop 55 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:26.241320775Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1622,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:26.286723322Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1622,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:26.287394744Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1622} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"abd5430f1508af0bc008143ea2566277ef714ec2\",\n git_commit_timestamp: \"2025-01-17T20:55:03.000000000Z\",\n git_branch: \"alan/tokio-to-140\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:26.287445854Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1622} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:26.287467166Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1622} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:26.28749Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1622} {"msg":"Crucible 8b761f00-11df-45e5-b40b-74ec68e232e2 has session id: af6be867-1c6a-4705-a5fb-5506a527518c","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:26.288087929Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1622,"session_id":"af6be867-1c6a-4705-a5fb-5506a527518c"} {"msg":"Upstairs opts: Upstairs UUID: 8b761f00-11df-45e5-b40b-74ec68e232e2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:26.288117016Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1622,"session_id":"af6be867-1c6a-4705-a5fb-5506a527518c"} {"msg":"Crucible stats registered with UUID: 8b761f00-11df-45e5-b40b-74ec68e232e2","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:26.288131715Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1622,"session_id":"af6be867-1c6a-4705-a5fb-5506a527518c"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:15:26.288144632Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1622,"session_id":"af6be867-1c6a-4705-a5fb-5506a527518c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:26.288622292Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1622,"task":"crutest"} {"msg":"The guest has requested activation with gen:110","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:26.288643724Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1622} {"msg":"8b761f00-11df-45e5-b40b-74ec68e232e2 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:26.288715696Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1622,"session_id":"af6be867-1c6a-4705-a5fb-5506a527518c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:26.288731966Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1622,"client":"0","":"downstairs","session_id":"af6be867-1c6a-4705-a5fb-5506a527518c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:26.28875579Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1622,"client":"0","":"downstairs","session_id":"af6be867-1c6a-4705-a5fb-5506a527518c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:26.28877375Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1622,"client":"1","":"downstairs","session_id":"af6be867-1c6a-4705-a5fb-5506a527518c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:26.288788089Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1622,"client":"1","":"downstairs","session_id":"af6be867-1c6a-4705-a5fb-5506a527518c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:26.288801196Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1622,"client":"2","":"downstairs","session_id":"af6be867-1c6a-4705-a5fb-5506a527518c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:26.288814964Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1622,"client":"2","":"downstairs","session_id":"af6be867-1c6a-4705-a5fb-5506a527518c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:26.288827712Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1622,"":"io task","client":"2","":"downstairs","session_id":"af6be867-1c6a-4705-a5fb-5506a527518c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:26.288840959Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1622,"":"io task","client":"0","":"downstairs","session_id":"af6be867-1c6a-4705-a5fb-5506a527518c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:26.288853957Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1622,"":"io task","client":"1","":"downstairs","session_id":"af6be867-1c6a-4705-a5fb-5506a527518c"} {"msg":"ds_connection connected from Ok(127.0.0.1:39414)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:26.289000002Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1622,"":"io task","client":"0","":"downstairs","session_id":"af6be867-1c6a-4705-a5fb-5506a527518c"} {"msg":"ds_connection connected from Ok(127.0.0.1:60218)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:26.289113219Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1622,"":"io task","client":"1","":"downstairs","session_id":"af6be867-1c6a-4705-a5fb-5506a527518c"} {"msg":"ds_connection connected from Ok(127.0.0.1:47449)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:26.289146308Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1622,"":"io task","client":"2","":"downstairs","session_id":"af6be867-1c6a-4705-a5fb-5506a527518c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:26.289665911Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1622,"client":"2","":"downstairs","session_id":"af6be867-1c6a-4705-a5fb-5506a527518c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:26.289706455Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1622,"client":"0","":"downstairs","session_id":"af6be867-1c6a-4705-a5fb-5506a527518c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:26.289836551Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1622,"client":"1","":"downstairs","session_id":"af6be867-1c6a-4705-a5fb-5506a527518c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:26.290397399Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1622,"client":"0","":"downstairs","session_id":"af6be867-1c6a-4705-a5fb-5506a527518c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:26.290428997Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1622,"client":"2","":"downstairs","session_id":"af6be867-1c6a-4705-a5fb-5506a527518c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:26.290540823Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1622,"client":"1","":"downstairs","session_id":"af6be867-1c6a-4705-a5fb-5506a527518c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:26.290989675Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1622,"session_id":"af6be867-1c6a-4705-a5fb-5506a527518c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:26.291024766Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1622,"session_id":"af6be867-1c6a-4705-a5fb-5506a527518c"} {"msg":"[0]R flush_numbers[0..12]: [181, 197, 196, 196, 196, 177, 197, 195, 197, 197, 193, 197]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:26.29111687Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1622,"":"downstairs","":"downstairs","session_id":"af6be867-1c6a-4705-a5fb-5506a527518c"} {"msg":"[0]R generation[0..12]: [98, 108, 107, 107, 107, 96, 108, 106, 108, 108, 104, 108]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:26.291145426Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1622,"":"downstairs","":"downstairs","session_id":"af6be867-1c6a-4705-a5fb-5506a527518c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:26.291161426Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1622,"":"downstairs","":"downstairs","session_id":"af6be867-1c6a-4705-a5fb-5506a527518c"} {"msg":"[1]R flush_numbers[0..12]: [181, 197, 196, 196, 196, 177, 197, 195, 197, 197, 193, 197]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:26.291175584Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1622,"":"downstairs","":"downstairs","session_id":"af6be867-1c6a-4705-a5fb-5506a527518c"} {"msg":"[1]R generation[0..12]: [98, 108, 107, 107, 107, 96, 108, 106, 108, 108, 104, 108]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:26.291189242Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1622,"":"downstairs","":"downstairs","session_id":"af6be867-1c6a-4705-a5fb-5506a527518c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:26.29120319Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1622,"":"downstairs","":"downstairs","session_id":"af6be867-1c6a-4705-a5fb-5506a527518c"} {"msg":"[2]R flush_numbers[0..12]: [181, 197, 196, 196, 196, 177, 197, 195, 197, 197, 193, 197]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:26.291216758Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1622,"":"downstairs","":"downstairs","session_id":"af6be867-1c6a-4705-a5fb-5506a527518c"} {"msg":"[2]R generation[0..12]: [98, 108, 107, 107, 107, 96, 108, 106, 108, 108, 104, 108]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:26.291230046Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1622,"":"downstairs","":"downstairs","session_id":"af6be867-1c6a-4705-a5fb-5506a527518c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:26.291243404Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1622,"":"downstairs","":"downstairs","session_id":"af6be867-1c6a-4705-a5fb-5506a527518c"} {"msg":"Max found gen is 109","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:26.291256011Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1622,"":"downstairs","":"downstairs","session_id":"af6be867-1c6a-4705-a5fb-5506a527518c"} {"msg":"Generation requested: 110 >= found:109","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-17T21:15:26.291273101Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1622,"":"downstairs","":"downstairs","session_id":"af6be867-1c6a-4705-a5fb-5506a527518c"} {"msg":"Next flush: 198","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:26.291290661Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1622,"":"downstairs","":"downstairs","session_id":"af6be867-1c6a-4705-a5fb-5506a527518c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:26.291303469Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1622,"":"downstairs","":"downstairs","session_id":"af6be867-1c6a-4705-a5fb-5506a527518c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:26.291316296Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1622,"session_id":"af6be867-1c6a-4705-a5fb-5506a527518c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:26.291327983Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1622,"client":"0","":"downstairs","session_id":"af6be867-1c6a-4705-a5fb-5506a527518c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:26.291340801Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1622,"client":"1","":"downstairs","session_id":"af6be867-1c6a-4705-a5fb-5506a527518c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:26.291353568Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1622,"client":"2","":"downstairs","session_id":"af6be867-1c6a-4705-a5fb-5506a527518c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:26.291366305Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1622,"session_id":"af6be867-1c6a-4705-a5fb-5506a527518c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:26.291378352Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1622,"session_id":"af6be867-1c6a-4705-a5fb-5506a527518c"} {"msg":"8b761f00-11df-45e5-b40b-74ec68e232e2 is now active with session: af6be867-1c6a-4705-a5fb-5506a527518c","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:26.29139081Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1622,"session_id":"af6be867-1c6a-4705-a5fb-5506a527518c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:26.291402586Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1622,"session_id":"af6be867-1c6a-4705-a5fb-5506a527518c"} {"msg":"The guest has finished waiting for activation with:110","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:26.291414353Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1622} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 213 len 8192 data: 11 10 02/30 Read block 493 len 28672 03/30 Read block 475 len 16384 04/30 Write block 161 len 20480 data: 10 9 7 8 10 05/30 Read block 487 len 32768 06/30 Read block 191 len 20480 07/30 Flush 08/30 Write block 69 len 32768 data: 7 7 11 9 8 8 11 11 09/30 Read block 49 len 16384 10/30 Write block 30 len 28672 data: 11 9 12 12 11 13 8 11/30 Write block 412 len 32768 data: 7 8 8 8 7 7 6 7 12/30 Read block 488 len 8192 13/30 Read block 28 len 16384 14/30 Read block 45 len 28672 15/30 Write block 554 len 28672 data: 4 5 6 8 10 9 13 16/30 Read block 390 len 8192 17/30 Read block 583 len 8192 18/30 Write block 480 len 36864 data: 8 7 6 5 6 5 6 6 6 19/30 Flush 20/30 Flush 21/30 Write block 259 len 4096 data: 10 22/30 Read block 68 len 8192 23/30 Read block 376 len 36864 24/30 Write block 524 len 4096 data: 11 25/30 Read block 510 len 24576 26/30 Write block 345 len 4096 data: 8 27/30 Read block 408 len 24576 28/30 Read block 372 len 4096 29/30 Write block 127 len 12288 data: 6 6 6 30/30 Read block 65 len 24576 ---------------------------------------------------------------- Crucible gen:110 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1020 Acked Write 1 Done Done Done false 1021 Acked Read 2 Done Done Done false 1022 Acked Read 9 Done Done Done false 1023 Acked Write 1 Done Done Done false 1024 Acked Read 6 Done Done Done false 1025 Acked Write 1 Done Done Done false 1026 Acked Read 6 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Write 3 Done Done Sent false 1029 Acked Read 6 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 2 3 Done 10 9 8 27 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1019 1019 1019 Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:15:26.3108452Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1622} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 98 98 98 181 181 181 F F F 1 020-039 110 110 110 199 199 199 F F F 2 040-059 107 107 107 196 196 196 F F F 3 060-079 110 110 110 199 199 199 F F F 4 080-099 107 107 107 196 196 196 F F F 5 100-119 96 96 96 177 177 177 F F F 6 120-139 108 108 108 197 197 197 T T T 7 140-159 106 106 106 195 195 195 F F F 8 160-179 110 110 110 198 198 198 F F F 9 180-199 108 108 108 197 197 197 F F F 10 200-219 110 110 110 198 198 198 F F F 11 220-239 108 108 108 197 197 197 F F F 12 240-259 108 108 108 197 197 197 T T T 13 260-279 106 106 106 195 195 195 F F F 14 280-299 104 104 104 193 193 193 F F F 15 300-319 104 104 104 192 192 192 F F F 16 320-339 106 106 106 195 195 195 F F F 17 340-359 106 106 106 195 195 195 T T T 18 360-379 104 104 104 193 193 193 F F F 19 380-399 108 108 108 197 197 197 F F F 20 400-419 110 110 110 199 199 199 F F F 21 420-439 105 105 105 194 194 194 F F F 22 440-459 106 106 106 195 195 195 F F F 23 460-479 106 106 106 195 195 195 F F F 24 480-499 110 110 110 199 199 199 F F F 25 500-519 107 107 107 196 196 196 F F F 26 520-539 107 107 107 196 196 196 T T T 27 540-559 110 110 110 199 199 199 F F F 28 560-579 110 110 110 199 199 199 F F F 29 580-599 104 104 104 193 193 193 F F F Max gen: 110, Max flush: 199 A Difference in extent metadata is expected here On loop 55 Verifying data now No change after verify Loop: 55 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 98 98 98 181 181 181 F F F 1 020-039 110 110 110 199 199 199 F F F 2 040-059 107 107 107 196 196 196 F F F 3 060-079 110 110 110 199 199 199 F F F 4 080-099 107 107 107 196 196 196 F F F 5 100-119 96 96 96 177 177 177 F F F 6 120-139 111 111 111 200 200 200 F F F 7 140-159 106 106 106 195 195 195 F F F 8 160-179 110 110 110 198 198 198 F F F 9 180-199 108 108 108 197 197 197 F F F 10 200-219 110 110 110 198 198 198 F F F 11 220-239 108 108 108 197 197 197 F F F 12 240-259 111 111 111 200 200 200 F F F 13 260-279 106 106 106 195 195 195 F F F 14 280-299 104 104 104 193 193 193 F F F 15 300-319 104 104 104 192 192 192 F F F 16 320-339 106 106 106 195 195 195 F F F 17 340-359 111 111 111 200 200 200 F F F 18 360-379 104 104 104 193 193 193 F F F 19 380-399 108 108 108 197 197 197 F F F 20 400-419 110 110 110 199 199 199 F F F 21 420-439 105 105 105 194 194 194 F F F 22 440-459 106 106 106 195 195 195 F F F 23 460-479 106 106 106 195 195 195 F F F 24 480-499 110 110 110 199 199 199 F F F 25 500-519 107 107 107 196 196 196 F F F 26 520-539 111 111 111 200 200 200 F F F 27 540-559 110 110 110 199 199 199 F F F 28 560-579 110 110 110 199 199 199 F F F 29 580-599 104 104 104 193 193 193 F F F Max gen: 111, Max flush: 200 Begin loop 56 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:30.545367278Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1630,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:30.589535583Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1630,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:30.590204574Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1630} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"abd5430f1508af0bc008143ea2566277ef714ec2\",\n git_commit_timestamp: \"2025-01-17T20:55:03.000000000Z\",\n git_branch: \"alan/tokio-to-140\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:30.590244477Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1630} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:30.590258515Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1630} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:30.590272653Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1630} {"msg":"Crucible 527fd9ea-9803-401e-b433-fd08f0a7e2aa has session id: 8ec25eac-b2d7-47f8-bc21-4c8c5a2fb3e9","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:30.590842657Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1630,"session_id":"8ec25eac-b2d7-47f8-bc21-4c8c5a2fb3e9"} {"msg":"Upstairs opts: Upstairs UUID: 527fd9ea-9803-401e-b433-fd08f0a7e2aa, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:30.590932299Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1630,"session_id":"8ec25eac-b2d7-47f8-bc21-4c8c5a2fb3e9"} {"msg":"Crucible stats registered with UUID: 527fd9ea-9803-401e-b433-fd08f0a7e2aa","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:30.590962807Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1630,"session_id":"8ec25eac-b2d7-47f8-bc21-4c8c5a2fb3e9"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:15:30.590989592Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1630,"session_id":"8ec25eac-b2d7-47f8-bc21-4c8c5a2fb3e9"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:30.591509126Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1630,"task":"crutest"} {"msg":"The guest has requested activation with gen:112","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:30.591547649Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1630} {"msg":"527fd9ea-9803-401e-b433-fd08f0a7e2aa active request set","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:30.591605853Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1630,"session_id":"8ec25eac-b2d7-47f8-bc21-4c8c5a2fb3e9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:30.591622733Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1630,"client":"0","":"downstairs","session_id":"8ec25eac-b2d7-47f8-bc21-4c8c5a2fb3e9"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:30.591644515Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1630,"client":"0","":"downstairs","session_id":"8ec25eac-b2d7-47f8-bc21-4c8c5a2fb3e9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:30.591663726Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1630,"client":"1","":"downstairs","session_id":"8ec25eac-b2d7-47f8-bc21-4c8c5a2fb3e9"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:30.591689702Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1630,"client":"1","":"downstairs","session_id":"8ec25eac-b2d7-47f8-bc21-4c8c5a2fb3e9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:30.591713736Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1630,"client":"2","":"downstairs","session_id":"8ec25eac-b2d7-47f8-bc21-4c8c5a2fb3e9"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:30.59173764Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1630,"client":"2","":"downstairs","session_id":"8ec25eac-b2d7-47f8-bc21-4c8c5a2fb3e9"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:30.591762214Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1630,"":"io task","client":"2","":"downstairs","session_id":"8ec25eac-b2d7-47f8-bc21-4c8c5a2fb3e9"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:30.591787449Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1630,"":"io task","client":"0","":"downstairs","session_id":"8ec25eac-b2d7-47f8-bc21-4c8c5a2fb3e9"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:30.591810312Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1630,"":"io task","client":"1","":"downstairs","session_id":"8ec25eac-b2d7-47f8-bc21-4c8c5a2fb3e9"} {"msg":"ds_connection connected from Ok(127.0.0.1:62437)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:30.591851096Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1630,"":"io task","client":"1","":"downstairs","session_id":"8ec25eac-b2d7-47f8-bc21-4c8c5a2fb3e9"} {"msg":"ds_connection connected from Ok(127.0.0.1:60499)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:30.591882054Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1630,"":"io task","client":"2","":"downstairs","session_id":"8ec25eac-b2d7-47f8-bc21-4c8c5a2fb3e9"} {"msg":"ds_connection connected from Ok(127.0.0.1:54304)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:30.591919446Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1630,"":"io task","client":"0","":"downstairs","session_id":"8ec25eac-b2d7-47f8-bc21-4c8c5a2fb3e9"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:30.592674447Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1630,"client":"0","":"downstairs","session_id":"8ec25eac-b2d7-47f8-bc21-4c8c5a2fb3e9"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:30.592712239Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1630,"client":"2","":"downstairs","session_id":"8ec25eac-b2d7-47f8-bc21-4c8c5a2fb3e9"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:30.593027013Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1630,"client":"1","":"downstairs","session_id":"8ec25eac-b2d7-47f8-bc21-4c8c5a2fb3e9"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:30.593559474Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1630,"client":"2","":"downstairs","session_id":"8ec25eac-b2d7-47f8-bc21-4c8c5a2fb3e9"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:30.593606392Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1630,"client":"0","":"downstairs","session_id":"8ec25eac-b2d7-47f8-bc21-4c8c5a2fb3e9"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:30.593839638Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1630,"client":"1","":"downstairs","session_id":"8ec25eac-b2d7-47f8-bc21-4c8c5a2fb3e9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:30.594190563Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1630,"session_id":"8ec25eac-b2d7-47f8-bc21-4c8c5a2fb3e9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:30.594279815Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1630,"session_id":"8ec25eac-b2d7-47f8-bc21-4c8c5a2fb3e9"} {"msg":"[0]R flush_numbers[0..12]: [181, 199, 196, 199, 196, 177, 200, 195, 198, 197, 198, 197]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:30.594394173Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1630,"":"downstairs","":"downstairs","session_id":"8ec25eac-b2d7-47f8-bc21-4c8c5a2fb3e9"} {"msg":"[0]R generation[0..12]: [98, 110, 107, 110, 107, 96, 111, 106, 110, 108, 110, 108]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:30.594413144Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1630,"":"downstairs","":"downstairs","session_id":"8ec25eac-b2d7-47f8-bc21-4c8c5a2fb3e9"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:30.594427842Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1630,"":"downstairs","":"downstairs","session_id":"8ec25eac-b2d7-47f8-bc21-4c8c5a2fb3e9"} {"msg":"[1]R flush_numbers[0..12]: [181, 199, 196, 199, 196, 177, 200, 195, 198, 197, 198, 197]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:30.59444161Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1630,"":"downstairs","":"downstairs","session_id":"8ec25eac-b2d7-47f8-bc21-4c8c5a2fb3e9"} {"msg":"[1]R generation[0..12]: [98, 110, 107, 110, 107, 96, 111, 106, 110, 108, 110, 108]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:30.594455188Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1630,"":"downstairs","":"downstairs","session_id":"8ec25eac-b2d7-47f8-bc21-4c8c5a2fb3e9"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:30.594472038Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1630,"":"downstairs","":"downstairs","session_id":"8ec25eac-b2d7-47f8-bc21-4c8c5a2fb3e9"} {"msg":"[2]R flush_numbers[0..12]: [181, 199, 196, 199, 196, 177, 200, 195, 198, 197, 198, 197]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:30.594485586Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1630,"":"downstairs","":"downstairs","session_id":"8ec25eac-b2d7-47f8-bc21-4c8c5a2fb3e9"} {"msg":"[2]R generation[0..12]: [98, 110, 107, 110, 107, 96, 111, 106, 110, 108, 110, 108]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:30.594498944Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1630,"":"downstairs","":"downstairs","session_id":"8ec25eac-b2d7-47f8-bc21-4c8c5a2fb3e9"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:30.594512582Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1630,"":"downstairs","":"downstairs","session_id":"8ec25eac-b2d7-47f8-bc21-4c8c5a2fb3e9"} {"msg":"Max found gen is 112","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:30.594525039Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1630,"":"downstairs","":"downstairs","session_id":"8ec25eac-b2d7-47f8-bc21-4c8c5a2fb3e9"} {"msg":"Generation requested: 112 >= found:112","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:30.594537716Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1630,"":"downstairs","":"downstairs","session_id":"8ec25eac-b2d7-47f8-bc21-4c8c5a2fb3e9"} {"msg":"Next flush: 201","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:30.594550274Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1630,"":"downstairs","":"downstairs","session_id":"8ec25eac-b2d7-47f8-bc21-4c8c5a2fb3e9"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:30.594562761Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1630,"":"downstairs","":"downstairs","session_id":"8ec25eac-b2d7-47f8-bc21-4c8c5a2fb3e9"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:30.594575258Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1630,"session_id":"Wait for a query_work_queue command to finish before sending IO 8ec25eac-b2d7-47f8-bc21-4c8c5a2fb3e9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:30.59459628Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1630,"client":"0","":"downstairs","session_id":"8ec25eac-b2d7-47f8-bc21-4c8c5a2fb3e9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:30.594609178Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1630,"client":"1","":"downstairs","session_id":"8ec25eac-b2d7-47f8-bc21-4c8c5a2fb3e9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:30.594621995Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1630,"client":"2","":"downstairs","session_id":"8ec25eac-b2d7-47f8-bc21-4c8c5a2fb3e9"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:30.594635103Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1630,"session_id":"8ec25eac-b2d7-47f8-bc21-4c8c5a2fb3e9"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:30.59464698Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1630,"session_id":"8ec25eac-b2d7-47f8-bc21-4c8c5a2fb3e9"} {"msg":"527fd9ea-9803-401e-b433-fd08f0a7e2aa is now active with session: 8ec25eac-b2d7-47f8-bc21-4c8c5a2fb3e9","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:30.594659307Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1630,"session_id":"8ec25eac-b2d7-47f8-bc21-4c8c5a2fb3e9"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:30.594672555Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1630,"session_id":"8ec25eac-b2d7-47f8-bc21-4c8c5a2fb3e9"} {"msg":"The guest has finished waiting for activation with:112","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:30.594684482Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1630} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 449 len 28672 data: 7 6 5 6 5 7 7 02/30 Write block 35 len 40960 data: 14 9 8 8 7 6 7 6 5 6 03/30 Write block 32 len 36864 data: 13 13 12 15 10 9 9 8 7 04/30 Write block 254 len 16384 data: 7 9 10 11 05/30 Flush 06/30 Write block 177 len 8192 data: 11 8 07/30 Read block 577 len 24576 08/30 Flush 09/30 Write block 401 len 4096 data: 9 10/30 Read block 544 len 32768 11/30 Read block 497 len 40960 12/30 Read block 148 len 12288 13/30 Write block 547 len 24576 data: 10 11 10 10 7 5 14/30 Write block 34 len 8192 data: 13 16 15/30 Read block 200 len 32768 16/30 Write block 497 len 20480 data: 13 15 13 15 15 17/30 Read block 536 len 24576 18/30 Read block 417 len 40960 19/30 Write block 288 len 32768 data: 9 9 10 12 13 12 10 12 20/30 Write block 76 len 24576 data: 12 9 9 8 7 8 21/30 Write block 47 len 32768 data: 8 8 8 8 10 14 14 14 22/30 Read block 139 len 24576 23/30 Flush 24/30 Write block 259 len 4096 data: 11 25/30 Write block 413 len 16384 data: 9 9 9 8 26/30 Flush 27/30 Write block 254 len 4096 data: 8 28/30 Write block 40 len 12288 data: 8 8 7 29/30 Write block 308 len 4096 data: 7 30/30 Write block 505 len 24576 data: 13 11 12 11 13 11 ---------------------------------------------------------------- Crucible gen:112 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 1 Done Done Done false 1027 Acked Write 3 Done Done Done false 1028 Acked Write 1 Done Done Done false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:15:30.61218438Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1630} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 98 98 98 181 181 181 F F F 1 020-039 112 112 112 203 203 203 F F F 2 040-059 112 112 112 203 203 203 T T T 3 060-079 112 112 112 203 203 203 F F F 4 080-099 112 112 112 203 203 203 F F F 5 100-119 96 96 96 177 177 177 F F F 6 120-139 111 111 111 200 200 200 F F F 7 140-159 106 106 106 195 195 195 F F F 8 160-179 112 112 112 202 202 202 F F F 9 180-199 108 108 108 197 197 197 F F F 10 200-219 110 110 110 198 198 198 F F F 11 220-239 108 108 108 197 197 197 F F F 12 240-259 112 112 112 204 204 204 T T T 13 260-279 106 106 106 195 195 195 F F F 14 280-299 112 112 112 203 203 203 F F F 15 300-319 104 104 104 192 192 192 T T T 16 320-339 106 106 106 195 195 195 F F F 17 340-359 111 111 111 200 200 200 F F F 18 360-379 104 104 104 193 193 193 F F F 19 380-399 108 108 108 197 197 197 F F F 20 400-419 112 112 112 204 204 204 F F F 21 420-439 105 105 105 194 194 194 F F F 22 440-459 112 112 112 201 201 201 F F F 23 460-479 106 106 106 195 195 195 F F F 24 480-499 112 112 112 203 203 203 F F F 25 500-519 112 112 112 203 203 203 T T T 26 520-539 111 111 111 200 200 200 F F F 27 540-559 112 112 112 203 203 203 F F F 28 560-579 110 110 110 199 199 199 F F F 29 580-599 104 104 104 193 193 193 F F F Max gen: 112, Max flush: 204 A Difference in extent metadata is expected here On loop 56 Verifying data now No change after verify Loop: 56 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 98 98 98 181 181 181 F F F 1 020-039 112 112 112 203 203 203 F F F 2 040-059 113 113 113 205 205 205 F F F 3 060-079 112 112 112 203 203 203 F F F 4 080-099 112 112 112 203 203 203 F F F 5 100-119 96 96 96 177 177 177 F F F 6 120-139 111 111 111 200 200 200 F F F 7 140-159 106 106 106 195 195 195 F F F 8 160-179 112 112 112 202 202 202 F F F 9 180-199 108 108 108 197 197 197 F F F 10 200-219 110 110 110 198 198 198 F F F 11 220-239 108 108 108 197 197 197 F F F 12 240-259 113 113 113 205 205 205 F F F 13 260-279 106 106 106 195 195 195 F F F 14 280-299 112 112 112 203 203 203 F F F 15 300-319 113 113 113 205 205 205 F F F 16 320-339 106 106 106 195 195 195 F F F 17 340-359 111 111 111 200 200 200 F F F 18 360-379 104 104 104 193 193 193 F F F 19 380-399 108 108 108 197 197 197 F F F 20 400-419 112 112 112 204 204 204 F F F 21 420-439 105 105 105 194 194 194 F F F 22 440-459 112 112 112 201 201 201 F F F 23 460-479 106 106 106 195 195 195 F F F 24 480-499 112 112 112 203 203 203 F F F 25 500-519 113 113 113 205 205 205 F F F 26 520-539 111 111 111 200 200 200 F F F 27 540-559 112 112 112 203 203 203 F F F 28 560-579 110 110 110 199 199 199 F F F 29 580-599 104 104 104 193 193 193 F F F Max gen: 113, Max flush: 205 Begin loop 57 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:34.851959937Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1638,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:34.896427196Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1638,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:34.897228465Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1638} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"abd5430f1508af0bc008143ea2566277ef714ec2\",\n git_commit_timestamp: \"2025-01-17T20:55:03.000000000Z\",\n git_branch: \"alan/tokio-to-140\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:34.897273421Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1638} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:34.897287739Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1638} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:34.897305229Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1638} {"msg":"Crucible e2bb6b2b-cfcd-4e20-9471-c27590e3ef5b has session id: b283a3af-cf91-42cc-bc6b-d24dc5f7b6c4","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:34.8978364Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1638,"session_id":"b283a3af-cf91-42cc-bc6b-d24dc5f7b6c4"} {"msg":"Upstairs opts: Upstairs UUID: e2bb6b2b-cfcd-4e20-9471-c27590e3ef5b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:34.897876193Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1638,"session_id":"b283a3af-cf91-42cc-bc6b-d24dc5f7b6c4"} {"msg":"Crucible stats registered with UUID: e2bb6b2b-cfcd-4e20-9471-c27590e3ef5b","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:34.897902329Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1638,"session_id":"b283a3af-cf91-42cc-bc6b-d24dc5f7b6c4"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:15:34.897923361Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1638,"session_id":"b283a3af-cf91-42cc-bc6b-d24dc5f7b6c4"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:34.898320013Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1638,"task":"crutest"} {"msg":"The guest has requested activation with gen:114","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:34.898344707Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1638} {"msg":"e2bb6b2b-cfcd-4e20-9471-c27590e3ef5b active request set","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:34.898415849Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1638,"session_id":"b283a3af-cf91-42cc-bc6b-d24dc5f7b6c4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:34.898433759Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1638,"client":"0","":"downstairs","session_id":"b283a3af-cf91-42cc-bc6b-d24dc5f7b6c4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:34.898456662Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1638,"client":"0","":"downstairs","session_id":"b283a3af-cf91-42cc-bc6b-d24dc5f7b6c4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:34.898470241Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1638,"client":"1","":"downstairs","session_id":"b283a3af-cf91-42cc-bc6b-d24dc5f7b6c4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:34.898484109Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1638,"client":"1","":"downstairs","session_id":"b283a3af-cf91-42cc-bc6b-d24dc5f7b6c4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:34.898496876Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1638,"client":"2","":"downstairs","session_id":"b283a3af-cf91-42cc-bc6b-d24dc5f7b6c4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:34.898514286Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1638,"client":"2","":"downstairs","session_id":"b283a3af-cf91-42cc-bc6b-d24dc5f7b6c4"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:34.898536519Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1638,"":"io task","client":"2","":"downstairs","session_id":"b283a3af-cf91-42cc-bc6b-d24dc5f7b6c4"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:34.898563255Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1638,"":"io task","client":"0","":"downstairs","session_id":"b283a3af-cf91-42cc-bc6b-d24dc5f7b6c4"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:34.898585968Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1638,"":"io task","client":"1","":"downstairs","session_id":"b283a3af-cf91-42cc-bc6b-d24dc5f7b6c4"} {"msg":"ds_connection connected from Ok(127.0.0.1:37966)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:34.898814922Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1638,"":"io task","client":"0","":"downstairs","session_id":"b283a3af-cf91-42cc-bc6b-d24dc5f7b6c4"} {"msg":"ds_connection connected from Ok(127.0.0.1:37013)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:34.898849532Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1638,"":"io task","client":"2","":"downstairs","session_id":"b283a3af-cf91-42cc-bc6b-d24dc5f7b6c4"} {"msg":"ds_connection connected from Ok(127.0.0.1:46303)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:34.898875467Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1638,"":"io task","client":"1","":"downstairs","session_id":"b283a3af-cf91-42cc-bc6b-d24dc5f7b6c4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:34.899404276Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1638,"client":"2","":"downstairs","session_id":"b283a3af-cf91-42cc-bc6b-d24dc5f7b6c4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:34.899430872Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1638,"client":"0","":"downstairs","session_id":"b283a3af-cf91-42cc-bc6b-d24dc5f7b6c4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:34.899910212Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1638,"client":"1","":"downstairs","session_id":"b283a3af-cf91-42cc-bc6b-d24dc5f7b6c4"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:34.900041489Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1638,"client":"0","":"downstairs","session_id":"b283a3af-cf91-42cc-bc6b-d24dc5f7b6c4"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:34.900151213Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1638,"client":"2","":"downstairs","session_id":"b283a3af-cf91-42cc-bc6b-d24dc5f7b6c4"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:34.90059032Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1638,"client":"1","":"downstairs","session_id":"b283a3af-cf91-42cc-bc6b-d24dc5f7b6c4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:34.900616555Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1638,"session_id":"b283a3af-cf91-42cc-bc6b-d24dc5f7b6c4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:34.900629823Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1638,"session_id":"b283a3af-cf91-42cc-bc6b-d24dc5f7b6c4"} {"msg":"[0]R flush_numbers[0..12]: [181, 203, 205, 203, 203, 177, 200, 195, 202, 197, 198, 197]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:34.901163425Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1638,"":"downstairs","":"downstairs","session_id":"b283a3af-cf91-42cc-bc6b-d24dc5f7b6c4"} {"msg":"[0]R generation[0..12]: [98, 112, 113, 112, 112, 96, 111, 106, 112, 108, 110, 108]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:34.901186038Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1638,"":"downstairs","":"downstairs","session_id":"b283a3af-cf91-42cc-bc6b-d24dc5f7b6c4"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:34.901200476Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1638,"":"downstairs","":"downstairs","session_id":"b283a3af-cf91-42cc-bc6b-d24dc5f7b6c4"} {"msg":"[1]R flush_numbers[0..12]: [181, 203, 205, 203, 203, 177, 200, 195, 202, 197, 198, 197]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:34.901213994Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1638,"":"downstairs","":"downstairs","session_id":"b283a3af-cf91-42cc-bc6b-d24dc5f7b6c4"} {"msg":"[1]R generation[0..12]: [98, 112, 113, 112, 112, 96, 111, 106, 112, 108, 110, 108]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:34.901227202Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1638,"":"downstairs","":"downstairs","session_id":"b283a3af-cf91-42cc-bc6b-d24dc5f7b6c4"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:34.90124081Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1638,"":"downstairs","":"downstairs","session_id":"b283a3af-cf91-42cc-bc6b-d24dc5f7b6c4"} {"msg":"[2]R flush_numbers[0..12]: [181, 203, 205, 203, 203, 177, 200, 195, 202, 197, 198, 197]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:34.901254458Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1638,"":"downstairs","":"downstairs","session_id":"b283a3af-cf91-42cc-bc6b-d24dc5f7b6c4"} {"msg":"[2]R generation[0..12]: [98, 112, 113, 112, 112, 96, 111, 106, 112, 108, 110, 108]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:34.901267776Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1638,"":"downstairs","":"downstairs","session_id":"b283a3af-cf91-42cc-bc6b-d24dc5f7b6c4"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:34.901281504Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1638,"":"downstairs","":"downstairs","session_id":"b283a3af-cf91-42cc-bc6b-d24dc5f7b6c4"} {"msg":"Max found gen is 114","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:34.901294251Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1638,"":"downstairs","":"downstairs","session_id":"b283a3af-cf91-42cc-bc6b-d24dc5f7b6c4"} {"msg":"Generation requested: 114 >= found:114","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:34.901307088Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1638,"":"downstairs","":"downstairs","session_id":"b283a3af-cf91-42cc-bc6b-d24dc5f7b6c4"} {"msg":"Next flush: 206","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:34.901319416Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1638,"":"downstairs","":"downstairs","session_id":"b283a3af-cf91-42cc-bc6b-d24dc5f7b6c4"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-17T21:15:34.901331863Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1638,"":"downstairs","":"downstairs","session_id":"b283a3af-cf91-42cc-bc6b-d24dc5f7b6c4"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:34.901349653Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1638,"session_id":"b283a3af-cf91-42cc-bc6b-d24dc5f7b6c4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:34.90136124Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1638,"client":"0","":"downstairs","session_id":"b283a3af-cf91-42cc-bc6b-d24dc5f7b6c4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:34.901375849Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1638,"client":"1","":"downstairs","session_id":"b283a3af-cf91-42cc-bc6b-d24dc5f7b6c4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:34.901388636Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1638,"client":"2","":"downstairs","session_id":"b283a3af-cf91-42cc-bc6b-d24dc5f7b6c4"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:34.901401564Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1638,"session_id":"b283a3af-cf91-42cc-bc6b-d24dc5f7b6c4"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:34.901413371Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1638,"session_id":"b283a3af-cf91-42cc-bc6b-d24dc5f7b6c4"} {"msg":"e2bb6b2b-cfcd-4e20-9471-c27590e3ef5b is now active with session: b283a3af-cf91-42cc-bc6b-d24dc5f7b6c4","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:34.901425868Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1638,"session_id":"b283a3af-cf91-42cc-bc6b-d24dc5f7b6c4"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:34.901437365Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1638,"session_id":"b283a3af-cf91-42cc-bc6b-d24dc5f7b6c4"} {"msg":"The guest has finished waiting for activation with:114","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:34.901449222Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1638} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 496 len 4096 data: 13 03/30 Read block 374 len 32768 04/30 Read block 554 len 24576 05/30 Read block 583 len 16384 06/30 Read block 551 len 8192 07/30 Read block 576 len 8192 08/30 Read block 233 len 32768 09/30 Read block 308 len 40960 10/30 Read block 401 len 28672 11/30 Flush 12/30 Write block 517 len 12288 data: 12 12 12 13/30 Write block 210 len 16384 data: 8 9 8 12 14/30 Write block 358 len 24576 data: 11 12 9 8 7 7 15/30 Write block 74 len 32768 data: 9 12 13 10 10 9 8 9 16/30 Read block 474 len 36864 17/30 Read block 108 len 16384 18/30 Write block 509 len 28672 data: 14 12 9 9 11 13 16 19/30 Read block 526 len 20480 20/30 Read block 398 len 28672 21/30 Flush 22/30 Write block 363 len 8192 data: 8 6 23/30 Read block 160 len 20480 24/30 Read block 77 len 16384 25/30 Read block 34 len 36864 26/30 Write block 410 len 16384 data: 6 5 8 10 27/30 Write block 197 len 12288 data: 8 10 14 28/30 Read block 186 len 20480 29/30 Write block 303 len 12288 data: 10 9 10 30/30 Read block 567 len 4096 ---------------------------------------------------------------- Crucible gen:114 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 2 Done Done Done false 1022 Acked Read 5 Done Done Done false 1023 Acked Read 4 Done Done Done false 1024 Acked Read 9 Done Done Done false 1025 Acked Write 4 Done Done Done false 1026 Acked Write 3 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Write 3 Done Done Done false 1029 Acked Read 1 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 9 9 9 27 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1020 1020 1020 Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:15:34.920239254Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1638} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 98 98 98 181 181 181 F F F 1 020-039 112 112 112 203 203 203 F F F 2 040-059 113 113 113 205 205 205 F F F 3 060-079 114 114 114 208 208 208 F F F 4 080-099 114 114 114 208 208 208 F F F 5 100-119 96 96 96 177 177 177 F F F 6 120-139 111 111 111 200 200 200 F F F 7 140-159 106 106 106 195 195 195 F F F 8 160-179 112 112 112 202 202 202 F F F 9 180-199 108 108 108 197 197 197 T T T 10 200-219 114 114 114 208 208 208 F F F 11 220-239 108 108 108 197 197 197 F F F 12 240-259 113 113 113 205 205 205 F F F 13 260-279 106 106 106 195 195 195 F F F 14 280-299 112 112 112 203 203 203 F F F 15 300-319 113 113 113 205 205 205 T T T 16 320-339 106 106 106 195 195 195 F F F 17 340-359 114 114 114 208 208 208 F F F 18 360-379 114 114 114 208 208 208 T T T 19 380-399 108 108 108 197 197 197 F F F 20 400-419 112 112 112 204 204 204 T T T 21 420-439 105 105 105 194 194 194 F F F 22 440-459 112 112 112 201 201 201 F F F 23 460-479 106 106 106 195 195 195 F F F 24 480-499 114 114 114 207 207 207 F F F 25 500-519 114 114 114 208 208 208 F F F 26 520-539 111 111 111 200 200 200 F F F 27 540-559 112 112 112 203 203 203 F F F 28 560-579 110 110 110 199 199 199 F F F 29 580-599 104 104 104 193 193 193 F F F Max gen: 114, Max flush: 208 A Difference in extent metadata is expected here On loop 57 Verifying data now No change after verify Loop: 57 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 98 98 98 181 181 181 F F F 1 020-039 112 112 112 203 203 203 F F F 2 040-059 113 113 113 205 205 205 F F F 3 060-079 114 114 114 208 208 208 F F F 4 080-099 114 114 114 208 208 208 F F F 5 100-119 96 96 96 177 177 177 F F F 6 120-139 111 111 111 200 200 200 F F F 7 140-159 106 106 106 195 195 195 F F F 8 160-179 112 112 112 202 202 202 F F F 9 180-199 115 115 115 209 209 209 F F F 10 200-219 114 114 114 208 208 208 F F F 11 220-239 108 108 108 197 197 197 F F F 12 240-259 113 113 113 205 205 205 F F F 13 260-279 106 106 106 195 195 195 F F F 14 280-299 112 112 112 203 203 203 F F F 15 300-319 115 115 115 209 209 209 F F F 16 320-339 106 106 106 195 195 195 F F F 17 340-359 114 114 114 208 208 208 F F F 18 360-379 115 115 115 209 209 209 F F F 19 380-399 108 108 108 197 197 197 F F F 20 400-419 115 115 115 209 209 209 F F F 21 420-439 105 105 105 194 194 194 F F F 22 440-459 112 112 112 201 201 201 F F F 23 460-479 106 106 106 195 195 195 F F F 24 480-499 114 114 114 207 207 207 F F F 25 500-519 114 114 114 208 208 208 F F F 26 520-539 111 111 111 200 200 200 F F F 27 540-559 112 112 112 203 203 203 F F F 28 560-579 110 110 110 199 199 199 F F F 29 580-599 104 104 104 193 193 193 F F F Max gen: 115, Max flush: 209 Begin loop 58 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:39.160379886Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1646,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:15:39.203024911Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1646,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:39.203484379Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1646,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:39.235011636Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1646,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:39.235609685Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1646} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"abd5430f1508af0bc008143ea2566277ef714ec2\",\n git_commit_timestamp: \"2025-01-17T20:55:03.000000000Z\",\n git_branch: \"alan/tokio-to-140\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:39.235642575Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1646} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:39.235655812Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1646} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:39.23567Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1646} {"msg":"Crucible 886cfaaa-35a9-43a3-aa55-f374488812ec has session id: c3549b76-43b8-45f6-9fb1-2add71e5a690","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:39.236270742Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1646,"session_id":"c3549b76-43b8-45f6-9fb1-2add71e5a690"} {"msg":"Upstairs opts: Upstairs UUID: 886cfaaa-35a9-43a3-aa55-f374488812ec, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:39.236309154Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1646,"session_id":"c3549b76-43b8-45f6-9fb1-2add71e5a690"} {"msg":"Crucible stats registered with UUID: 886cfaaa-35a9-43a3-aa55-f374488812ec","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:39.236331938Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1646,"session_id":"c3549b76-43b8-45f6-9fb1-2add71e5a690"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:15:39.236352639Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1646,"session_id":"c3549b76-43b8-45f6-9fb1-2add71e5a690"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:39.236933659Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1646,"task":"crutest"} {"msg":"The guest has requested activation with gen:116","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:39.236955752Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1646} {"msg":"886cfaaa-35a9-43a3-aa55-f374488812ec active request set","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:39.237027684Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1646,"session_id":"c3549b76-43b8-45f6-9fb1-2add71e5a690"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:39.237047966Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1646,"client":"0","":"downstairs","session_id":"c3549b76-43b8-45f6-9fb1-2add71e5a690"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:39.237078894Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1646,"client":"0","":"downstairs","session_id":"c3549b76-43b8-45f6-9fb1-2add71e5a690"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:39.237104499Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1646,"client":"1","":"downstairs","session_id":"c3549b76-43b8-45f6-9fb1-2add71e5a690"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:39.237131845Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1646,"client":"1","":"downstairs","session_id":"c3549b76-43b8-45f6-9fb1-2add71e5a690"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:39.237189799Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1646,"client":"2","":"downstairs","session_id":"c3549b76-43b8-45f6-9fb1-2add71e5a690"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:39.237220097Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1646,"client":"2","":"downstairs","session_id":"c3549b76-43b8-45f6-9fb1-2add71e5a690"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:39.237246102Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1646,"":"io task","client":"2","":"downstairs","session_id":"c3549b76-43b8-45f6-9fb1-2add71e5a690"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:39.237276239Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1646,"":"io task","client":"0","":"downstairs","session_id":"c3549b76-43b8-45f6-9fb1-2add71e5a690"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:39.237303425Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1646,"":"io task","client":"1","":"downstairs","session_id":"c3549b76-43b8-45f6-9fb1-2add71e5a690"} {"msg":"ds_connection connected from Ok(127.0.0.1:63179)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:39.237332372Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1646,"":"io task","client":"2","":"downstairs","session_id":"c3549b76-43b8-45f6-9fb1-2add71e5a690"} {"msg":"ds_connection connected from Ok(127.0.0.1:45988)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:39.237358427Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1646,"":"io task","client":"0","":"downstairs","session_id":"c3549b76-43b8-45f6-9fb1-2add71e5a690"} {"msg":"ds_connection connected from Ok(127.0.0.1:51854)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:39.237383682Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1646,"":"io task","client":"1","":"downstairs","session_id":"c3549b76-43b8-45f6-9fb1-2add71e5a690"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:39.238019834Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1646,"client":"1","":"downstairs","session_id":"c3549b76-43b8-45f6-9fb1-2add71e5a690"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:39.23804681Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1646,"client":"2","":"downstairs","session_id":"c3549b76-43b8-45f6-9fb1-2add71e5a690"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:39.23821864Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1646,"client":"0","":"downstairs","session_id":"c3549b76-43b8-45f6-9fb1-2add71e5a690"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:39.238683892Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1646,"client":"1","":"downstairs","session_id":"c3549b76-43b8-45f6-9fb1-2add71e5a690"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:39.238711989Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1646,"client":"2","":"downstairs","session_id":"c3549b76-43b8-45f6-9fb1-2add71e5a690"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:39.238727698Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1646,"client":"0","":"downstairs","session_id":"c3549b76-43b8-45f6-9fb1-2add71e5a690"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:39.239285024Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1646,"session_id":"c3549b76-43b8-45f6-9fb1-2add71e5a690"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:39.239310879Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1646,"session_id":"c3549b76-43b8-45f6-9fb1-2add71e5a690"} {"msg":"[0]R flush_numbers[0..12]: [181, 203, 205, 208, 208, 177, 200, 195, 202, 209, 208, 197]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:39.239367982Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1646,"":"downstairs","":"downstairs","session_id":"c3549b76-43b8-45f6-9fb1-2add71e5a690"} {"msg":"[0]R generation[0..12]: [98, 112, 113, 114, 114, 96, 111, 106, 112, 115, 114, 108]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:39.239387664Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1646,"":"downstairs","":"downstairs","session_id":"c3549b76-43b8-45f6-9fb1-2add71e5a690"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:39.239402172Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1646,"":"downstairs","":"downstairs","session_id":"c3549b76-43b8-45f6-9fb1-2add71e5a690"} {"msg":"[1]R flush_numbers[0..12]: [181, 203, 205, 208, 208, 177, 200, 195, 202, 209, 208, 197]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:39.23941568Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1646,"":"downstairs","":"downstairs","session_id":"c3549b76-43b8-45f6-9fb1-2add71e5a690"} {"msg":"[1]R generation[0..12]: [98, 112, 113, 114, 114, 96, 111, 106, 112, 115, 114, 108]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:39.239429088Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1646,"":"downstairs","":"downstairs","session_id":"c3549b76-43b8-45f6-9fb1-2add71e5a690"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:39.239442786Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1646,"":"downstairs","":"downstairs","session_id":"c3549b76-43b8-45f6-9fb1-2add71e5a690"} {"msg":"[2]R flush_numbers[0..12]: [181, 203, 205, 208, 208, 177, 200, 195, 202, 209, 208, 197]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:39.239456273Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1646,"":"downstairs","":"downstairs","session_id":"c3549b76-43b8-45f6-9fb1-2add71e5a690"} {"msg":"[2]R generation[0..12]: [98, 112, 113, 114, 114, 96, 111, 106, 112, 115, 114, 108]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:39.239469721Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1646,"":"downstairs","":"downstairs","session_id":"c3549b76-43b8-45f6-9fb1-2add71e5a690"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:39.239483549Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1646,"":"downstairs","":"downstairs","session_id":"c3549b76-43b8-45f6-9fb1-2add71e5a690"} {"msg":"Max found gen is 116","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:39.239499179Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1646,"":"downstairs","":"downstairs","session_id":"c3549b76-43b8-45f6-9fb1-2add71e5a690"} {"msg":"Generation requested: 116 >= found:116","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:39.239512276Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1646,"":"downstairs","":"downstairs","session_id":"c3549b76-43b8-45f6-9fb1-2add71e5a690"} {"msg":"Next flush: 210","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:39.239524874Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1646,"":"downstairs","":"downstairs","session_id":"c3549b76-43b8-45f6-9fb1-2add71e5a690"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:39.239537221Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1646,"":"downstairs","":"downstairs","session_id":"c3549b76-43b8-45f6-9fb1-2add71e5a690"} Wait for a query_work_queue command to finish before sending IO {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:39.239557332Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1646,"session_id":"c3549b76-43b8-45f6-9fb1-2add71e5a690"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:39.23956913Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1646,"client":"0","":"downstairs","session_id":"c3549b76-43b8-45f6-9fb1-2add71e5a690"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:39.239582017Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1646,"client":"1","":"downstairs","session_id":"c3549b76-43b8-45f6-9fb1-2add71e5a690"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:39.239594725Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1646,"client":"2","":"downstairs","session_id":"c3549b76-43b8-45f6-9fb1-2add71e5a690"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:39.239607612Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1646,"session_id":"c3549b76-43b8-45f6-9fb1-2add71e5a690"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:39.239619459Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1646,"session_id":"c3549b76-43b8-45f6-9fb1-2add71e5a690"} {"msg":"886cfaaa-35a9-43a3-aa55-f374488812ec is now active with session: c3549b76-43b8-45f6-9fb1-2add71e5a690","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:39.239631686Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1646,"session_id":"c3549b76-43b8-45f6-9fb1-2add71e5a690"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:39.239643013Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1646,"session_id":"c3549b76-43b8-45f6-9fb1-2add71e5a690"} {"msg":"The guest has finished waiting for activation with:116","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:39.23965482Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1646} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 2 len 4096 data: 8 02/30 Read block 361 len 40960 03/30 Write block 348 len 16384 data: 9 8 10 7 04/30 Flush 05/30 Write block 164 len 20480 data: 9 11 10 9 10 06/30 Flush 07/30 Write block 508 len 20480 data: 12 15 13 10 10 08/30 Read block 591 len 12288 09/30 Read block 420 len 4096 10/30 Read block 164 len 12288 11/30 Write block 120 len 28672 data: 9 6 5 5 2 5 5 12/30 Write block 141 len 8192 data: 9 10 13/30 Read block 65 len 16384 14/30 Write block 72 len 4096 data: 10 15/30 Flush 16/30 Write block 206 len 28672 data: 5 6 8 9 9 10 9 17/30 Read block 564 len 8192 18/30 Write block 454 len 36864 data: 8 8 8 9 10 10 10 12 12 19/30 Flush 20/30 Write block 507 len 8192 data: 13 13 21/30 Write block 329 len 20480 data: 8 8 8 7 6 22/30 Read block 335 len 16384 23/30 Read block 333 len 32768 24/30 Read block 505 len 32768 25/30 Read block 142 len 28672 26/30 Write block 355 len 36864 data: 11 12 13 12 13 10 9 8 9 27/30 Read block 195 len 16384 28/30 Write block 525 len 8192 data: 9 10 29/30 Read block 498 len 8192 30/30 Read block 475 len 24576 ---------------------------------------------------------------- Crucible gen:116 GIO:true work queues: Upstairs:0 downstairs:11 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1019 Acked Write 2 Done Done Done false 1020 Acked Write 5 Done Done Done false 1021 Acked Read 4 Done Done Done false 1022 Acked Read 8 Done Done Done false 1023 Acked Read 8 Done Done Done false 1024 Acked Read 7 Done Done Done false 1025 Acked Write 9 Done Done Done false 1026 Acked Read 4 Done Done Done false 1027 Acked Write 2 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Read 6 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 11 10 10 31 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1018 1018 1018 Downstairs last five completed: 1018 1017 1016 1015 1014 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:15:39.258538927Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1646} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 116 116 116 210 210 210 F F F 1 020-039 112 112 112 203 203 203 F F F 2 040-059 113 113 113 205 205 205 F F F 3 060-079 116 116 116 212 212 212 F F F 4 080-099 114 114 114 208 208 208 F F F 5 100-119 96 96 96 177 177 177 F F F 6 120-139 116 116 116 212 212 212 F F F 7 140-159 116 116 116 212 212 212 F F F 8 160-179 116 116 116 211 211 211 F F F 9 180-199 115 115 115 209 209 209 F F F 10 200-219 116 116 116 213 213 213 F F F 11 220-239 108 108 108 197 197 197 F F F 12 240-259 113 113 113 205 205 205 F F F 13 260-279 106 106 106 195 195 195 F F F 14 280-299 112 112 112 203 203 203 F F F 15 300-319 115 115 115 209 209 209 F F F 16 320-339 106 106 106 195 195 195 T T T 17 340-359 116 116 116 210 210 210 T T T 18 360-379 115 115 115 209 209 209 T T T 19 380-399 108 108 108 197 197 197 F F F 20 400-419 115 115 115 209 209 209 F F F 21 420-439 105 105 105 194 194 194 F F F 22 440-459 116 116 116 213 213 213 F F F 23 460-479 116 116 116 213 213 213 F F F 24 480-499 114 114 114 207 207 207 F F F 25 500-519 116 116 116 212 212 212 T T T 26 520-539 111 111 111 200 200 200 T T T 27 540-559 112 112 112 203 203 203 F F F 28 560-579 110 110 110 199 199 199 F F F 29 580-599 104 104 104 193 193 193 F F F Max gen: 116, Max flush: 213 A Difference in extent metadata is expected here On loop 58 Verifying data now No change after verify Loop: 58 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 116 116 116 210 210 210 F F F 1 020-039 112 112 112 203 203 203 F F F 2 040-059 113 113 113 205 205 205 F F F 3 060-079 116 116 116 212 212 212 F F F 4 080-099 114 114 114 208 208 208 F F F 5 100-119 96 96 96 177 177 177 F F F 6 120-139 116 116 116 212 212 212 F F F 7 140-159 116 116 116 212 212 212 F F F 8 160-179 116 116 116 211 211 211 F F F 9 180-199 115 115 115 209 209 209 F F F 10 200-219 116 116 116 213 213 213 F F F 11 220-239 108 108 108 197 197 197 F F F 12 240-259 113 113 113 205 205 205 F F F 13 260-279 106 106 106 195 195 195 F F F 14 280-299 112 112 112 203 203 203 F F F 15 300-319 115 115 115 209 209 209 F F F 16 320-339 117 117 117 214 214 214 F F F 17 340-359 117 117 117 214 214 214 F F F 18 360-379 117 117 117 214 214 214 F F F 19 380-399 108 108 108 197 197 197 F F F 20 400-419 115 115 115 209 209 209 F F F 21 420-439 105 105 105 194 194 194 F F F 22 440-459 116 116 116 213 213 213 F F F 23 460-479 116 116 116 213 213 213 F F F 24 480-499 114 114 114 207 207 207 F F F 25 500-519 117 117 117 214 214 214 F F F 26 520-539 117 117 117 214 214 214 F F F 27 540-559 112 112 112 203 203 203 F F F 28 560-579 110 110 110 199 199 199 F F F 29 580-599 104 104 104 193 193 193 F F F Max gen: 117, Max flush: 214 Begin loop 59 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:43.53475225Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1654,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:15:43.578571281Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1654,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:43.57901456Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1654,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:43.611500017Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1654,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:43.612111165Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1654} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"abd5430f1508af0bc008143ea2566277ef714ec2\",\n git_commit_timestamp: \"2025-01-17T20:55:03.000000000Z\",\n git_branch: \"alan/tokio-to-140\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:43.612160253Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1654} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:43.612176113Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1654} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:43.612193263Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1654} {"msg":"Crucible b15c1c02-3565-4f68-be92-15036205b4a9 has session id: 8d1ff0c1-2a86-44be-befe-9b15032d8868","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:43.612766748Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1654,"session_id":"8d1ff0c1-2a86-44be-befe-9b15032d8868"} {"msg":"Upstairs opts: Upstairs UUID: b15c1c02-3565-4f68-be92-15036205b4a9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:43.61280534Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1654,"session_id":"8d1ff0c1-2a86-44be-befe-9b15032d8868"} {"msg":"Crucible stats registered with UUID: b15c1c02-3565-4f68-be92-15036205b4a9","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:43.612823941Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1654,"session_id":"8d1ff0c1-2a86-44be-befe-9b15032d8868"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:15:43.61283948Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1654,"session_id":"8d1ff0c1-2a86-44be-befe-9b15032d8868"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:43.613358884Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1654,"task":"crutest"} {"msg":"The guest has requested activation with gen:118","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:43.61338634Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1654} {"msg":"b15c1c02-3565-4f68-be92-15036205b4a9 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:43.613421Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1654,"session_id":"8d1ff0c1-2a86-44be-befe-9b15032d8868"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:43.61343796Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1654,"client":"0","":"downstairs","session_id":"8d1ff0c1-2a86-44be-befe-9b15032d8868"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:43.613465676Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1654,"client":"0","":"downstairs","session_id":"8d1ff0c1-2a86-44be-befe-9b15032d8868"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:43.613479434Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1654,"client":"1","":"downstairs","session_id":"8d1ff0c1-2a86-44be-befe-9b15032d8868"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:43.613493323Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1654,"client":"1","":"downstairs","session_id":"8d1ff0c1-2a86-44be-befe-9b15032d8868"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:43.61350599Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1654,"client":"2","":"downstairs","session_id":"8d1ff0c1-2a86-44be-befe-9b15032d8868"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:43.613519868Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1654,"client":"2","":"downstairs","session_id":"8d1ff0c1-2a86-44be-befe-9b15032d8868"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:43.61354101Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1654,"":"io task","client":"2","":"downstairs","session_id":"8d1ff0c1-2a86-44be-befe-9b15032d8868"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:43.613564684Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1654,"":"io task","client":"0","":"downstairs","session_id":"8d1ff0c1-2a86-44be-befe-9b15032d8868"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:43.61359241Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1654,"":"io task","client":"1","":"downstairs","session_id":"8d1ff0c1-2a86-44be-befe-9b15032d8868"} {"msg":"ds_connection connected from Ok(127.0.0.1:55616)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:43.613653226Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1654,"":"io task","client":"0","":"downstairs","session_id":"8d1ff0c1-2a86-44be-befe-9b15032d8868"} {"msg":"ds_connection connected from Ok(127.0.0.1:49874)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:43.613686976Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1654,"":"io task","client":"2","":"downstairs","session_id":"8d1ff0c1-2a86-44be-befe-9b15032d8868"} {"msg":"ds_connection connected from Ok(127.0.0.1:33898)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:43.61371205Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1654,"":"io task","client":"1","":"downstairs","session_id":"8d1ff0c1-2a86-44be-befe-9b15032d8868"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:43.61446353Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1654,"client":"1","":"downstairs","session_id":"8d1ff0c1-2a86-44be-befe-9b15032d8868"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:43.614504043Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1654,"client":"2","":"downstairs","session_id":"8d1ff0c1-2a86-44be-befe-9b15032d8868"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:43.614609235Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1654,"client":"0","":"downstairs","session_id":"8d1ff0c1-2a86-44be-befe-9b15032d8868"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:43.615155384Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1654,"client":"2","":"downstairs","session_id":"8d1ff0c1-2a86-44be-befe-9b15032d8868"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:43.615194497Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1654,"client":"1","":"downstairs","session_id":"8d1ff0c1-2a86-44be-befe-9b15032d8868"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:43.615316868Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1654,"client":"0","":"downstairs","session_id":"8d1ff0c1-2a86-44be-befe-9b15032d8868"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:43.615744778Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1654,"session_id":"8d1ff0c1-2a86-44be-befe-9b15032d8868"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:43.615826066Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1654,"session_id":"8d1ff0c1-2a86-44be-befe-9b15032d8868"} {"msg":"[0]R flush_numbers[0..12]: [210, 203, 205, 212, 208, 177, 212, 212, 211, 209, 213, 197]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:43.615941573Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1654,"":"downstairs","":"downstairs","session_id":"8d1ff0c1-2a86-44be-befe-9b15032d8868"} {"msg":"[0]R generation[0..12]: [116, 112, 113, 116, 114, 96, 116, 116, 116, 115, 116, 108]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:43.615961125Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1654,"":"downstairs","":"downstairs","session_id":"8d1ff0c1-2a86-44be-befe-9b15032d8868"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:43.615979876Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1654,"":"downstairs","":"downstairs","session_id":"8d1ff0c1-2a86-44be-befe-9b15032d8868"} {"msg":"[1]R flush_numbers[0..12]: [210, 203, 205, 212, 208, 177, 212, 212, 211, 209, 213, 197]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:43.615993924Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1654,"":"downstairs","":"downstairs","session_id":"8d1ff0c1-2a86-44be-befe-9b15032d8868"} {"msg":"[1]R generation[0..12]: [116, 112, 113, 116, 114, 96, 116, 116, 116, 115, 116, 108]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:43.616007402Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1654,"":"downstairs","":"downstairs","session_id":"8d1ff0c1-2a86-44be-befe-9b15032d8868"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:43.61602116Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1654,"":"downstairs","":"downstairs","session_id":"8d1ff0c1-2a86-44be-befe-9b15032d8868"} {"msg":"[2]R flush_numbers[0..12]: [210, 203, 205, 212, 208, 177, 212, 212, 211, 209, 213, 197]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:43.616034738Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1654,"":"downstairs","":"downstairs","session_id":"8d1ff0c1-2a86-44be-befe-9b15032d8868"} {"msg":"[2]R generation[0..12]: [116, 112, 113, 116, 114, 96, 116, 116, 116, 115, 116, 108]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:43.616048025Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1654,"":"downstairs","":"downstairs","session_id":"8d1ff0c1-2a86-44be-befe-9b15032d8868"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:43.616061353Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1654,"":"downstairs","":"downstairs","session_id":"8d1ff0c1-2a86-44be-befe-9b15032d8868"} {"msg":"Max found gen is 118","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:43.616073821Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1654,"":"downstairs","":"downstairs","session_id":"8d1ff0c1-2a86-44be-befe-9b15032d8868"} {"msg":"Generation requested: 118 >= found:118","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:43.616086628Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1654,"":"downstairs","":"downstairs","session_id":"8d1ff0c1-2a86-44be-befe-9b15032d8868"} {Wait for a query_work_queue command to finish before sending IO "msg":"Next flush: 215","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:43.616105099Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1654,"":"downstairs","":"downstairs","session_id":"8d1ff0c1-2a86-44be-befe-9b15032d8868"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:43.616127142Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1654,"":"downstairs","":"downstairs","session_id":"8d1ff0c1-2a86-44be-befe-9b15032d8868"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:43.61614181Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1654,"session_id":"8d1ff0c1-2a86-44be-befe-9b15032d8868"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:43.616153637Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1654,"client":"0","":"downstairs","session_id":"8d1ff0c1-2a86-44be-befe-9b15032d8868"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:43.616166675Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1654,"client":"1","":"downstairs","session_id":"8d1ff0c1-2a86-44be-befe-9b15032d8868"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:43.616179593Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1654,"client":"2","":"downstairs","session_id":"8d1ff0c1-2a86-44be-befe-9b15032d8868"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:43.616194251Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1654,"session_id":"8d1ff0c1-2a86-44be-befe-9b15032d8868"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:43.616206198Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1654,"session_id":"8d1ff0c1-2a86-44be-befe-9b15032d8868"} {"msg":"b15c1c02-3565-4f68-be92-15036205b4a9 is now active with session: 8d1ff0c1-2a86-44be-befe-9b15032d8868","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:43.616218735Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1654,"session_id":"8d1ff0c1-2a86-44be-befe-9b15032d8868"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:43.616230162Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1654,"session_id":"8d1ff0c1-2a86-44be-befe-9b15032d8868"} {"msg":"The guest has finished waiting for activation with:118","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:43.616241979Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1654} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 485 len 36864 data: 6 7 7 7 9 10 10 8 9 02/30 Read block 335 len 36864 03/30 Read block 586 len 36864 04/30 Read block 98 len 8192 05/30 Write block 33 len 20480 data: 14 14 17 11 10 06/30 Read block 317 len 40960 07/30 Write block 576 len 28672 data: 9 9 8 10 9 10 9 08/30 Write block 131 len 8192 data: 6 7 09/30 Read block 590 len 24576 10/30 Write block 318 len 16384 data: 7 8 9 8 11/30 Read block 488 len 24576 12/30 Flush 13/30 Write block 356 len 20480 data: 13 14 13 14 11 14/30 Write block 370 len 36864 data: 8 8 12 10 12 14 13 15 16 15/30 Write block 228 len 4096 data: 8 16/30 Read block 26 len 28672 17/30 Read block 577 len 8192 18/30 Read block 429 len 12288 19/30 Read block 220 len 16384 20/30 Write block 550 len 36864 data: 11 8 6 4 5 6 7 9 11 21/30 Write block 37 len 24576 data: 11 10 9 9 9 8 22/30 Write block 103 len 40960 data: 11 11 12 9 7 7 8 9 12 13 23/30 Flush 24/30 Write block 511 len 28672 data: 11 11 12 14 17 16 13 25/30 Read block 68 len 40960 26/30 Read block 476 len 24576 27/30 Read block 108 len 36864 28/30 Write block 565 len 4096 data: 9 29/30 Write block 227 len 28672 data: 9 9 9 10 9 8 9 30/30 Read block 34 len 20480 ---------------------------------------------------------------- Crucible gen:118 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 7 Done Done Done false 1024 Acked Read 10 Done Done Done false 1025 Acked Read 6 Done Done Done false 1026 Acked Read 9 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Write 7 Done Sent Done false 1029 Acked Read 5 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 2 1 3 Done 7 5 6 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:15:43.636588571Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1654} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 116 116 116 210 210 210 F F F 1 020-039 118 118 118 216 216 216 F F F 2 040-059 118 118 118 216 216 216 F F F 3 060-079 116 116 116 212 212 212 F F F 4 080-099 114 114 114 208 208 208 F F F 5 100-119 118 118 118 216 216 216 F F F 6 120-139 118 118 118 215 215 215 F F F 7 140-159 116 116 116 212 212 212 F F F 8 160-179 116 116 116 211 211 211 F F F 9 180-199 115 115 115 209 209 209 F F F 10 200-219 116 116 116 213 213 213 F F F 11 220-239 118 118 118 216 216 216 T T T 12 240-259 113 113 113 205 205 205 F F F 13 260-279 106 106 106 195 195 195 F F F 14 280-299 112 112 112 203 203 203 F F F 15 300-319 118 118 118 215 215 215 F F F 16 320-339 118 118 118 215 215 215 F F F 17 340-359 118 118 118 216 216 216 F F F 18 360-379 118 118 118 216 216 216 F F F 19 380-399 108 108 108 197 197 197 F F F 20 400-419 115 115 115 209 209 209 F F F 21 420-439 105 105 105 194 194 194 F F F 22 440-459 116 116 116 213 213 213 F F F 23 460-479 116 116 116 213 213 213 F F F 24 480-499 118 118 118 215 215 215 F F F 25 500-519 117 117 117 214 214 214 T T T 26 520-539 117 117 117 214 214 214 F F F 27 540-559 118 118 118 216 216 216 F F F 28 560-579 118 118 118 215 215 215 T T T 29 580-599 118 118 118 215 215 215 F F F Max gen: 118, Max flush: 216 A Difference in extent metadata is expected here On loop 59 Verifying data now No change after verify Loop: 59 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 116 116 116 210 210 210 F F F 1 020-039 118 118 118 216 216 216 F F F 2 040-059 118 118 118 216 216 216 F F F 3 060-079 116 116 116 212 212 212 F F F 4 080-099 114 114 114 208 208 208 F F F 5 100-119 118 118 118 216 216 216 F F F 6 120-139 118 118 118 215 215 215 F F F 7 140-159 116 116 116 212 212 212 F F F 8 160-179 116 116 116 211 211 211 F F F 9 180-199 115 115 115 209 209 209 F F F 10 200-219 116 116 116 213 213 213 F F F 11 220-239 119 119 119 217 217 217 F F F 12 240-259 113 113 113 205 205 205 F F F 13 260-279 106 106 106 195 195 195 F F F 14 280-299 112 112 112 203 203 203 F F F 15 300-319 118 118 118 215 215 215 F F F 16 320-339 118 118 118 215 215 215 F F F 17 340-359 118 118 118 216 216 216 F F F 18 360-379 118 118 118 216 216 216 F F F 19 380-399 108 108 108 197 197 197 F F F 20 400-419 115 115 115 209 209 209 F F F 21 420-439 105 105 105 194 194 194 F F F 22 440-459 116 116 116 213 213 213 F F F 23 460-479 116 116 116 213 213 213 F F F 24 480-499 118 118 118 215 215 215 F F F 25 500-519 119 119 119 217 217 217 F F F 26 520-539 117 117 117 214 214 214 F F F 27 540-559 118 118 118 216 216 216 F F F 28 560-579 119 119 119 217 217 217 F F F 29 580-599 118 118 118 215 215 215 F F F Max gen: 119, Max flush: 217 Begin loop 60 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:47.892381059Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1662,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:15:47.936100131Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1662,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:47.93652577Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1662,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:47.968755257Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1662,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:47.969365034Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1662} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"abd5430f1508af0bc008143ea2566277ef714ec2\",\n git_commit_timestamp: \"2025-01-17T20:55:03.000000000Z\",\n git_branch: \"alan/tokio-to-140\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:47.969403456Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1662} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:47.969419216Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1662} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:47.969436446Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1662} {"msg":"Crucible 4acbab33-4bd5-44bf-bc5d-04fb9cfda705 has session id: 965df20f-4d4d-4134-b041-727cab3622cc","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:47.969935617Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1662,"session_id":"965df20f-4d4d-4134-b041-727cab3622cc"} {"msg":"Upstairs opts: Upstairs UUID: 4acbab33-4bd5-44bf-bc5d-04fb9cfda705, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:47.969970538Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1662,"session_id":"965df20f-4d4d-4134-b041-727cab3622cc"} {"msg":"Crucible stats registered with UUID: 4acbab33-4bd5-44bf-bc5d-04fb9cfda705","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:47.969988498Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1662,"session_id":"965df20f-4d4d-4134-b041-727cab3622cc"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:15:47.970004287Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1662,"session_id":"965df20f-4d4d-4134-b041-727cab3622cc"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:47.97040113Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1662,"task":"crutest"} {"msg":"The guest has requested activation with gen:120","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:47.970424433Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1662} {"msg":"4acbab33-4bd5-44bf-bc5d-04fb9cfda705 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:47.970482397Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1662,"session_id":"965df20f-4d4d-4134-b041-727cab3622cc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:47.970508532Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1662,"client":"0","":"downstairs","session_id":"965df20f-4d4d-4134-b041-727cab3622cc"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:47.970537869Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1662,"client":"0","":"downstairs","session_id":"965df20f-4d4d-4134-b041-727cab3622cc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:47.970552378Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1662,"client":"1","":"downstairs","session_id":"965df20f-4d4d-4134-b041-727cab3622cc"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:47.970566696Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1662,"client":"1","":"downstairs","session_id":"965df20f-4d4d-4134-b041-727cab3622cc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:47.970588849Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1662,"client":"2","":"downstairs","session_id":"965df20f-4d4d-4134-b041-727cab3622cc"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:47.970603698Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1662,"client":"2","":"downstairs","session_id":"965df20f-4d4d-4134-b041-727cab3622cc"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:47.970616605Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1662,"":"io task","client":"2","":"downstairs","session_id":"965df20f-4d4d-4134-b041-727cab3622cc"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:47.970629893Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1662,"":"io task","client":"0","":"downstairs","session_id":"965df20f-4d4d-4134-b041-727cab3622cc"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:47.970643101Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1662,"":"io task","client":"1","":"downstairs","session_id":"965df20f-4d4d-4134-b041-727cab3622cc"} {"msg":"ds_connection connected from Ok(127.0.0.1:59512)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:47.970710731Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1662,"":"io task","client":"1","":"downstairs","session_id":"965df20f-4d4d-4134-b041-727cab3622cc"} {"msg":"ds_connection connected from Ok(127.0.0.1:60979)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:47.970746712Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1662,"":"io task","client":"2","":"downstairs","session_id":"965df20f-4d4d-4134-b041-727cab3622cc"} {"msg":"ds_connection connected from Ok(127.0.0.1:60844)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:47.970977286Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1662,"":"io task","client":"0","":"downstairs","session_id":"965df20f-4d4d-4134-b041-727cab3622cc"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:47.971464261Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1662,"client":"2","":"downstairs","session_id":"965df20f-4d4d-4134-b041-727cab3622cc"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:47.971501573Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1662,"client":"1","":"downstairs","session_id":"965df20f-4d4d-4134-b041-727cab3622cc"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:47.97153157Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1662,"client":"0","":"downstairs","session_id":"965df20f-4d4d-4134-b041-727cab3622cc"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:47.972144388Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1662,"client":"1","":"downstairs","session_id":"965df20f-4d4d-4134-b041-727cab3622cc"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:47.97224992Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1662,"client":"0","":"downstairs","session_id":"965df20f-4d4d-4134-b041-727cab3622cc"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:47.972273824Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1662,"client":"2","":"downstairs","session_id":"965df20f-4d4d-4134-b041-727cab3622cc"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:47.972651865Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1662,"session_id":"965df20f-4d4d-4134-b041-727cab3622cc"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:47.97272856Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1662,"session_id":"965df20f-4d4d-4134-b041-727cab3622cc"} {"msg":"[0]R flush_numbers[0..12]: [210, 216, 216, 212, 208, 216, 215, 212, 211, 209, 213, 217]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:47.972841296Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1662,"":"downstairs","":"downstairs","session_id":"965df20f-4d4d-4134-b041-727cab3622cc"} {"msg":"[0]R generation[0..12]: [116, 118, 118, 116, 114, 118, 118, 116, 116, 115, 116, 119]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:47.972860367Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1662,"":"downstairs","":"downstairs","session_id":"965df20f-4d4d-4134-b041-727cab3622cc"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:47.972874675Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1662,"":"downstairs","":"downstairs","session_id":"965df20f-4d4d-4134-b041-727cab3622cc"} {"msg":"[1]R flush_numbers[0..12]: [210, 216, 216, 212, 208, 216, 215, 212, 211, 209, 213, 217]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:47.972888303Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1662,"":"downstairs","":"downstairs","session_id":"965df20f-4d4d-4134-b041-727cab3622cc"} {"msg":"[1]R generation[0..12]: [116, 118, 118, 116, 114, 118, 118, 116, 116, 115, 116, 119]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:47.972901841Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1662,"":"downstairs","":"downstairs","session_id":"965df20f-4d4d-4134-b041-727cab3622cc"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:47.972915729Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1662,"":"downstairs","":"downstairs","session_id":"965df20f-4d4d-4134-b041-727cab3622cc"} {"msg":"[2]R flush_numbers[0..12]: [210, 216, 216, 212, 208, 216, 215, 212, 211, 209, 213, 217]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:47.972929447Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1662,"":"downstairs","":"downstairs","session_id":"965df20f-4d4d-4134-b041-727cab3622cc"} {"msg":"[2]R generation[0..12]: [116, 118, 118, 116, 114, 118, 118, 116, 116, 115, 116, 119]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:47.972942825Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1662,"":"downstairs","":"downstairs","session_id":"965df20f-4d4d-4134-b041-727cab3622cc"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:47.972956273Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1662,"":"downstairs","":"downstairs","session_id":"965df20f-4d4d-4134-b041-727cab3622cc"} {"msg":"Max found gen is 120","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:47.972972002Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1662,"":"downstairs","":"downstairs","session_id":"965df20f-4d4d-4134-b041-727cab3622cc"} {"msg":"Generation requested: 120 >= found:120","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:47.97298532Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1662,"":"downstairs","":"downstairs","session_id":"965df20f-4d4d-4134-b041-727cab3622cc"} {"msg":"Next flush: 218","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:47.972997757Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1662,"":"downstairs","":"downstairs","session_id":"965df20f-4d4d-4134-b041-727cab3622cc"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:47.973010404Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1662,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO 965df20f-4d4d-4134-b041-727cab3622cc"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:47.973030496Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1662,"session_id":"965df20f-4d4d-4134-b041-727cab3622cc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:47.973041843Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1662,"client":"0","":"downstairs","session_id":"965df20f-4d4d-4134-b041-727cab3622cc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:47.97305463Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1662,"client":"1","":"downstairs","session_id":"965df20f-4d4d-4134-b041-727cab3622cc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:47.973067568Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1662,"client":"2","":"downstairs","session_id":"965df20f-4d4d-4134-b041-727cab3622cc"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:47.973080385Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1662,"session_id":"965df20f-4d4d-4134-b041-727cab3622cc"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:47.973092122Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1662,"session_id":"965df20f-4d4d-4134-b041-727cab3622cc"} {"msg":"4acbab33-4bd5-44bf-bc5d-04fb9cfda705 is now active with session: 965df20f-4d4d-4134-b041-727cab3622cc","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:47.973104459Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1662,"session_id":"965df20f-4d4d-4134-b041-727cab3622cc"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:47.973116036Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1662,"session_id":"965df20f-4d4d-4134-b041-727cab3622cc"} {"msg":"The guest has finished waiting for activation with:120","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:47.973127993Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1662} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 55 len 36864 data: 13 12 11 11 10 8 8 5 4 02/30 Read block 150 len 32768 03/30 Flush 04/30 Write block 373 len 20480 data: 11 13 15 14 16 05/30 Read block 299 len 24576 06/30 Read block 144 len 28672 07/30 Write block 57 len 24576 data: 12 12 11 9 9 6 08/30 Read block 506 len 16384 09/30 Read block 551 len 32768 10/30 Flush 11/30 Write block 464 len 16384 data: 11 12 10 12 12/30 Write block 337 len 12288 data: 13 14 15 13/30 Write block 127 len 24576 data: 7 7 7 8 7 8 14/30 Write block 560 len 28672 data: 14 14 16 15 10 10 12 15/30 Read block 560 len 32768 16/30 Read block 49 len 36864 17/30 Read block 437 len 8192 18/30 Write block 240 len 40960 data: 7 9 9 9 9 8 8 10 10 8 19/30 Flush 20/30 Write block 303 len 24576 data: 11 10 11 9 9 8 21/30 Read block 471 len 36864 22/30 Write block 326 len 4096 data: 10 23/30 Flush 24/30 Write block 323 len 12288 data: 10 9 10 25/30 Write block 228 len 32768 data: 10 10 11 10 9 10 11 12 26/30 Write block 585 len 20480 data: 10 10 9 8 8 27/30 Write block 461 len 16384 data: 13 13 11 12 28/30 Read block 116 len 8192 29/30 Write block 166 len 16384 data: 11 10 11 9 30/30 Read block 45 len 20480 ---------------------------------------------------------------- Crucible gen:120 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 3 Done Done Done false 1024 Acked Write 8 Done Done Done false 1025 Acked Write 5 Done Done Done false 1026 Acked Write 4 Done Done Done false 1027 Acked Read 2 Done Done Done false 1028 Acked Write 4 Done Done Done false 1029 Acked Read 5 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 7 7 6 20 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:15:47.992363266Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1662} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 116 116 116 210 210 210 F F F 1 020-039 118 118 118 216 216 216 F F F 2 040-059 120 120 120 219 219 219 F F F 3 060-079 120 120 120 219 219 219 F F F 4 080-099 114 114 114 208 208 208 F F F 5 100-119 118 118 118 216 216 216 F F F 6 120-139 120 120 120 220 220 220 F F F 7 140-159 116 116 116 212 212 212 F F F 8 160-179 116 116 116 211 211 211 T T T 9 180-199 115 115 115 209 209 209 F F F 10 200-219 116 116 116 213 213 213 F F F 11 220-239 119 119 119 217 217 217 T T T 12 240-259 120 120 120 220 220 220 F F F 13 260-279 106 106 106 195 195 195 F F F 14 280-299 112 112 112 203 203 203 F F F 15 300-319 120 120 120 221 221 221 F F F 16 320-339 120 120 120 221 221 221 T T T 17 340-359 118 118 118 216 216 216 F F F 18 360-379 120 120 120 219 219 219 F F F 19 380-399 108 108 108 197 197 197 F F F 20 400-419 115 115 115 209 209 209 F F F 21 420-439 105 105 105 194 194 194 F F F 22 440-459 116 116 116 213 213 213 F F F 23 460-479 120 120 120 220 220 220 T T T 24 480-499 118 118 118 215 215 215 F F F 25 500-519 119 119 119 217 217 217 F F F 26 520-539 117 117 117 214 214 214 F F F 27 540-559 118 118 118 216 216 216 F F F 28 560-579 120 120 120 220 220 220 F F F 29 580-599 118 118 118 215 215 215 T T T Max gen: 120, Max flush: 221 A Difference in extent metadata is expected here On loop 60 Verifying data now No change after verify Loop: 60 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 116 116 116 210 210 210 F F F 1 020-039 118 118 118 216 216 216 F F F 2 040-059 120 120 120 219 219 219 F F F 3 060-079 120 120 120 219 219 219 F F F 4 080-099 114 114 114 208 208 208 F F F 5 100-119 118 118 118 216 216 216 F F F 6 120-139 120 120 120 220 220 220 F F F 7 140-159 116 116 116 212 212 212 F F F 8 160-179 121 121 121 222 222 222 F F F 9 180-199 115 115 115 209 209 209 F F F 10 200-219 116 116 116 213 213 213 F F F 11 220-239 121 121 121 222 222 222 F F F 12 240-259 120 120 120 220 220 220 F F F 13 260-279 106 106 106 195 195 195 F F F 14 280-299 112 112 112 203 203 203 F F F 15 300-319 120 120 120 221 221 221 F F F 16 320-339 121 121 121 222 222 222 F F F 17 340-359 118 118 118 216 216 216 F F F 18 360-379 120 120 120 219 219 219 F F F 19 380-399 108 108 108 197 197 197 F F F 20 400-419 115 115 115 209 209 209 F F F 21 420-439 105 105 105 194 194 194 F F F 22 440-459 116 116 116 213 213 213 F F F 23 460-479 121 121 121 222 222 222 F F F 24 480-499 118 118 118 215 215 215 F F F 25 500-519 119 119 119 217 217 217 F F F 26 520-539 117 117 117 214 214 214 F F F 27 540-559 118 118 118 216 216 216 F F F 28 560-579 120 120 120 220 220 220 F F F 29 580-599 121 121 121 222 222 222 F F F Max gen: 121, Max flush: 222 Begin loop 61 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:52.261680469Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1670,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:52.306352869Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1670,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:52.307030365Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1670} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"abd5430f1508af0bc008143ea2566277ef714ec2\",\n git_commit_timestamp: \"2025-01-17T20:55:03.000000000Z\",\n git_branch: \"alan/tokio-to-140\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:52.307076211Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1670} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:52.307093361Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1670} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:52.307111382Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1670} {"msg":"Crucible 03be0674-8a92-4b6c-a6a9-2bd15dddcfe2 has session id: d8774a30-4c49-4dec-b3f1-83180a236b74","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:52.307667657Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1670,"session_id":"d8774a30-4c49-4dec-b3f1-83180a236b74"} {"msg":"Upstairs opts: Upstairs UUID: 03be0674-8a92-4b6c-a6a9-2bd15dddcfe2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:52.30770628Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1670,"session_id":"d8774a30-4c49-4dec-b3f1-83180a236b74"} {"msg":"Crucible stats registered with UUID: 03be0674-8a92-4b6c-a6a9-2bd15dddcfe2","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:52.307732465Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1670,"session_id":"d8774a30-4c49-4dec-b3f1-83180a236b74"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:15:52.307746633Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1670,"session_id":"d8774a30-4c49-4dec-b3f1-83180a236b74"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:52.308336848Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1670,"task":"crutest"} {"msg":"The guest has requested activation with gen:122","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:52.308370338Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1670} {"msg":"03be0674-8a92-4b6c-a6a9-2bd15dddcfe2 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:52.308424609Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1670,"session_id":"d8774a30-4c49-4dec-b3f1-83180a236b74"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:52.308451155Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1670,"client":"0","":"downstairs","session_id":"d8774a30-4c49-4dec-b3f1-83180a236b74"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:52.30847671Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1670,"client":"0","":"downstairs","session_id":"d8774a30-4c49-4dec-b3f1-83180a236b74"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:52.308490878Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1670,"client":"1","":"downstairs","session_id":"d8774a30-4c49-4dec-b3f1-83180a236b74"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:52.308505467Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1670,"client":"1","":"downstairs","session_id":"d8774a30-4c49-4dec-b3f1-83180a236b74"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:52.308518544Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1670,"client":"2","":"downstairs","session_id":"d8774a30-4c49-4dec-b3f1-83180a236b74"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:52.308532272Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1670,"client":"2","":"downstairs","session_id":"d8774a30-4c49-4dec-b3f1-83180a236b74"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:52.30854485Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1670,"":"io task","client":"2","":"downstairs","session_id":"d8774a30-4c49-4dec-b3f1-83180a236b74"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:52.308558187Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1670,"":"io task","client":"0","":"downstairs","session_id":"d8774a30-4c49-4dec-b3f1-83180a236b74"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:52.308571715Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1670,"":"io task","client":"1","":"downstairs","session_id":"d8774a30-4c49-4dec-b3f1-83180a236b74"} {"msg":"ds_connection connected from Ok(127.0.0.1:41384)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:52.308591407Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1670,"":"io task","client":"2","":"downstairs","session_id":"d8774a30-4c49-4dec-b3f1-83180a236b74"} {"msg":"ds_connection connected from Ok(127.0.0.1:55649)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:52.308674385Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1670,"":"io task","client":"1","":"downstairs","session_id":"d8774a30-4c49-4dec-b3f1-83180a236b74"} {"msg":"ds_connection connected from Ok(127.0.0.1:39930)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:52.308714719Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1670,"":"io task","client":"0","":"downstairs","session_id":"d8774a30-4c49-4dec-b3f1-83180a236b74"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:52.309429296Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1670,"client":"2","":"downstairs","session_id":"d8774a30-4c49-4dec-b3f1-83180a236b74"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:52.309457303Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1670,"client":"0","":"downstairs","session_id":"d8774a30-4c49-4dec-b3f1-83180a236b74"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:52.309472622Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1670,"client":"1","":"downstairs","session_id":"d8774a30-4c49-4dec-b3f1-83180a236b74"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:52.310012277Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1670,"client":"2","":"downstairs","session_id":"d8774a30-4c49-4dec-b3f1-83180a236b74"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:52.310037482Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1670,"client":"1","":"downstairs","session_id":"d8774a30-4c49-4dec-b3f1-83180a236b74"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:52.310096417Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1670,"client":"0","":"downstairs","session_id":"d8774a30-4c49-4dec-b3f1-83180a236b74"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:52.310538255Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1670,"session_id":"d8774a30-4c49-4dec-b3f1-83180a236b74"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:52.31065003Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1670,"session_id":"d8774a30-4c49-4dec-b3f1-83180a236b74"} {"msg":"[0]R flush_numbers[0..12]: [210, 216, 219, 219, 208, 216, 220, 212, 222, 209, 213, 222]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:52.31078647Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1670,"":"downstairs","":"downstairs","session_id":"d8774a30-4c49-4dec-b3f1-83180a236b74"} {"msg":"[0]R generation[0..12]: [116, 118, 120, 120, 114, 118, 120, 116, 121, 115, 116, 121]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:52.310806661Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1670,"":"downstairs","":"downstairs","session_id":"d8774a30-4c49-4dec-b3f1-83180a236b74"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:52.31082091Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1670,"":"downstairs","":"downstairs","session_id":"d8774a30-4c49-4dec-b3f1-83180a236b74"} {"msg":"[1]R flush_numbers[0..12]: [210, 216, 219, 219, 208, 216, 220, 212, 222, 209, 213, 222]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:52.310834728Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1670,"":"downstairs","":"downstairs","session_id":"d8774a30-4c49-4dec-b3f1-83180a236b74"} {"msg":"[1]R generation[0..12]: [116, 118, 120, 120, 114, 118, 120, 116, 121, 115, 116, 121]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:52.310848166Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1670,"":"downstairs","":"downstairs","session_id":"d8774a30-4c49-4dec-b3f1-83180a236b74"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:52.310864836Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1670,"":"downstairs","":"downstairs","session_id":"d8774a30-4c49-4dec-b3f1-83180a236b74"} {"msg":"[2]R flush_numbers[0..12]: [210, 216, 219, 219, 208, 216, 220, 212, 222, 209, 213, 222]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:52.310878614Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1670,"":"downstairs","":"downstairs","session_id":"d8774a30-4c49-4dec-b3f1-83180a236b74"} {"msg":"[2]R generation[0..12]: [116, 118, 120, 120, 114, 118, 120, 116, 121, 115, 116, 121]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:52.310892122Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1670,"":"downstairs","":"downstairs","session_id":"d8774a30-4c49-4dec-b3f1-83180a236b74"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:52.31090623Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1670,"":"downstairs","":"downstairs","session_id":"d8774a30-4c49-4dec-b3f1-83180a236b74"} {"msg":"Max found gen is 122","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:52.310918817Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1670,"":"downstairs","":"downstairs","session_id":"d8774a30-4c49-4dec-b3f1-83180a236b74"} {"msg":"Generation requested: 122 >= found:122","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:52.310931385Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1670,"":"downstairs","":"downstairs","session_id":"d8774a30-4c49-4dec-b3f1-83180a236b74"} {"msg":"Next flush: 223","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:52.310943862Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1670,"":"downstairs","":"downstairs","session_id":"d8774a30-4c49-4dec-b3f1-83180a236b74"} {Wait for a query_work_queue command to finish before sending IO "msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:52.310965064Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1670,"":"downstairs","":"downstairs","session_id":"d8774a30-4c49-4dec-b3f1-83180a236b74"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:52.310988638Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1670,"session_id":"d8774a30-4c49-4dec-b3f1-83180a236b74"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:52.311011781Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1670,"client":"0","":"downstairs","session_id":"d8774a30-4c49-4dec-b3f1-83180a236b74"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:52.311037797Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1670,"client":"1","":"downstairs","session_id":"d8774a30-4c49-4dec-b3f1-83180a236b74"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:52.311063992Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1670,"client":"2","":"downstairs","session_id":"d8774a30-4c49-4dec-b3f1-83180a236b74"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:52.311090768Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1670,"session_id":"d8774a30-4c49-4dec-b3f1-83180a236b74"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:52.311115172Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1670,"session_id":"d8774a30-4c49-4dec-b3f1-83180a236b74"} {"msg":"03be0674-8a92-4b6c-a6a9-2bd15dddcfe2 is now active with session: d8774a30-4c49-4dec-b3f1-83180a236b74","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:52.311141267Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1670,"session_id":"d8774a30-4c49-4dec-b3f1-83180a236b74"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:52.311171255Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1670,"session_id":"d8774a30-4c49-4dec-b3f1-83180a236b74"} {"msg":"The guest has finished waiting for activation with:122","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:52.31119733Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1670} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 56 len 40960 data: 13 13 13 12 10 10 7 5 5 5 02/30 Read block 119 len 8192 03/30 Write block 480 len 24576 data: 9 8 7 6 7 7 04/30 Write block 22 len 36864 data: 8 7 7 8 10 11 9 8 12 05/30 Flush 06/30 Write block 26 len 24576 data: 11 12 10 9 13 10 07/30 Read block 297 len 24576 08/30 Read block 569 len 16384 09/30 Read block 497 len 36864 10/30 Read block 289 len 4096 11/30 Read block 37 len 28672 12/30 Flush 13/30 Write block 101 len 16384 data: 12 12 12 12 14/30 Read block 268 len 28672 15/30 Write block 588 len 28672 data: 9 9 8 5 7 6 5 16/30 Read block 248 len 12288 17/30 Flush 18/30 Write block 531 len 36864 data: 8 9 9 8 8 7 9 7 8 19/30 Write block 464 len 36864 data: 13 13 11 13 11 12 10 9 11 20/30 Write block 508 len 28672 data: 14 16 14 12 12 13 15 21/30 Read block 75 len 20480 22/30 Read block 274 len 24576 23/30 Flush 24/30 Write block 294 len 24576 data: 11 13 12 10 9 11 25/30 Read block 481 len 36864 26/30 Read block 112 len 16384 27/30 Write block 355 len 12288 data: 12 14 15 28/30 Read block 440 len 40960 29/30 Write block 316 len 24576 data: 6 7 8 9 10 9 30/30 Read block 192 len 12288 ---------------------------------------------------------------- Crucible gen:122 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 6 Done Done Done false 1024 Acked Read 9 Done Done Done false 1025 Acked Read 4 Done Done Done false 1026 Acked Write 3 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Write 6 Done Done Done false 1029 Acked Read 3 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 7 7 7 21 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:15:52.332096955Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1670} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 116 116 116 210 210 210 F F F 1 020-039 122 122 122 224 224 224 F F F 2 040-059 122 122 122 223 223 223 F F F 3 060-079 122 122 122 223 223 223 F F F 4 080-099 114 114 114 208 208 208 F F F 5 100-119 122 122 122 225 225 225 F F F 6 120-139 120 120 120 220 220 220 F F F 7 140-159 116 116 116 212 212 212 F F F 8 160-179 121 121 121 222 222 222 F F F 9 180-199 115 115 115 209 209 209 F F F 10 200-219 116 116 116 213 213 213 F F F 11 220-239 121 121 121 222 222 222 F F F 12 240-259 120 120 120 220 220 220 F F F 13 260-279 106 106 106 195 195 195 F F F 14 280-299 112 112 112 203 203 203 T T T 15 300-319 120 120 120 221 221 221 T T T 16 320-339 121 121 121 222 222 222 T T T 17 340-359 118 118 118 216 216 216 T T T 18 360-379 120 120 120 219 219 219 F F F 19 380-399 108 108 108 197 197 197 F F F 20 400-419 115 115 115 209 209 209 F F F 21 420-439 105 105 105 194 194 194 F F F 22 440-459 116 116 116 213 213 213 F F F 23 460-479 122 122 122 226 226 226 F F F 24 480-499 122 122 122 223 223 223 F F F 25 500-519 122 122 122 226 226 226 F F F 26 520-539 122 122 122 226 226 226 F F F 27 540-559 118 118 118 216 216 216 F F F 28 560-579 120 120 120 220 220 220 F F F 29 580-599 122 122 122 225 225 225 F F F Max gen: 122, Max flush: 226 A Difference in extent metadata is expected here On loop 61 Verifying data now No change after verify Loop: 61 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 116 116 116 210 210 210 F F F 1 020-039 122 122 122 224 224 224 F F F 2 040-059 122 122 122 223 223 223 F F F 3 060-079 122 122 122 223 223 223 F F F 4 080-099 114 114 114 208 208 208 F F F 5 100-119 122 122 122 225 225 225 F F F 6 120-139 120 120 120 220 220 220 F F F 7 140-159 116 116 116 212 212 212 F F F 8 160-179 121 121 121 222 222 222 F F F 9 180-199 115 115 115 209 209 209 F F F 10 200-219 116 116 116 213 213 213 F F F 11 220-239 121 121 121 222 222 222 F F F 12 240-259 120 120 120 220 220 220 F F F 13 260-279 106 106 106 195 195 195 F F F 14 280-299 123 123 123 227 227 227 F F F 15 300-319 123 123 123 227 227 227 F F F 16 320-339 123 123 123 227 227 227 F F F 17 340-359 123 123 123 227 227 227 F F F 18 360-379 120 120 120 219 219 219 F F F 19 380-399 108 108 108 197 197 197 F F F 20 400-419 115 115 115 209 209 209 F F F 21 420-439 105 105 105 194 194 194 F F F 22 440-459 116 116 116 213 213 213 F F F 23 460-479 122 122 122 226 226 226 F F F 24 480-499 122 122 122 223 223 223 F F F 25 500-519 122 122 122 226 226 226 F F F 26 520-539 122 122 122 226 226 226 F F F 27 540-559 118 118 118 216 216 216 F F F 28 560-579 120 120 120 220 220 220 F F F 29 580-599 122 122 122 225 225 225 F F F Max gen: 123, Max flush: 227 Begin loop 62 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:56.568059654Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1678,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:56.614183592Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1678,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:56.614835533Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1678} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"abd5430f1508af0bc008143ea2566277ef714ec2\",\n git_commit_timestamp: \"2025-01-17T20:55:03.000000000Z\",\n git_branch: \"alan/tokio-to-140\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:56.614875786Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1678} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:56.614893757Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1678} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:56.614908605Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1678} {"msg":"Crucible dccd941a-1e9f-4f84-9cd4-c87f93eb166b has session id: cfc96896-7eea-49de-b7f8-ac664c14a25c","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:56.615451893Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1678,"session_id":"cfc96896-7eea-49de-b7f8-ac664c14a25c"} {"msg":"Upstairs opts: Upstairs UUID: dccd941a-1e9f-4f84-9cd4-c87f93eb166b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:56.615501162Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1678,"session_id":"cfc96896-7eea-49de-b7f8-ac664c14a25c"} {"msg":"Crucible stats registered with UUID: dccd941a-1e9f-4f84-9cd4-c87f93eb166b","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:56.615523024Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1678,"session_id":"cfc96896-7eea-49de-b7f8-ac664c14a25c"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:15:56.615536032Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1678,"session_id":"cfc96896-7eea-49de-b7f8-ac664c14a25c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:56.615981292Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1678,"task":"crutest"} {"msg":"The guest has requested activation with gen:124","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:56.616020295Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1678} {"msg":"dccd941a-1e9f-4f84-9cd4-c87f93eb166b active request set","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:56.616085153Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1678,"session_id":"cfc96896-7eea-49de-b7f8-ac664c14a25c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:56.616102974Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1678,"client":"0","":"downstairs","session_id":"cfc96896-7eea-49de-b7f8-ac664c14a25c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:56.616127468Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1678,"client":"0","":"downstairs","session_id":"cfc96896-7eea-49de-b7f8-ac664c14a25c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:56.616141436Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1678,"client":"1","":"downstairs","session_id":"cfc96896-7eea-49de-b7f8-ac664c14a25c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:56.61616549Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1678,"client":"1","":"downstairs","session_id":"cfc96896-7eea-49de-b7f8-ac664c14a25c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:56.616259275Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1678,"client":"2","":"downstairs","session_id":"cfc96896-7eea-49de-b7f8-ac664c14a25c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:56.616494042Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1678,"client":"2","":"downstairs","session_id":"cfc96896-7eea-49de-b7f8-ac664c14a25c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:56.616533235Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1678,"":"io task","client":"2","":"downstairs","session_id":"cfc96896-7eea-49de-b7f8-ac664c14a25c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:56.616557949Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1678,"":"io task","client":"0","":"downstairs","session_id":"cfc96896-7eea-49de-b7f8-ac664c14a25c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:56.616583294Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1678,"":"io task","client":"1","":"downstairs","session_id":"cfc96896-7eea-49de-b7f8-ac664c14a25c"} {"msg":"ds_connection connected from Ok(127.0.0.1:55832)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:56.616608049Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1678,"":"io task","client":"2","":"downstairs","session_id":"cfc96896-7eea-49de-b7f8-ac664c14a25c"} {"msg":"ds_connection connected from Ok(127.0.0.1:37005)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:56.616623027Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1678,"":"io task","client":"0","":"downstairs","session_id":"cfc96896-7eea-49de-b7f8-ac664c14a25c"} {"msg":"ds_connection connected from Ok(127.0.0.1:54115)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:56.616636646Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1678,"":"io task","client":"1","":"downstairs","session_id":"cfc96896-7eea-49de-b7f8-ac664c14a25c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:56.61705438Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1678,"client":"2","":"downstairs","session_id":"cfc96896-7eea-49de-b7f8-ac664c14a25c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:56.617217275Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1678,"client":"0","":"downstairs","session_id":"cfc96896-7eea-49de-b7f8-ac664c14a25c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:56.617371675Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1678,"client":"1","":"downstairs","session_id":"cfc96896-7eea-49de-b7f8-ac664c14a25c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:56.617689441Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1678,"client":"0","":"downstairs","session_id":"cfc96896-7eea-49de-b7f8-ac664c14a25c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:56.617717367Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1678,"client":"2","":"downstairs","session_id":"cfc96896-7eea-49de-b7f8-ac664c14a25c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:56.617983913Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1678,"client":"1","":"downstairs","session_id":"cfc96896-7eea-49de-b7f8-ac664c14a25c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:56.618225474Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1678,"session_id":"cfc96896-7eea-49de-b7f8-ac664c14a25c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:56.618266538Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1678,"session_id":"cfc96896-7eea-49de-b7f8-ac664c14a25c"} {"msg":"[0]R flush_numbers[0..12]: [210, 224, 223, 223, 208, 225, 220, 212, 222, 209, 213, 222]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:56.61850895Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1678,"":"downstairs","":"downstairs","session_id":"cfc96896-7eea-49de-b7f8-ac664c14a25c"} {"msg":"[0]R generation[0..12]: [116, 122, 122, 122, 114, 122, 120, 116, 121, 115, 116, 121]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:56.618528881Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1678,"":"downstairs","":"downstairs","session_id":"cfc96896-7eea-49de-b7f8-ac664c14a25c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:56.61854324Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1678,"":"downstairs","":"downstairs","session_id":"cfc96896-7eea-49de-b7f8-ac664c14a25c"} {"msg":"[1]R flush_numbers[0..12]: [210, 224, 223, 223, 208, 225, 220, 212, 222, 209, 213, 222]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:56.618556758Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1678,"":"downstairs","":"downstairs","session_id":"cfc96896-7eea-49de-b7f8-ac664c14a25c"} {"msg":"[1]R generation[0..12]: [116, 122, 122, 122, 114, 122, 120, 116, 121, 115, 116, 121]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:56.618570165Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1678,"":"downstairs","":"downstairs","session_id":"cfc96896-7eea-49de-b7f8-ac664c14a25c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:56.618584044Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1678,"":"downstairs","":"downstairs","session_id":"cfc96896-7eea-49de-b7f8-ac664c14a25c"} {"msg":"[2]R flush_numbers[0..12]: [210, 224, 223, 223, 208, 225, 220, 212, 222, 209, 213, 222]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:56.618597612Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1678,"":"downstairs","":"downstairs","session_id":"cfc96896-7eea-49de-b7f8-ac664c14a25c"} {"msg":"[2]R generation[0..12]: [116, 122, 122, 122, 114, 122, 120, 116, 121, 115, 116, 121]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:56.618610949Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1678,"":"downstairs","":"downstairs","session_id":"cfc96896-7eea-49de-b7f8-ac664c14a25c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:56.618624257Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1678,"":"downstairs","":"downstairs","session_id":"cfc96896-7eea-49de-b7f8-ac664c14a25c"} {"msg":"Max found gen is 124","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:56.618637025Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1678,"":"downstairs","":"downstairs","session_id":"cfc96896-7eea-49de-b7f8-ac664c14a25c"} {"msg":"Generation requested: 124 >= found:124","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:56.618649822Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1678,"":"downstairs","":"downstairs","session_id":"cfc96896-7eea-49de-b7f8-ac664c14a25c"} {"msg":"Next flush: 228","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:56.618662309Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1678,"":"downstairs","":"downstairs","session_id":"cfc96896-7eea-49de-b7f8-ac664c14a25c"} Wait for a query_work_queue command to finish before sending IO {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:56.618682501Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1678,"":"downstairs","":"downstairs","session_id":"cfc96896-7eea-49de-b7f8-ac664c14a25c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:56.618706025Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1678,"session_id":"cfc96896-7eea-49de-b7f8-ac664c14a25c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:56.61873198Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1678,"client":"0","":"downstairs","session_id":"cfc96896-7eea-49de-b7f8-ac664c14a25c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:56.618758005Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1678,"client":"1","":"downstairs","session_id":"cfc96896-7eea-49de-b7f8-ac664c14a25c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:56.61878316Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1678,"client":"2","":"downstairs","session_id":"cfc96896-7eea-49de-b7f8-ac664c14a25c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:56.618809756Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1678,"session_id":"cfc96896-7eea-49de-b7f8-ac664c14a25c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:56.61883416Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1678,"session_id":"cfc96896-7eea-49de-b7f8-ac664c14a25c"} {"msg":"dccd941a-1e9f-4f84-9cd4-c87f93eb166b is now active with session: cfc96896-7eea-49de-b7f8-ac664c14a25c","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:56.618859905Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1678,"session_id":"cfc96896-7eea-49de-b7f8-ac664c14a25c"} {"msg":"Set Active after no reconciliation","v":Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 0,"name":"crucible","level":30,"time":"2025-01-17T21:15:56.618886861Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1678,"session_id":"cfc96896-7eea-49de-b7f8-ac664c14a25c"} {"msg":"The guest has finished waiting for activation with:124","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:15:56.618911865Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1678} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 592 len 12288 data: 8 7 6 02/30 Read block 260 len 12288 03/30 Write block 118 len 32768 data: 13 11 10 7 6 6 3 6 04/30 Read block 425 len 8192 05/30 Read block 133 len 28672 06/30 Read block 534 len 36864 07/30 Read block 372 len 40960 08/30 Flush 09/30 Flush 10/30 Flush 11/30 Write block 89 len 16384 data: 9 12 11 13 12/30 Read block 228 len 4096 13/30 Write block 566 len 28672 data: 13 8 9 9 10 10 8 14/30 Read block 78 len 4096 15/30 Read block 344 len 36864 16/30 Write block 209 len 8192 data: 10 10 17/30 Read block 504 len 16384 18/30 Write block 508 len 40960 data: 15 17 15 13 13 14 16 18 17 14 19/30 Read block 93 len 12288 20/30 Flush 21/30 Write block 527 len 40960 data: 9 8 6 6 9 10 10 9 9 8 22/30 Write block 84 len 32768 data: 9 8 7 7 8 10 13 12 23/30 Write block 412 len 36864 data: 9 11 10 10 9 8 7 8 7 24/30 Read block 491 len 40960 25/30 Read block 103 len 8192 26/30 Read block 431 len 4096 27/30 Write block 73 len 36864 data: 9 10 13 14 11 11 10 9 10 28/30 Write block 474 len 20480 data: 10 11 10 8 7 29/30 Write block 155 len 40960 data: 10 10 11 10 10 9 11 10 8 10 30/30 Write block 287 len 28672 data: 7 10 10 11 13 14 13 ---------------------------------------------------------------- Crucible gen:124 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1020 Acked Write 10 Done Done Done false 1021 Acked Write 8 Done Done Done false 1022 Acked Write 9 Done Done Done false 1023 Acked Read 10 Done Done Done false 1024 Acked Read 2 Done Done Done false 1025 Acked Read 1 Done Done Done false 1026 Acked Write 9 Done Done Done false 1027 Acked Write 5 Done Done Done false 1028 Acked Write 10 Sent Sent Sent false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 8 8 8 24 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1019 1019 1019 Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:15:56.6390581Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1678} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 116 116 116 210 210 210 F F F 1 020-039 122 122 122 224 224 224 F F F 2 040-059 122 122 122 223 223 223 F F F 3 060-079 122 122 122 223 223 223 T T T 4 080-099 124 124 124 231 231 231 T T T 5 100-119 124 124 124 228 228 228 F F F 6 120-139 124 124 124 228 228 228 F F F 7 140-159 116 116 116 212 212 212 T T T 8 160-179 121 121 121 222 222 222 T T T 9 180-199 115 115 115 209 209 209 F F F 10 200-219 124 124 124 231 231 231 F F F 11 220-239 121 121 121 222 222 222 F F F 12 240-259 120 120 120 220 220 220 F F F 13 260-279 106 106 106 195 195 195 F F F 14 280-299 123 123 123 227 227 227 T T T 15 300-319 123 123 123 227 227 227 F F F 16 320-339 123 123 123 227 227 227 F F F 17 340-359 123 123 123 227 227 227 F F F 18 360-379 120 120 120 219 219 219 F F F 19 380-399 108 108 108 197 197 197 F F F 20 400-419 115 115 115 209 209 209 T T T 21 420-439 105 105 105 194 194 194 T T T 22 440-459 116 116 116 213 213 213 F F F 23 460-479 122 122 122 226 226 226 T T T 24 480-499 122 122 122 223 223 223 F F F 25 500-519 124 124 124 231 231 231 F F F 26 520-539 122 122 122 226 226 226 T T T 27 540-559 118 118 118 216 216 216 F F F 28 560-579 124 124 124 231 231 231 F F F 29 580-599 124 124 124 228 228 228 F F F Max gen: 124, Max flush: 231 A Difference in extent metadata is expected here On loop 62 Verifying data now No change after verify Loop: 62 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 116 116 116 210 210 210 F F F 1 020-039 122 122 122 224 224 224 F F F 2 040-059 122 122 122 223 223 223 F F F 3 060-079 125 125 125 232 232 232 F F F 4 080-099 125 125 125 232 232 232 F F F 5 100-119 124 124 124 228 228 228 F F F 6 120-139 124 124 124 228 228 228 F F F 7 140-159 125 125 125 232 232 232 F F F 8 160-179 125 125 125 232 232 232 F F F 9 180-199 115 115 115 209 209 209 F F F 10 200-219 124 124 124 231 231 231 F F F 11 220-239 121 121 121 222 222 222 F F F 12 240-259 120 120 120 220 220 220 F F F 13 260-279 106 106 106 195 195 195 F F F 14 280-299 125 125 125 232 232 232 F F F 15 300-319 123 123 123 227 227 227 F F F 16 320-339 123 123 123 227 227 227 F F F 17 340-359 123 123 123 227 227 227 F F F 18 360-379 120 120 120 219 219 219 F F F 19 380-399 108 108 108 197 197 197 F F F 20 400-419 125 125 125 232 232 232 F F F 21 420-439 125 125 125 232 232 232 F F F 22 440-459 116 116 116 213 213 213 F F F 23 460-479 125 125 125 232 232 232 F F F 24 480-499 122 122 122 223 223 223 F F F 25 500-519 124 124 124 231 231 231 F F F 26 520-539 125 125 125 232 232 232 F F F 27 540-559 118 118 118 216 216 216 F F F 28 560-579 124 124 124 231 231 231 F F F 29 580-599 124 124 124 228 228 228 F F F Max gen: 125, Max flush: 232 Begin loop 63 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:10.901175518Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1686,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:10.945700732Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1686,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:10.94631379Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1686} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"abd5430f1508af0bc008143ea2566277ef714ec2\",\n git_commit_timestamp: \"2025-01-17T20:55:03.000000000Z\",\n git_branch: \"alan/tokio-to-140\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:10.946355234Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1686} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:10.946370233Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1686} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:10.946384752Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1686} {"msg":"Crucible 1625b814-5fd2-47a3-805a-f801abf4dcb8 has session id: 554af49b-81e6-4d03-bd2d-e389231f8f61","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:10.946892378Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1686,"session_id":"554af49b-81e6-4d03-bd2d-e389231f8f61"} {"msg":"Upstairs opts: Upstairs UUID: 1625b814-5fd2-47a3-805a-f801abf4dcb8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:10.94693024Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1686,"session_id":"554af49b-81e6-4d03-bd2d-e389231f8f61"} {"msg":"Crucible stats registered with UUID: 1625b814-5fd2-47a3-805a-f801abf4dcb8","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:10.946954554Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1686,"session_id":"554af49b-81e6-4d03-bd2d-e389231f8f61"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:16:10.946974756Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1686,"session_id":"554af49b-81e6-4d03-bd2d-e389231f8f61"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:10.947486866Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1686,"task":"crutest"} {"msg":"The guest has requested activation with gen:126","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:10.947518824Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1686} {"msg":"1625b814-5fd2-47a3-805a-f801abf4dcb8 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:10.947582201Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1686,"session_id":"554af49b-81e6-4d03-bd2d-e389231f8f61"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:10.947609417Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1686,"client":"0","":"downstairs","session_id":"554af49b-81e6-4d03-bd2d-e389231f8f61"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:10.947634882Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1686,"client":"0","":"downstairs","session_id":"554af49b-81e6-4d03-bd2d-e389231f8f61"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:10.9476488Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1686,"client":"1","":"downstairs","session_id":"554af49b-81e6-4d03-bd2d-e389231f8f61"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:10.947671994Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1686,"client":"1","":"downstairs","session_id":"554af49b-81e6-4d03-bd2d-e389231f8f61"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:10.947695017Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1686,"client":"2","":"downstairs","session_id":"554af49b-81e6-4d03-bd2d-e389231f8f61"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:10.947720422Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1686,"client":"2","":"downstairs","session_id":"554af49b-81e6-4d03-bd2d-e389231f8f61"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:10.947743355Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1686,"":"io task","client":"2","":"downstairs","session_id":"554af49b-81e6-4d03-bd2d-e389231f8f61"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:10.947769641Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1686,"":"io task","client":"0","":"downstairs","session_id":"554af49b-81e6-4d03-bd2d-e389231f8f61"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:10.947793425Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1686,"":"io task","client":"1","":"downstairs","session_id":"554af49b-81e6-4d03-bd2d-e389231f8f61"} {"msg":"ds_connection connected from Ok(127.0.0.1:59905)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:10.947817049Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1686,"":"io task","client":"0","":"downstairs","session_id":"554af49b-81e6-4d03-bd2d-e389231f8f61"} {"msg":"ds_connection connected from Ok(127.0.0.1:61329)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:10.947841283Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1686,"":"io task","client":"2","":"downstairs","session_id":"554af49b-81e6-4d03-bd2d-e389231f8f61"} {"msg":"ds_connection connected from Ok(127.0.0.1:42865)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:10.947866848Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1686,"":"io task","client":"1","":"downstairs","session_id":"554af49b-81e6-4d03-bd2d-e389231f8f61"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:10.948460134Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1686,"client":"2","":"downstairs","session_id":"554af49b-81e6-4d03-bd2d-e389231f8f61"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:10.948496566Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1686,"client":"0","":"downstairs","session_id":"554af49b-81e6-4d03-bd2d-e389231f8f61"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:10.948980719Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1686,"client":"2","":"downstairs","session_id":"554af49b-81e6-4d03-bd2d-e389231f8f61"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:10.949097567Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1686,"client":"0","":"downstairs","session_id":"554af49b-81e6-4d03-bd2d-e389231f8f61"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:10.949451474Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1686,"session_id":"554af49b-81e6-4d03-bd2d-e389231f8f61"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:10.94965043Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1686,"session_id":"554af49b-81e6-4d03-bd2d-e389231f8f61"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:10.950190716Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1686,"client":"1","":"downstairs","session_id":"554af49b-81e6-4d03-bd2d-e389231f8f61"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:10.95069403Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1686,"client":"1","":"downstairs","session_id":"554af49b-81e6-4d03-bd2d-e389231f8f61"} {"msg":"[0]R flush_numbers[0..12]: [210, 224, 223, 232, 232, 228, 228, 232, 232, 209, 231, 222]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:10.951193122Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1686,"":"downstairs","":"downstairs","session_id":"554af49b-81e6-4d03-bd2d-e389231f8f61"} {"msg":"[0]R generation[0..12]: [116, 122, 122, 125, 125, 124, 124, 125, 125, 115, 124, 121]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:10.951216476Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1686,"":"downstairs","":"downstairs","session_id":"554af49b-81e6-4d03-bd2d-e389231f8f61"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:10.951231434Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1686,"":"downstairs","":"downstairs","session_id":"554af49b-81e6-4d03-bd2d-e389231f8f61"} {"msg":"[1]R flush_numbers[0..12]: [210, 224, 223, 232, 232, 228, 228, 232, 232, 209, 231, 222]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:10.951245413Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1686,"":"downstairs","":"downstairs","session_id":"554af49b-81e6-4d03-bd2d-e389231f8f61"} {"msg":"[1]R generation[0..12]: [116, 122, 122, 125, 125, 124, 124, 125, 125, 115, 124, 121]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:10.95125886Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1686,"":"downstairs","":"downstairs","session_id":"554af49b-81e6-4d03-bd2d-e389231f8f61"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:10.951272628Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1686,"":"downstairs","":"downstairs","session_id":"554af49b-81e6-4d03-bd2d-e389231f8f61"} {"msg":"[2]R flush_numbers[0..12]: [210, 224, 223, 232, 232, 228, 228, 232, 232, 209, 231, 222]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:10.951285966Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1686,"":"downstairs","":"downstairs","session_id":"554af49b-81e6-4d03-bd2d-e389231f8f61"} {"msg":"[2]R generation[0..12]: [116, 122, 122, 125, 125, 124, 124, 125, 125, 115, 124, 121]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:10.951299374Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1686,"":"downstairs","":"downstairs","session_id":"554af49b-81e6-4d03-bd2d-e389231f8f61"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:10.951315743Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1686,"":"downstairs","":"downstairs","session_id":"554af49b-81e6-4d03-bd2d-e389231f8f61"} {"msg":"Max found gen is 126","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:10.951328781Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1686,"":"downstairs","":"downstairs","session_id":"554af49b-81e6-4d03-bd2d-e389231f8f61"} {"msg":"Generation requested: 126 >= found:126","v":0,"name":"crucible","level":30,"Wait for a query_work_queue command to finish before sending IO time":"2025-01-17T21:16:10.951341829Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1686,"":"downstairs","":"downstairs","session_id":"554af49b-81e6-4d03-bd2d-e389231f8f61"} {"msg":"Next flush: 233","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:10.95136088Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1686,"":"downstairs","":"downstairs","session_id":"554af49b-81e6-4d03-bd2d-e389231f8f61"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:10.951373497Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1686,"":"downstairs","":"downstairs","session_id":"554af49b-81e6-4d03-bd2d-e389231f8f61"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:10.951386305Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1686,"session_id":"554af49b-81e6-4d03-bd2d-e389231f8f61"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:10.951397791Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1686,"client":"0","":"downstairs","session_id":"554af49b-81e6-4d03-bd2d-e389231f8f61"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:10.951410689Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1686,"client":"1","":"downstairs","session_id":"554af49b-81e6-4d03-bd2d-e389231f8f61"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:10.951423397Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1686,"client":"2","":"downstairs","session_id":"554af49b-81e6-4d03-bd2d-e389231f8f61"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:10.951436084Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1686,"session_id":"554af49b-81e6-4d03-bd2d-e389231f8f61"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:10.951448001Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1686,"session_id":"554af49b-81e6-4d03-bd2d-e389231f8f61"} {"msg":"1625b814-5fd2-47a3-805a-f801abf4dcb8 is now active with session: 554af49b-81e6-4d03-bd2d-e389231f8f61","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:10.951460388Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1686,"session_id":"554af49b-81e6-4d03-bd2d-e389231f8f61"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:10.951471915Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1686,"session_id":"554af49b-81e6-4d03-bd2d-e389231f8f61"} {"msg":"The guest has finished waiting for activation with:126","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:10.951483552Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1686} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 341 len 12288 data: 14 11 14 02/30 Read block 425 len 32768 03/30 Write block 467 len 20480 data: 14 12 13 11 10 04/30 Write block 519 len 8192 data: 13 10 05/30 Read block 276 len 24576 06/30 Read block 121 len 16384 07/30 Read block 28 len 32768 08/30 Read block 331 len 32768 09/30 Read block 218 len 24576 10/30 Write block 108 len 20480 data: 8 9 10 13 14 11/30 Read block 506 len 28672 12/30 Flush 13/30 Write block 226 len 28672 data: 9 10 11 11 12 11 10 14/30 Read block 199 len 24576 15/30 Write block 598 len 8192 data: 5 3 16/30 Write block 425 len 4096 data: 8 17/30 Read block 539 len 40960 18/30 Write block 63 len 24576 data: 6 6 6 6 6 7 19/30 Write block 43 len 8192 data: 6 7 20/30 Write block 538 len 24576 data: 8 9 9 9 9 11 21/30 Write block 501 len 28672 data: 16 13 13 12 14 12 14 22/30 Read block 387 len 4096 23/30 Write block 180 len 12288 data: 8 8 7 24/30 Read block 89 len 36864 25/30 Read block 441 len 16384 26/30 Read block 395 len 24576 27/30 Read block 424 len 32768 28/30 Write block 432 len 24576 data: 10 7 9 7 6 7 29/30 Read block 264 len 12288 30/30 Read block 413 len 12288 ---------------------------------------------------------------- Crucible gen:126 GIO:true work queues: Upstairs:0 downstairs:18 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1012 Acked Write 7 Done Done Done false 1013 Acked Read 6 Done Done Done false 1014 Acked Write 2 Done Done Done false 1015 Acked Write 1 Done Done Done false 1016 Acked Read 10 Done Done Done false 1017 Acked Write 6 Done Done Done false 1018 Acked Write 2 Done Done Done false 1019 Acked Write 6 Done Done Done false 1020 Acked Write 7 Done Done Done false 1021 Acked Read 1 Done Done Done false 1022 Acked Write 3 Done Done Done false 1023 Acked Read 9 Done Done Done false 1024 Acked Read 4 Done Done Done false 1025 Acked Read 6 Done Done Done false 1026 Acked Read 8 Done Done Done false 1027 Acked Write 6 Done Done Done false 1028 Acked Read 3 Done Done Done false 1029 Acked Read 3 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 18 17 17 52 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1011 1011 1011 Downstairs last five completed: 1011 1010 1009 1008 1007 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:16:10.970529733Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1686} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 116 116 116 210 210 210 F F F 1 020-039 122 122 122 224 224 224 F F F 2 040-059 122 122 122 223 223 223 T T T 3 060-079 125 125 125 232 232 232 T T T 4 080-099 125 125 125 232 232 232 F F F 5 100-119 126 126 126 233 233 233 F F F 6 120-139 124 124 124 228 228 228 F F F 7 140-159 125 125 125 232 232 232 F F F 8 160-179 125 125 125 232 232 232 F F F 9 180-199 115 115 115 209 209 209 T T T 10 200-219 124 124 124 231 231 231 F F F 11 220-239 121 121 121 222 222 222 T T T 12 240-259 120 120 120 220 220 220 F F F 13 260-279 106 106 106 195 195 195 F F F 14 280-299 125 125 125 232 232 232 F F F 15 300-319 123 123 123 227 227 227 F F F 16 320-339 123 123 123 227 227 227 F F F 17 340-359 126 126 126 233 233 233 F F F 18 360-379 120 120 120 219 219 219 F F F 19 380-399 108 108 108 197 197 197 F F F 20 400-419 125 125 125 232 232 232 F F F 21 420-439 125 125 125 232 232 232 T T T 22 440-459 116 116 116 213 213 213 F F F 23 460-479 126 126 126 233 233 233 F F F 24 480-499 122 122 122 223 223 223 F F F 25 500-519 126 126 126 233 233 233 T T T 26 520-539 126 126 126 233 233 233 T T T 27 540-559 118 118 118 216 216 216 T T T 28 560-579 124 124 124 231 231 231 F F F 29 580-599 124 124 124 228 228 228 T T T Max gen: 126, Max flush: 233 A Difference in extent metadata is expected here On loop 63 Verifying data now No change after verify Loop: 63 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 116 116 116 210 210 210 F F F 1 020-039 122 122 122 224 224 224 F F F 2 040-059 127 127 127 234 234 234 F F F 3 060-079 127 127 127 234 234 234 F F F 4 080-099 125 125 125 232 232 232 F F F 5 100-119 126 126 126 233 233 233 F F F 6 120-139 124 124 124 228 228 228 F F F 7 140-159 125 125 125 232 232 232 F F F 8 160-179 125 125 125 232 232 232 F F F 9 180-199 127 127 127 234 234 234 F F F 10 200-219 124 124 124 231 231 231 F F F 11 220-239 127 127 127 234 234 234 F F F 12 240-259 120 120 120 220 220 220 F F F 13 260-279 106 106 106 195 195 195 F F F 14 280-299 125 125 125 232 232 232 F F F 15 300-319 123 123 123 227 227 227 F F F 16 320-339 123 123 123 227 227 227 F F F 17 340-359 126 126 126 233 233 233 F F F 18 360-379 120 120 120 219 219 219 F F F 19 380-399 108 108 108 197 197 197 F F F 20 400-419 125 125 125 232 232 232 F F F 21 420-439 127 127 127 234 234 234 F F F 22 440-459 116 116 116 213 213 213 F F F 23 460-479 126 126 126 233 233 233 F F F 24 480-499 122 122 122 223 223 223 F F F 25 500-519 127 127 127 234 234 234 F F F 26 520-539 127 127 127 234 234 234 F F F 27 540-559 127 127 127 234 234 234 F F F 28 560-579 124 124 124 231 231 231 F F F 29 580-599 127 127 127 234 234 234 F F F Max gen: 127, Max flush: 234 Begin loop 64 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:25.23365415Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1694,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:25.278160002Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1694,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:25.278858311Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1694} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"abd5430f1508af0bc008143ea2566277ef714ec2\",\n git_commit_timestamp: \"2025-01-17T20:55:03.000000000Z\",\n git_branch: \"alan/tokio-to-140\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:25.27889248Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1694} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:25.278905898Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1694} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:25.278919736Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1694} {"msg":"Crucible a4b44a4b-980c-453b-9066-ef0e01b33c2e has session id: fe1e4a25-339c-4e0f-813c-8038add4cef9","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:25.2796449Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1694,"session_id":"fe1e4a25-339c-4e0f-813c-8038add4cef9"} {"msg":"Upstairs opts: Upstairs UUID: a4b44a4b-980c-453b-9066-ef0e01b33c2e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:25.279685184Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1694,"session_id":"fe1e4a25-339c-4e0f-813c-8038add4cef9"} {"msg":"Crucible stats registered with UUID: a4b44a4b-980c-453b-9066-ef0e01b33c2e","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:25.279708607Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1694,"session_id":"fe1e4a25-339c-4e0f-813c-8038add4cef9"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:16:25.279723616Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1694,"session_id":"fe1e4a25-339c-4e0f-813c-8038add4cef9"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:25.280098976Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1694,"task":"crutest"} {"msg":"The guest has requested activation with gen:128","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:25.280128553Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1694} {"msg":"a4b44a4b-980c-453b-9066-ef0e01b33c2e active request set","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:25.280191319Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1694,"session_id":"fe1e4a25-339c-4e0f-813c-8038add4cef9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:25.280217725Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1694,"client":"0","":"downstairs","session_id":"fe1e4a25-339c-4e0f-813c-8038add4cef9"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:25.28024389Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1694,"client":"0","":"downstairs","session_id":"fe1e4a25-339c-4e0f-813c-8038add4cef9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:25.280257869Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1694,"client":"1","":"downstairs","session_id":"fe1e4a25-339c-4e0f-813c-8038add4cef9"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:25.280271817Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1694,"client":"1","":"downstairs","session_id":"fe1e4a25-339c-4e0f-813c-8038add4cef9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:25.280284784Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1694,"client":"2","":"downstairs","session_id":"fe1e4a25-339c-4e0f-813c-8038add4cef9"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:25.280298362Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1694,"client":"2","":"downstairs","session_id":"fe1e4a25-339c-4e0f-813c-8038add4cef9"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:25.28031108Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1694,"":"io task","client":"2","":"downstairs","session_id":"fe1e4a25-339c-4e0f-813c-8038add4cef9"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:25.280332172Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1694,"":"io task","client":"0","":"downstairs","session_id":"fe1e4a25-339c-4e0f-813c-8038add4cef9"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:25.280358977Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1694,"":"io task","client":"1","":"downstairs","session_id":"fe1e4a25-339c-4e0f-813c-8038add4cef9"} {"msg":"ds_connection connected from Ok(127.0.0.1:52768)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:25.280383202Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1694,"":"io task","client":"2","":"downstairs","session_id":"fe1e4a25-339c-4e0f-813c-8038add4cef9"} {"msg":"ds_connection connected from Ok(127.0.0.1:46239)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:25.280406525Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1694,"":"io task","client":"1","":"downstairs","session_id":"fe1e4a25-339c-4e0f-813c-8038add4cef9"} {"msg":"ds_connection connected from Ok(127.0.0.1:63360)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:25.280603531Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1694,"":"io task","client":"0","":"downstairs","session_id":"fe1e4a25-339c-4e0f-813c-8038add4cef9"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:25.281109987Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1694,"client":"0","":"downstairs","session_id":"fe1e4a25-339c-4e0f-813c-8038add4cef9"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:25.281135731Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1694,"client":"1","":"downstairs","session_id":"fe1e4a25-339c-4e0f-813c-8038add4cef9"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:25.281256282Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1694,"client":"2","":"downstairs","session_id":"fe1e4a25-339c-4e0f-813c-8038add4cef9"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:25.281738574Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1694,"client":"0","":"downstairs","session_id":"fe1e4a25-339c-4e0f-813c-8038add4cef9"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:25.281775846Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1694,"client":"1","":"downstairs","session_id":"fe1e4a25-339c-4e0f-813c-8038add4cef9"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:25.282029954Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1694,"client":"2","":"downstairs","session_id":"fe1e4a25-339c-4e0f-813c-8038add4cef9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:25.282357986Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1694,"session_id":"fe1e4a25-339c-4e0f-813c-8038add4cef9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:25.282384261Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1694,"session_id":"fe1e4a25-339c-4e0f-813c-8038add4cef9"} {"msg":"[0]R flush_numbers[0..12]: [210, 224, 234, 234, 232, 233, 228, 232, 232, 234, 231, 234]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:25.282574052Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1694,"":"downstairs","":"downstairs","session_id":"fe1e4a25-339c-4e0f-813c-8038add4cef9"} {"msg":"[0]R generation[0..12]: [116, 122, 127, 127, 125, 126, 124, 125, 125, 127, 124, 127]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:25.282602839Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1694,"":"downstairs","":"downstairs","session_id":"fe1e4a25-339c-4e0f-813c-8038add4cef9"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:25.282623151Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1694,"":"downstairs","":"downstairs","session_id":"fe1e4a25-339c-4e0f-813c-8038add4cef9"} {"msg":"[1]R flush_numbers[0..12]: [210, 224, 234, 234, 232, 233, 228, 232, 232, 234, 231, 234]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:25.28263853Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1694,"":"downstairs","":"downstairs","session_id":"fe1e4a25-339c-4e0f-813c-8038add4cef9"} {"msg":"[1]R generation[0..12]: [116, 122, 127, 127, 125, 126, 124, 125, 125, 127, 124, 127]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:25.282653068Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1694,"":"downstairs","":"downstairs","session_id":"fe1e4a25-339c-4e0f-813c-8038add4cef9"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:25.282677633Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1694,"":"downstairs","":"downstairs","session_id":"fe1e4a25-339c-4e0f-813c-8038add4cef9"} {"msg":"[2]R flush_numbers[0..12]: [210, 224, 234, 234, 232, 233, 228, 232, 232, 234, 231, 234]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:25.282704478Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1694,"":"downstairs","":"downstairs","session_id":"fe1e4a25-339c-4e0f-813c-8038add4cef9"} {"msg":"[2]R generation[0..12]: [116, 122, 127, 127, 125, 126, 124, 125, 125, 127, 124, 127]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:25.282729933Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1694,"":"Wait for a query_work_queue command to finish before sending IO downstairs","":"downstairs","session_id":"fe1e4a25-339c-4e0f-813c-8038add4cef9"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:25.282761702Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1694,"":"downstairs","":"downstairs","session_id":"fe1e4a25-339c-4e0f-813c-8038add4cef9"} {"msg":"Max found gen is 128","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:25.282786836Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1694,"":"downstairs","":"downstairs","session_id":"fe1e4a25-339c-4e0f-813c-8038add4cef9"} {"msg":"Generation requested: 128 >= found:128","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:25.282805227Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1694,"":"downstairs","":"downstairs","session_id":"fe1e4a25-339c-4e0f-813c-8038add4cef9"} {"msg":"Next flush: 235","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:25.282819085Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1694,"":"downstairs","":"downstairs","session_id":"fe1e4a25-339c-4e0f-813c-8038add4cef9"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:25.282835235Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1694,"":"downstairs","":"downstairs","session_id":"fe1e4a25-339c-4e0f-813c-8038add4cef9"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:25.282857448Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1694,"session_id":"fe1e4a25-339c-4e0f-813c-8038add4cef9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:25.282880691Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1694,"client":"0","":"downstairs","session_id":"fe1e4a25-339c-4e0f-813c-8038add4cef9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:25.282897801Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1694,"client":"1","":"downstairs","session_id":"fe1e4a25-339c-4e0f-813c-8038add4cef9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:25.282922646Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1694,"client":"2","":"downstairs","session_id":"fe1e4a25-339c-4e0f-813c-8038add4cef9"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:25.282947541Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1694,"session_id":"fe1e4a25-339c-4e0f-813c-8038add4cef9"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:25.282970804Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1694,"session_id":"fe1e4a25-339c-4e0f-813c-8038add4cef9"} {"msg":"a4b44a4b-980c-453b-9066-ef0e01b33c2e is now active with session: fe1e4a25-339c-4e0f-813c-8038add4cef9","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:25.282987634Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1694,"session_id":"fe1e4a25-339c-4e0f-813c-8038add4cef9"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:25.283005274Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1694,"session_id":"fe1e4a25-339c-4e0f-813c-8038add4cef9"} {"msg":"The guest has finished waiting for activation with:128","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:25.283026187Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1694} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 510 len 28672 data: 16 14 14 15 17 19 18 02/30 Write block 154 len 8192 data: 9 11 03/30 Read block 425 len 16384 04/30 Read block 588 len 12288 05/30 Read block 548 len 40960 06/30 Read block 8 len 4096 07/30 Write block 48 len 16384 data: 9 9 9 11 08/30 Write block 129 len 12288 data: 8 9 8 09/30 Read block 492 len 8192 10/30 Read block 581 len 8192 11/30 Flush 12/30 Write block 520 len 28672 data: 11 11 8 14 12 10 11 13/30 Write block 79 len 8192 data: 11 10 14/30 Read block 598 len 4096 15/30 Flush 16/30 Write block 511 len 24576 data: 15 15 16 18 20 19 17/30 Read block 173 len 12288 18/30 Write block 111 len 32768 data: 14 15 12 13 13 11 12 14 19/30 Read block 387 len 16384 20/30 Write block 162 len 40960 data: 11 9 11 12 12 11 12 10 10 11 21/30 Read block 10 len 16384 22/30 Read block 405 len 8192 23/30 Write block 379 len 36864 data: 16 13 10 8 9 9 7 7 7 24/30 Read block 93 len 12288 25/30 Write block 436 len 32768 data: 7 8 6 4 4 6 5 7 26/30 Read block 451 len 36864 27/30 Read block 594 len 16384 28/30 Write block 481 len 24576 data: 9 8 7 8 8 8 29/30 Read block 273 len 4096 30/30 Write block 195 len 20480 data: 11 11 9 11 15 ---------------------------------------------------------------- Crucible gen:128 GIO:true work queues: Upstairs:0 downstairs:15 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1015 Acked Write 6 Done Done Done false 1016 Acked Read 3 Done Done Done false 1017 Acked Write 8 Done Done Done false 1018 Acked Read 4 Done Done Done false 1019 Acked Write 10 Done Done Done false 1020 Acked Read 4 Done Done Done false 1021 Acked Read 2 Done Done Done false 1022 Acked Write 9 Done Done Done false 1023 Acked Read 3 Done Done Done false 1024 Acked Write 8 Done Done Done false 1025 Acked Read 9 Done Done Done false 1026 Acked Read 4 Done Done Done false 1027 Acked Write 6 Done Done Done false 1028 Acked Read 1 Done Done Done false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 14 14 14 42 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1014 1014 1014 Downstairs last five completed: 1014 1013 1012 1011 1010 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:16:25.303002252Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1694} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 116 116 116 210 210 210 F F F 1 020-039 122 122 122 224 224 224 F F F 2 040-059 128 128 128 235 235 235 F F F 3 060-079 128 128 128 236 236 236 F F F 4 080-099 128 128 128 236 236 236 F F F 5 100-119 126 126 126 233 233 233 T T T 6 120-139 128 128 128 235 235 235 F F F 7 140-159 128 128 128 235 235 235 F F F 8 160-179 125 125 125 232 232 232 T T T 9 180-199 127 127 127 234 234 234 T T T 10 200-219 124 124 124 231 231 231 F F F 11 220-239 127 127 127 234 234 234 F F F 12 240-259 120 120 120 220 220 220 F F F 13 260-279 106 106 106 195 195 195 F F F 14 280-299 125 125 125 232 232 232 F F F 15 300-319 123 123 123 227 227 227 F F F 16 320-339 123 123 123 227 227 227 F F F 17 340-359 126 126 126 233 233 233 F F F 18 360-379 120 120 120 219 219 219 T T T 19 380-399 108 108 108 197 197 197 T T T 20 400-419 125 125 125 232 232 232 F F F 21 420-439 127 127 127 234 234 234 T T T 22 440-459 116 116 116 213 213 213 T T T 23 460-479 126 126 126 233 233 233 F F F 24 480-499 122 122 122 223 223 223 T T T 25 500-519 128 128 128 235 235 235 T T T 26 520-539 128 128 128 236 236 236 F F F 27 540-559 127 127 127 234 234 234 F F F 28 560-579 124 124 124 231 231 231 F F F 29 580-599 127 127 127 234 234 234 F F F Max gen: 128, Max flush: 236 A Difference in extent metadata is expected here On loop 64 Verifying data now No change after verify Loop: 64 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 116 116 116 210 210 210 F F F 1 020-039 122 122 122 224 224 224 F F F 2 040-059 128 128 128 235 235 235 F F F 3 060-079 128 128 128 236 236 236 F F F 4 080-099 128 128 128 236 236 236 F F F 5 100-119 129 129 129 237 237 237 F F F 6 120-139 128 128 128 235 235 235 F F F 7 140-159 128 128 128 235 235 235 F F F 8 160-179 129 129 129 237 237 237 F F F 9 180-199 129 129 129 237 237 237 F F F 10 200-219 124 124 124 231 231 231 F F F 11 220-239 127 127 127 234 234 234 F F F 12 240-259 120 120 120 220 220 220 F F F 13 260-279 106 106 106 195 195 195 F F F 14 280-299 125 125 125 232 232 232 F F F 15 300-319 123 123 123 227 227 227 F F F 16 320-339 123 123 123 227 227 227 F F F 17 340-359 126 126 126 233 233 233 F F F 18 360-379 129 129 129 237 237 237 F F F 19 380-399 129 129 129 237 237 237 F F F 20 400-419 125 125 125 232 232 232 F F F 21 420-439 129 129 129 237 237 237 F F F 22 440-459 129 129 129 237 237 237 F F F 23 460-479 126 126 126 233 233 233 F F F 24 480-499 129 129 129 237 237 237 F F F 25 500-519 129 129 129 237 237 237 F F F 26 520-539 128 128 128 236 236 236 F F F 27 540-559 127 127 127 234 234 234 F F F 28 560-579 124 124 124 231 231 231 F F F 29 580-599 127 127 127 234 234 234 F F F Max gen: 129, Max flush: 237 Begin loop 65 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:29.566656095Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1702,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:29.610753789Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1702,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:29.611394714Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1702} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"abd5430f1508af0bc008143ea2566277ef714ec2\",\n git_commit_timestamp: \"2025-01-17T20:55:03.000000000Z\",\n git_branch: \"alan/tokio-to-140\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:29.611436999Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1702} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:29.611451757Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1702} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:29.611466096Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1702} {"msg":"Crucible 4cc4c96c-9c42-47a0-b0d5-fea937925338 has session id: 686f391b-7584-4b0b-bb34-73b18df98915","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:29.611990682Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1702,"session_id":"686f391b-7584-4b0b-bb34-73b18df98915"} {"msg":"Upstairs opts: Upstairs UUID: 4cc4c96c-9c42-47a0-b0d5-fea937925338, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:29.612036259Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1702,"session_id":"686f391b-7584-4b0b-bb34-73b18df98915"} {"msg":"Crucible stats registered with UUID: 4cc4c96c-9c42-47a0-b0d5-fea937925338","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:29.612052628Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1702,"session_id":"686f391b-7584-4b0b-bb34-73b18df98915"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:16:29.612074601Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1702,"session_id":"686f391b-7584-4b0b-bb34-73b18df98915"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:29.612565248Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1702,"task":"crutest"} {"msg":"The guest has requested activation with gen:130","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:29.6125868Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1702} {"msg":"4cc4c96c-9c42-47a0-b0d5-fea937925338 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:29.612656872Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1702,"session_id":"686f391b-7584-4b0b-bb34-73b18df98915"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:29.612674452Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1702,"client":"0","":"downstairs","session_id":"686f391b-7584-4b0b-bb34-73b18df98915"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:29.612702198Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1702,"client":"0","":"downstairs","session_id":"686f391b-7584-4b0b-bb34-73b18df98915"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:29.612716396Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1702,"client":"1","":"downstairs","session_id":"686f391b-7584-4b0b-bb34-73b18df98915"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:29.612734097Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1702,"client":"1","":"downstairs","session_id":"686f391b-7584-4b0b-bb34-73b18df98915"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:29.612758741Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1702,"client":"2","":"downstairs","session_id":"686f391b-7584-4b0b-bb34-73b18df98915"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:29.612785027Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1702,"client":"2","":"downstairs","session_id":"686f391b-7584-4b0b-bb34-73b18df98915"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:29.612809161Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1702,"":"io task","client":"2","":"downstairs","session_id":"686f391b-7584-4b0b-bb34-73b18df98915"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:29.612834245Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1702,"":"io task","client":"0","":"downstairs","session_id":"686f391b-7584-4b0b-bb34-73b18df98915"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:29.612898423Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1702,"":"io task","client":"1","":"downstairs","session_id":"686f391b-7584-4b0b-bb34-73b18df98915"} {"msg":"ds_connection connected from Ok(127.0.0.1:44153)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:29.6129277Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1702,"":"io task","client":"0","":"downstairs","session_id":"686f391b-7584-4b0b-bb34-73b18df98915"} {"msg":"ds_connection connected from Ok(127.0.0.1:39311)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:29.612955566Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1702,"":"io task","client":"2","":"downstairs","session_id":"686f391b-7584-4b0b-bb34-73b18df98915"} {"msg":"ds_connection connected from Ok(127.0.0.1:50950)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:29.612984223Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1702,"":"io task","client":"1","":"downstairs","session_id":"686f391b-7584-4b0b-bb34-73b18df98915"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:29.613570866Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1702,"client":"1","":"downstairs","session_id":"686f391b-7584-4b0b-bb34-73b18df98915"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:29.613611439Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1702,"client":"0","":"downstairs","session_id":"686f391b-7584-4b0b-bb34-73b18df98915"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:29.614311288Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1702,"client":"1","":"downstairs","session_id":"686f391b-7584-4b0b-bb34-73b18df98915"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:29.614338054Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1702,"client":"0","":"downstairs","session_id":"686f391b-7584-4b0b-bb34-73b18df98915"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:29.614855677Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1702,"session_id":"686f391b-7584-4b0b-bb34-73b18df98915"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:29.614952173Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1702,"session_id":"686f391b-7584-4b0b-bb34-73b18df98915"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:29.615024776Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1702,"client":"2","":"downstairs","session_id":"686f391b-7584-4b0b-bb34-73b18df98915"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:29.615535794Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1702,"client":"2","":"downstairs","session_id":"686f391b-7584-4b0b-bb34-73b18df98915"} {"msg":"[0]R flush_numbers[0..12]: [210, 224, 235, 236, 236, 237, 235, 235, 237, 237, 231, 234]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:29.616021918Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1702,"":"downstairs","":"downstairs","session_id":"686f391b-7584-4b0b-bb34-73b18df98915"} {"msg":"[0]R generation[0..12]: [116, 122, 128, 128, 128, 129, 128, 128, 129, 129, 124, 127]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:29.61604257Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1702,"":"downstairs","":"downstairs","session_id":"686f391b-7584-4b0b-bb34-73b18df98915"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:29.616056719Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1702,"":"downstairs","":"downstairs","session_id":"686f391b-7584-4b0b-bb34-73b18df98915"} {"msg":"[1]R flush_numbers[0..12]: [210, 224, 235, 236, 236, 237, 235, 235, 237, 237, 231, 234]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:29.616070307Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1702,"":"downstairs","":"downstairs","session_id":"686f391b-7584-4b0b-bb34-73b18df98915"} {"msg":"[1]R generation[0..12]: [116, 122, 128, 128, 128, 129, 128, 128, 129, 129, 124, 127]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:29.616083724Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1702,"":"downstairs","":"downstairs","session_id":"686f391b-7584-4b0b-bb34-73b18df98915"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:29.616097483Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1702,"":"downstairs","":"downstairs","session_id":"686f391b-7584-4b0b-bb34-73b18df98915"} {"msg":"[2]R flush_numbers[0..12]: [210, 224, 235, 236, 236, 237, 235, 235, 237, 237, 231, 234]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:29.61611078Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1702,"":"downstairs","":"downstairs","session_id":"686f391b-7584-4b0b-bb34-73b18df98915"} {"msg":"[2]R generation[0..12]: [116, 122, 128, 128, 128, 129, 128, 128, 129, 129, 124, 127]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:29.616124188Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1702,"":"downstairs","":"downstairs","session_id":"686f391b-7584-4b0b-bb34-73b18df98915"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:29.616137696Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1702,"":"downstairs","":"downstairs","session_id":"686f391b-7584-4b0b-bb34-73b18df98915"} {"msg":"Max found gen is 130","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:29.616154286Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1702,"":"downstairs","":"downstairs","session_id":"686f391b-7584-4b0b-bb34-73b18df98915Wait for a query_work_queue command to finish before sending IO "} {"msg":"Generation requested: 130 >= found:130","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:29.616174337Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1702,"":"downstairs","":"downstairs","session_id":"686f391b-7584-4b0b-bb34-73b18df98915"} {"msg":"Next flush: 238","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:29.616187175Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1702,"":"downstairs","":"downstairs","session_id":"686f391b-7584-4b0b-bb34-73b18df98915"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:29.616199702Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1702,"":"downstairs","":"downstairs","session_id":"686f391b-7584-4b0b-bb34-73b18df98915"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:29.61621249Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1702,"session_id":"686f391b-7584-4b0b-bb34-73b18df98915"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:29.616224217Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1702,"client":"0","":"downstairs","session_id":"686f391b-7584-4b0b-bb34-73b18df98915"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:29.616237324Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1702,"client":"1","":"downstairs","session_id":"686f391b-7584-4b0b-bb34-73b18df98915"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:29.616250192Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1702,"client":"2","":"downstairs","session_id":"686f391b-7584-4b0b-bb34-73b18df98915"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:29.616262919Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1702,"session_id":"686f391b-7584-4b0b-bb34-73b18df98915"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:29.616274686Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1702,"session_id":"686f391b-7584-4b0b-bb34-73b18df98915"} {"msg":"4cc4c96c-9c42-47a0-b0d5-fea937925338 is now active with session: 686f391b-7584-4b0b-bb34-73b18df98915","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:29.616286983Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1702,"session_id":"686f391b-7584-4b0b-bb34-73b18df98915"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:29.61629833Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1702,"session_id":"686f391b-7584-4b0b-bb34-73b18df98915"} {"msg":"The guest has finished waiting for activation with:130","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:29.616310247Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1702} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 361 len 8192 data: 10 9 02/30 Read block 495 len 4096 03/30 Read block 22 len 32768 04/30 Read block 54 len 24576 05/30 Read block 14 len 32768 06/30 Write block 91 len 24576 data: 13 14 12 12 12 11 07/30 Write block 164 len 24576 data: 12 13 13 12 13 11 08/30 Read block 546 len 8192 09/30 Write block 263 len 4096 data: 7 10/30 Write block 287 len 32768 data: 8 11 11 12 14 15 14 12 11/30 Write block 568 len 8192 data: 10 10 12/30 Write block 188 len 36864 data: 6 6 9 9 11 13 14 12 12 13/30 Write block 229 len 40960 data: 12 13 12 11 11 12 13 11 12 7 14/30 Flush 15/30 Flush 16/30 Write block 240 len 20480 data: 8 10 10 10 10 17/30 Read block 529 len 28672 18/30 Read block 2 len 40960 19/30 Write block 203 len 32768 data: 8 7 7 6 7 9 11 11 20/30 Flush 21/30 Flush 22/30 Write block 35 len 8192 data: 18 12 23/30 Write block 501 len 28672 data: 17 14 14 13 15 13 15 24/30 Write block 477 len 24576 data: 9 8 7 10 10 9 25/30 Write block 103 len 20480 data: 13 13 13 10 8 26/30 Write block 311 len 16384 data: 4 5 6 5 27/30 Write block 124 len 8192 data: 4 7 28/30 Read block 165 len 24576 29/30 Read block 285 len 36864 30/30 Read block 388 len 40960 ---------------------------------------------------------------- Crucible gen:130 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 2 Done Done Done false 1022 Acked Write 7 Done Done Done false 1023 Acked Write 6 Done Done Done false 1024 Acked Write 5 Done Done Done false 1025 Acked Write 4 Done Done Done false 1026 Acked Write 2 Done Done Done false 1027 Acked Read 6 Done Done Done false 1028 Acked Read 9 Done Done Done false 1029 Acked Read 10 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 9 9 8 26 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1020 1020 1020 Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:16:29.635217888Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1702} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 116 116 116 210 210 210 F F F 1 020-039 122 122 122 224 224 224 T T T 2 040-059 128 128 128 235 235 235 F F F 3 060-079 128 128 128 236 236 236 F F F 4 080-099 130 130 130 238 238 238 F F F 5 100-119 129 129 129 237 237 237 T T T 6 120-139 128 128 128 235 235 235 T T T 7 140-159 128 128 128 235 235 235 F F F 8 160-179 130 130 130 238 238 238 F F F 9 180-199 130 130 130 238 238 238 F F F 10 200-219 130 130 130 240 240 240 F F F 11 220-239 130 130 130 238 238 238 F F F 12 240-259 130 130 130 240 240 240 F F F 13 260-279 130 130 130 238 238 238 F F F 14 280-299 130 130 130 238 238 238 F F F 15 300-319 123 123 123 227 227 227 T T T 16 320-339 123 123 123 227 227 227 F F F 17 340-359 126 126 126 233 233 233 F F F 18 360-379 130 130 130 238 238 238 F F F 19 380-399 129 129 129 237 237 237 F F F 20 400-419 125 125 125 232 232 232 F F F 21 420-439 129 129 129 237 237 237 F F F 22 440-459 129 129 129 237 237 237 F F F 23 460-479 126 126 126 233 233 233 T T T 24 480-499 129 129 129 237 237 237 T T T 25 500-519 129 129 129 237 237 237 T T T 26 520-539 128 128 128 236 236 236 F F F 27 540-559 127 127 127 234 234 234 F F F 28 560-579 130 130 130 238 238 238 F F F 29 580-599 127 127 127 234 234 234 F F F Max gen: 130, Max flush: 240 A Difference in extent metadata is expected here On loop 65 Verifying data now No change after verify Loop: 65 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 116 116 116 210 210 210 F F F 1 020-039 131 131 131 241 241 241 F F F 2 040-059 128 128 128 235 235 235 F F F 3 060-079 128 128 128 236 236 236 F F F 4 080-099 130 130 130 238 238 238 F F F 5 100-119 131 131 131 241 241 241 F F F 6 120-139 131 131 131 241 241 241 F F F 7 140-159 128 128 128 235 235 235 F F F 8 160-179 130 130 130 238 238 238 F F F 9 180-199 130 130 130 238 238 238 F F F 10 200-219 130 130 130 240 240 240 F F F 11 220-239 130 130 130 238 238 238 F F F 12 240-259 130 130 130 240 240 240 F F F 13 260-279 130 130 130 238 238 238 F F F 14 280-299 130 130 130 238 238 238 F F F 15 300-319 131 131 131 241 241 241 F F F 16 320-339 123 123 123 227 227 227 F F F 17 340-359 126 126 126 233 233 233 F F F 18 360-379 130 130 130 238 238 238 F F F 19 380-399 129 129 129 237 237 237 F F F 20 400-419 125 125 125 232 232 232 F F F 21 420-439 129 129 129 237 237 237 F F F 22 440-459 129 129 129 237 237 237 F F F 23 460-479 131 131 131 241 241 241 F F F 24 480-499 131 131 131 241 241 241 F F F 25 500-519 131 131 131 241 241 241 F F F 26 520-539 128 128 128 236 236 236 F F F 27 540-559 127 127 127 234 234 234 F F F 28 560-579 130 130 130 238 238 238 F F F 29 580-599 127 127 127 234 234 234 F F F Max gen: 131, Max flush: 241 Begin loop 66 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:33.8879361Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1710,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:33.93199421Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1710,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:33.932644801Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1710} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"abd5430f1508af0bc008143ea2566277ef714ec2\",\n git_commit_timestamp: \"2025-01-17T20:55:03.000000000Z\",\n git_branch: \"alan/tokio-to-140\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:33.932684174Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1710} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:33.932698622Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1710} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:33.932712881Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1710} {"msg":"Crucible 165f3a2c-7e6b-43d8-9dd9-efbbb9652c29 has session id: 2fcf4af8-00c2-430f-b281-e6cb744499c6","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:33.933244671Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1710,"session_id":"2fcf4af8-00c2-430f-b281-e6cb744499c6"} {"msg":"Upstairs opts: Upstairs UUID: 165f3a2c-7e6b-43d8-9dd9-efbbb9652c29, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:33.933297212Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1710,"session_id":"2fcf4af8-00c2-430f-b281-e6cb744499c6"} {"msg":"Crucible stats registered with UUID: 165f3a2c-7e6b-43d8-9dd9-efbbb9652c29","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:33.933324878Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1710,"session_id":"2fcf4af8-00c2-430f-b281-e6cb744499c6"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:16:33.933345871Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1710,"session_id":"2fcf4af8-00c2-430f-b281-e6cb744499c6"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:33.933870517Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1710,"task":"crutest"} {"msg":"The guest has requested activation with gen:132","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:33.933904287Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1710} {"msg":"165f3a2c-7e6b-43d8-9dd9-efbbb9652c29 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:33.933975178Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1710,"session_id":"2fcf4af8-00c2-430f-b281-e6cb744499c6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:33.934002004Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1710,"client":"0","":"downstairs","session_id":"2fcf4af8-00c2-430f-b281-e6cb744499c6"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:33.934027529Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1710,"client":"0","":"downstairs","session_id":"2fcf4af8-00c2-430f-b281-e6cb744499c6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:33.934041627Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1710,"client":"1","":"downstairs","session_id":"2fcf4af8-00c2-430f-b281-e6cb744499c6"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:33.934055875Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1710,"client":"1","":"downstairs","session_id":"2fcf4af8-00c2-430f-b281-e6cb744499c6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:33.934068423Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1710,"client":"2","":"downstairs","session_id":"2fcf4af8-00c2-430f-b281-e6cb744499c6"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:33.934081901Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1710,"client":"2","":"downstairs","session_id":"2fcf4af8-00c2-430f-b281-e6cb744499c6"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:33.934094528Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1710,"":"io task","client":"2","":"downstairs","session_id":"2fcf4af8-00c2-430f-b281-e6cb744499c6"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:33.934107796Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1710,"":"io task","client":"0","":"downstairs","session_id":"2fcf4af8-00c2-430f-b281-e6cb744499c6"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:33.934129989Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1710,"":"io task","client":"1","":"downstairs","session_id":"2fcf4af8-00c2-430f-b281-e6cb744499c6"} {"msg":"ds_connection connected from Ok(127.0.0.1:35496)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:33.934155664Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1710,"":"io task","client":"2","":"downstairs","session_id":"2fcf4af8-00c2-430f-b281-e6cb744499c6"} {"msg":"ds_connection connected from Ok(127.0.0.1:39178)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:33.934184681Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1710,"":"io task","client":"0","":"downstairs","session_id":"2fcf4af8-00c2-430f-b281-e6cb744499c6"} {"msg":"ds_connection connected from Ok(127.0.0.1:60019)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:33.93440556Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1710,"":"io task","client":"1","":"downstairs","session_id":"2fcf4af8-00c2-430f-b281-e6cb744499c6"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:33.934903101Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1710,"client":"0","":"downstairs","session_id":"2fcf4af8-00c2-430f-b281-e6cb744499c6"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:33.934938681Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1710,"client":"1","":"downstairs","session_id":"2fcf4af8-00c2-430f-b281-e6cb744499c6"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:33.935523103Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1710,"client":"1","":"downstairs","session_id":"2fcf4af8-00c2-430f-b281-e6cb744499c6"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:33.935636729Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1710,"client":"0","":"downstairs","session_id":"2fcf4af8-00c2-430f-b281-e6cb744499c6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:33.936035752Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1710,"session_id":"2fcf4af8-00c2-430f-b281-e6cb744499c6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:33.936235419Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1710,"session_id":"2fcf4af8-00c2-430f-b281-e6cb744499c6"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:33.936627038Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1710,"client":"2","":"downstairs","session_id":"2fcf4af8-00c2-430f-b281-e6cb744499c6"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:33.937136986Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1710,"client":"2","":"downstairs","session_id":"2fcf4af8-00c2-430f-b281-e6cb744499c6"} {"msg":"[0]R flush_numbers[0..12]: [210, 241, 235, 236, 238, 241, 241, 235, 238, 238, 240, 238]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:33.937611864Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1710,"":"downstairs","":"downstairs","session_id":"2fcf4af8-00c2-430f-b281-e6cb744499c6"} {"msg":"[0]R generation[0..12]: [116, 131, 128, 128, 130, 131, 131, 128, 130, 130, 130, 130]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:33.937635247Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1710,"":"downstairs","":"downstairs","session_id":"2fcf4af8-00c2-430f-b281-e6cb744499c6"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:33.937650126Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1710,"":"downstairs","":"downstairs","session_id":"2fcf4af8-00c2-430f-b281-e6cb744499c6"} {"msg":"[1]R flush_numbers[0..12]: [210, 241, 235, 236, 238, 241, 241, 235, 238, 238, 240, 238]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:33.937664204Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1710,"":"downstairs","":"downstairs","session_id":"2fcf4af8-00c2-430f-b281-e6cb744499c6"} {"msg":"[1]R generation[0..12]: [116, 131, 128, 128, 130, 131, 131, 128, 130, 130, 130, 130]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:33.937681164Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1710,"":"downstairs","":"downstairs","session_id":"2fcf4af8-00c2-430f-b281-e6cb744499c6"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:33.937695113Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1710,"":"downstairs","":"downstairs","session_id":"2fcf4af8-00c2-430f-b281-e6cb744499c6"} {"msg":"[2]R flush_numbers[0..12]: [210, 241, 235, 236, 238, 241, 241, 235, 238, 238, 240, 238]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:33.93770838Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1710,"":"downstairs","":"downstairs","session_id":"2fcf4af8-00c2-430f-b281-e6cb744499c6"} {"msg":"[2]R generation[0..12]: [116, 131, 128, 128, 130, 131, 131, 128, 130, 130, 130, 130]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:33.937721778Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1710,"":"downstairs","":"downstairs","session_id":"2fcf4af8-00c2-430f-b281-e6cb744499c6"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:33.937735336Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1710,"":"downstairs","":"downstairs","session_id":"2fcf4af8-00c2-430f-b281-e6cb744499c6"} {"msg":"Max found gen is 132","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:33.937747973Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1710,"":"downstairs","":"downstairs","session_id":"2fcf4af8-00c2-430f-b281-e6cb744499c6"} {"msg":"Generation requested: 132 >= found:132","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-17T21:16:33.937760731Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1710,"":"downstairs","":"downstairs","session_id":"2fcf4af8-00c2-430f-b281-e6cb744499c6"} {"msg":"Next flush: 242","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:33.937784205Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1710,"":"downstairs","":"downstairs","session_id":"2fcf4af8-00c2-430f-b281-e6cb744499c6"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:33.937808559Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1710,"":"downstairs","":"downstairs","session_id":"2fcf4af8-00c2-430f-b281-e6cb744499c6"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:33.937830652Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1710,"session_id":"2fcf4af8-00c2-430f-b281-e6cb744499c6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:33.937851664Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1710,"client":"0","":"downstairs","session_id":"2fcf4af8-00c2-430f-b281-e6cb744499c6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:33.937875538Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1710,"client":"1","":"downstairs","session_id":"2fcf4af8-00c2-430f-b281-e6cb744499c6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:33.937900743Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1710,"client":"2","":"downstairs","session_id":"2fcf4af8-00c2-430f-b281-e6cb744499c6"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2025-01-17T21:16:33.937927068Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1710,"session_id":"2fcf4af8-00c2-430f-b281-e6cb744499c6"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:33.937953283Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1710,"session_id":"2fcf4af8-00c2-430f-b281-e6cb744499c6"} {"msg":"165f3a2c-7e6b-43d8-9dd9-efbbb9652c29 is now active with session: 2fcf4af8-00c2-430f-b281-e6cb744499c6","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:33.937973935Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1710,"session_id":"2fcf4af8-00c2-430f-b281-e6cb744499c6"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:33.937986243Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1710,"session_id":"2fcf4af8-00c2-430f-b281-e6cb744499c6"} {"msg":"The guest has finished waiting for activation with:132","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:33.937998139Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1710} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 89 len 20480 data: 11 14 14 15 13 02/30 Write block 523 len 12288 data: 15 13 11 03/30 Flush 04/30 Write block 173 len 28672 data: 10 9 12 11 12 9 7 05/30 Read block 256 len 40960 06/30 Write block 86 len 40960 data: 8 8 9 12 15 15 16 14 13 13 07/30 Write block 215 len 12288 data: 10 10 8 08/30 Read block 75 len 20480 09/30 Flush 10/30 Write block 279 len 36864 data: 6 7 10 10 7 8 8 8 9 11/30 Read block 176 len 36864 12/30 Read block 77 len 40960 13/30 Write block 122 len 4096 data: 7 14/30 Write block 428 len 12288 data: 14 13 12 15/30 Write block 103 len 24576 data: 14 14 14 11 9 9 16/30 Read block 282 len 40960 17/30 Read block 67 len 8192 18/30 Write block 563 len 4096 data: 16 19/30 Read block 356 len 4096 20/30 Read block 303 len 4096 21/30 Read block 486 len 36864 22/30 Write block 362 len 12288 data: 10 10 7 23/30 Read block 2 len 28672 24/30 Flush 25/30 Write block 153 len 8192 data: 8 10 26/30 Write block 147 len 24576 data: 8 7 7 8 7 8 27/30 Read block 457 len 12288 28/30 Read block 5 len 24576 29/30 Write block 178 len 20480 data: 10 8 9 9 8 30/30 Write block 590 len 20480 data: 9 6 9 8 7 ---------------------------------------------------------------- Crucible gen:132 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 2 Done Done Done false 1025 Acked Write 6 Done Done Done false 1026 Acked Read 3 Done Done Done false 1027 Acked Read 6 Done Done Done false 1028 Acked Write 5 Sent Sent Sent false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:16:33.95658048Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1710} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 116 116 116 210 210 210 F F F 1 020-039 131 131 131 241 241 241 F F F 2 040-059 128 128 128 235 235 235 F F F 3 060-079 128 128 128 236 236 236 F F F 4 080-099 132 132 132 243 243 243 F F F 5 100-119 132 132 132 244 244 244 F F F 6 120-139 132 132 132 244 244 244 F F F 7 140-159 128 128 128 235 235 235 T T T 8 160-179 132 132 132 243 243 243 T T T 9 180-199 130 130 130 238 238 238 T T T 10 200-219 132 132 132 243 243 243 F F F 11 220-239 130 130 130 238 238 238 F F F 12 240-259 130 130 130 240 240 240 F F F 13 260-279 132 132 132 244 244 244 F F F 14 280-299 132 132 132 244 244 244 F F F 15 300-319 131 131 131 241 241 241 F F F 16 320-339 123 123 123 227 227 227 F F F 17 340-359 126 126 126 233 233 233 F F F 18 360-379 132 132 132 244 244 244 F F F 19 380-399 129 129 129 237 237 237 F F F 20 400-419 125 125 125 232 232 232 F F F 21 420-439 132 132 132 244 244 244 F F F 22 440-459 129 129 129 237 237 237 F F F 23 460-479 131 131 131 241 241 241 F F F 24 480-499 131 131 131 241 241 241 F F F 25 500-519 131 131 131 241 241 241 F F F 26 520-539 132 132 132 242 242 242 F F F 27 540-559 127 127 127 234 234 234 F F F 28 560-579 132 132 132 244 244 244 F F F 29 580-599 127 127 127 234 234 234 T T T Max gen: 132, Max flush: 244 A Difference in extent metadata is expected here On loop 66 Verifying data now No change after verify Loop: 66 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 116 116 116 210 210 210 F F F 1 020-039 131 131 131 241 241 241 F F F 2 040-059 128 128 128 235 235 235 F F F 3 060-079 128 128 128 236 236 236 F F F 4 080-099 132 132 132 243 243 243 F F F 5 100-119 132 132 132 244 244 244 F F F 6 120-139 132 132 132 244 244 244 F F F 7 140-159 133 133 133 245 245 245 F F F 8 160-179 133 133 133 245 245 245 F F F 9 180-199 133 133 133 245 245 245 F F F 10 200-219 132 132 132 243 243 243 F F F 11 220-239 130 130 130 238 238 238 F F F 12 240-259 130 130 130 240 240 240 F F F 13 260-279 132 132 132 244 244 244 F F F 14 280-299 132 132 132 244 244 244 F F F 15 300-319 131 131 131 241 241 241 F F F 16 320-339 123 123 123 227 227 227 F F F 17 340-359 126 126 126 233 233 233 F F F 18 360-379 132 132 132 244 244 244 F F F 19 380-399 129 129 129 237 237 237 F F F 20 400-419 125 125 125 232 232 232 F F F 21 420-439 132 132 132 244 244 244 F F F 22 440-459 129 129 129 237 237 237 F F F 23 460-479 131 131 131 241 241 241 F F F 24 480-499 131 131 131 241 241 241 F F F 25 500-519 131 131 131 241 241 241 F F F 26 520-539 132 132 132 242 242 242 F F F 27 540-559 127 127 127 234 234 234 F F F 28 560-579 132 132 132 244 244 244 F F F 29 580-599 133 133 133 245 245 245 F F F Max gen: 133, Max flush: 245 Begin loop 67 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:38.194566242Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1718,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:38.239266659Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1718,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:38.239939122Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1718} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"abd5430f1508af0bc008143ea2566277ef714ec2\",\n git_commit_timestamp: \"2025-01-17T20:55:03.000000000Z\",\n git_branch: \"alan/tokio-to-140\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:38.239977314Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1718} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:38.239996635Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1718} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:38.240011164Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1718} {"msg":"Crucible daf1eb37-4934-4d32-8829-9034d2aa9bc2 has session id: 8ef21eb8-b989-45e4-8a2e-13e8e501de4f","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:38.24050014Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1718,"session_id":"8ef21eb8-b989-45e4-8a2e-13e8e501de4f"} {"msg":"Upstairs opts: Upstairs UUID: daf1eb37-4934-4d32-8829-9034d2aa9bc2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:38.240538582Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1718,"session_id":"8ef21eb8-b989-45e4-8a2e-13e8e501de4f"} {"msg":"Crucible stats registered with UUID: daf1eb37-4934-4d32-8829-9034d2aa9bc2","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:38.240560905Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1718,"session_id":"8ef21eb8-b989-45e4-8a2e-13e8e501de4f"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:16:38.240582548Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1718,"session_id":"8ef21eb8-b989-45e4-8a2e-13e8e501de4f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:38.241267718Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1718,"task":"crutest"} {"msg":"The guest has requested activation with gen:134","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:38.241293313Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1718} {"msg":"daf1eb37-4934-4d32-8829-9034d2aa9bc2 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:38.241361453Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1718,"session_id":"8ef21eb8-b989-45e4-8a2e-13e8e501de4f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:38.241379113Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1718,"client":"0","":"downstairs","session_id":"8ef21eb8-b989-45e4-8a2e-13e8e501de4f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:38.241403728Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1718,"client":"0","":"downstairs","session_id":"8ef21eb8-b989-45e4-8a2e-13e8e501de4f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:38.241417486Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1718,"client":"1","":"downstairs","session_id":"8ef21eb8-b989-45e4-8a2e-13e8e501de4f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:38.241431434Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1718,"client":"1","":"downstairs","session_id":"8ef21eb8-b989-45e4-8a2e-13e8e501de4f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:38.241453717Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1718,"client":"2","":"downstairs","session_id":"8ef21eb8-b989-45e4-8a2e-13e8e501de4f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:38.241481903Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1718,"client":"2","":"downstairs","session_id":"8ef21eb8-b989-45e4-8a2e-13e8e501de4f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:38.24151055Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1718,"":"io task","client":"2","":"downstairs","session_id":"8ef21eb8-b989-45e4-8a2e-13e8e501de4f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:38.241537836Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1718,"":"io task","client":"0","":"downstairs","session_id":"8ef21eb8-b989-45e4-8a2e-13e8e501de4f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:38.241562881Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1718,"":"io task","client":"1","":"downstairs","session_id":"8ef21eb8-b989-45e4-8a2e-13e8e501de4f"} {"msg":"ds_connection connected from Ok(127.0.0.1:48454)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:38.241587025Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1718,"":"io task","client":"2","":"downstairs","session_id":"8ef21eb8-b989-45e4-8a2e-13e8e501de4f"} {"msg":"ds_connection connected from Ok(127.0.0.1:40286)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:38.24161322Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1718,"":"io task","client":"1","":"downstairs","session_id":"8ef21eb8-b989-45e4-8a2e-13e8e501de4f"} {"msg":"ds_connection connected from Ok(127.0.0.1:57583)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:38.241639716Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1718,"":"io task","client":"0","":"downstairs","session_id":"8ef21eb8-b989-45e4-8a2e-13e8e501de4f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:38.242355594Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1718,"client":"0","":"downstairs","session_id":"8ef21eb8-b989-45e4-8a2e-13e8e501de4f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:38.242382149Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1718,"client":"1","":"downstairs","session_id":"8ef21eb8-b989-45e4-8a2e-13e8e501de4f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:38.242397198Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1718,"client":"2","":"downstairs","session_id":"8ef21eb8-b989-45e4-8a2e-13e8e501de4f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:38.24299926Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1718,"client":"0","":"downstairs","session_id":"8ef21eb8-b989-45e4-8a2e-13e8e501de4f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:38.243029478Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1718,"client":"1","":"downstairs","session_id":"8ef21eb8-b989-45e4-8a2e-13e8e501de4f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:38.243044607Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1718,"client":"2","":"downstairs","session_id":"8ef21eb8-b989-45e4-8a2e-13e8e501de4f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:38.243537805Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1718,"session_id":"8ef21eb8-b989-45e4-8a2e-13e8e501de4f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:38.243639104Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1718,"session_id":"8ef21eb8-b989-45e4-8a2e-13e8e501de4f"} {"msg":"[0]R flush_numbers[0..12]: [210, 241, 235, 236, 243, 244, 244, 245, 245, 245, 243, 238]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:38.243674505Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1718,"":"downstairs","":"downstairs","session_id":"8ef21eb8-b989-45e4-8a2e-13e8e501de4f"} {"msg":"[0]R generation[0..12]: [116, 131, 128, 128, 132, 132, 132, 133, 133, 133, 132, 130]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:38.243691935Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1718,"":"downstairs","":"downstairs","session_id":"8ef21eb8-b989-45e4-8a2e-13e8e501de4f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:38.243706414Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1718,"":"downstairs","":"downstairs","session_id":"8ef21eb8-b989-45e4-8a2e-13e8e501de4f"} {"msg":"[1]R flush_numbers[0..12]: [210, 241, 235, 236, 243, 244, 244, 245, 245, 245, 243, 238]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:38.243720082Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1718,"":"downstairs","":"downstairs","session_id":"8ef21eb8-b989-45e4-8a2e-13e8e501de4f"} {"msg":"[1]R generation[0..12]: [116, 131, 128, 128, 132, 132, 132, 133, 133, 133, 132, 130]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:38.24373355Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1718,"":"downstairs","":"downstairs","session_id":"8ef21eb8-b989-45e4-8a2e-13e8e501de4f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:38.243747217Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1718,"":"downstairs","":"downstairs","session_id":"8ef21eb8-b989-45e4-8a2e-13e8e501de4f"} {"msg":"[2]R flush_numbers[0..12]: [210, 241, 235, 236, 243, 244, 244, 245, 245, 245, 243, 238]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:38.243760635Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1718,"":"downstairs","":"downstairs","session_id":"8ef21eb8-b989-45e4-8a2e-13e8e501de4f"} {"msg":"[2]R generation[0..12]: [116, 131, 128, 128, 132, 132, 132, 133, 133, 133, 132, 130]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:38.243773963Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1718,"":"downstairs","":"downstairs","session_id":"8ef21eb8-b989-45e4-8a2e-13e8e501de4f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:38.243787741Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1718,"":"downstairs","":"downstairs","session_id":"8ef21eb8-b989-45e4-8a2e-13e8e501de4f"} {"msg":"Max found gen is 134","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:38.243800509Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1718,"":"downstairs","":"downstairs","session_id":"8ef21eb8-b989-45e4-8a2e-13e8e501de4f"} {"msg":"Generation requested: 134 >= found:134","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:38.243813486Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1718,"":"downstairs","":"downstairs","session_id":"8ef21eb8-b989-45e4-8a2e-13e8e501de4f"} {"msg":"Next flush: 246","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:38.243826013Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1718,"":"downstairs","":"downstairs","session_id":"8ef21eb8-b989-45e4-8a2e-13e8e501de4f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:38.243838341Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1718,"":"downstairs","":"downstairs","Wait for a query_work_queue command to finish before sending IO session_id":"8ef21eb8-b989-45e4-8a2e-13e8e501de4f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:38.243864896Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1718,"session_id":"8ef21eb8-b989-45e4-8a2e-13e8e501de4f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:38.243891102Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1718,"client":"0","":"downstairs","session_id":"8ef21eb8-b989-45e4-8a2e-13e8e501de4f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:38.243916806Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1718,"client":"1","":"downstairs","session_id":"8ef21eb8-b989-45e4-8a2e-13e8e501de4f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:38.243941471Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1718,"client":"2","":"downstairs","session_id":"8ef21eb8-b989-45e4-8a2e-13e8e501de4f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:38.243965485Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1718,"session_id":"8ef21eb8-b989-45e4-8a2e-13e8e501de4f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:38.243979693Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1718,"session_id":"8ef21eb8-b989-45e4-8a2e-13e8e501de4f"} {"msg":"daf1eb37-4934-4d32-8829-9034d2aa9bc2 is now active with session: 8ef21eb8-b989-45e4-8a2e-13e8e501de4f","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:38.243992261Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1718,"session_id":"8ef21eb8-b989-45e4-8a2e-13e8e501de4f"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:38.244023709Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1718,"session_id":"8ef21eb8-b989-45e4-8a2e-13e8e501de4f"} {"msg":"The guest has finished waiting for activation with:134","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:38.244036777Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1718} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 182 len 16384 data: 9 6 4 3 02/30 Write block 496 len 32768 data: 14 14 16 14 16 18 15 15 03/30 Flush 04/30 Write block 49 len 32768 data: 10 10 12 15 15 15 14 14 05/30 Write block 295 len 12288 data: 14 13 11 06/30 Read block 159 len 20480 07/30 Write block 392 len 8192 data: 8 7 08/30 Write block 373 len 16384 data: 12 14 16 15 09/30 Write block 202 len 4096 data: 12 10/30 Write block 524 len 24576 data: 14 12 12 10 9 7 11/30 Write block 454 len 28672 data: 9 9 9 10 11 11 11 12/30 Write block 371 len 28672 data: 9 13 13 15 17 16 17 13/30 Read block 96 len 20480 14/30 Write block 128 len 24576 data: 8 9 10 9 9 6 15/30 Write block 575 len 40960 data: 8 10 10 9 11 10 11 10 10 11 16/30 Write block 45 len 8192 data: 6 8 17/30 Write block 380 len 40960 data: 14 11 9 10 10 8 8 8 8 10 18/30 Read block 252 len 8192 19/30 Write block 87 len 40960 data: 9 10 13 16 16 17 15 14 14 12 20/30 Read block 426 len 20480 21/30 Read block 521 len 4096 22/30 Write block 109 len 40960 data: 10 11 15 16 13 14 14 12 13 15 23/30 Flush 24/30 Flush 25/30 Write block 86 len 24576 data: 9 10 11 14 17 17 26/30 Read block 301 len 12288 27/30 Write block 232 len 40960 data: 12 12 13 14 12 13 8 6 9 11 28/30 Read block 37 len 16384 29/30 Write block 420 len 24576 data: 8 10 10 10 9 9 30/30 Write block 111 len 28672 data: 16 17 14 15 15 13 14 ---------------------------------------------------------------- Crucible gen:134 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 6 Done Done Done false 1025 Acked Read 3 Done Done Done false 1026 Acked Write 10 Done Done Done false 1027 Acked Read 4 Done Done Done false 1028 Acked Write 6 Sent Sent Done false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 1 5 Done 4 4 5 13 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:16:38.262953153Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1718} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 116 116 116 210 210 210 F F F 1 020-039 131 131 131 241 241 241 F F F 2 040-059 134 134 134 247 247 247 F F F 3 060-079 128 128 128 236 236 236 F F F 4 080-099 134 134 134 247 247 247 T T T 5 100-119 134 134 134 247 247 247 T T T 6 120-139 134 134 134 247 247 247 F F F 7 140-159 133 133 133 245 245 245 F F F 8 160-179 133 133 133 245 245 245 F F F 9 180-199 134 134 134 246 246 246 F F F 10 200-219 134 134 134 247 247 247 F F F 11 220-239 130 130 130 238 238 238 T T T 12 240-259 130 130 130 240 240 240 T T T 13 260-279 132 132 132 244 244 244 F F F 14 280-299 134 134 134 247 247 247 F F F 15 300-319 131 131 131 241 241 241 F F F 16 320-339 123 123 123 227 227 227 F F F 17 340-359 126 126 126 233 233 233 F F F 18 360-379 134 134 134 247 247 247 F F F 19 380-399 134 134 134 247 247 247 F F F 20 400-419 125 125 125 232 232 232 F F F 21 420-439 132 132 132 244 244 244 T T T 22 440-459 134 134 134 247 247 247 F F F 23 460-479 134 134 134 247 247 247 F F F 24 480-499 134 134 134 246 246 246 F F F 25 500-519 134 134 134 246 246 246 F F F 26 520-539 134 134 134 247 247 247 F F F 27 540-559 127 127 127 234 234 234 F F F 28 560-579 134 134 134 247 247 247 F F F 29 580-599 134 134 134 247 247 247 F F F Max gen: 134, Max flush: 247 A Difference in extent metadata is expected here On loop 67 Verifying data now No change after verify Loop: 67 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 116 116 116 210 210 210 F F F 1 020-039 131 131 131 241 241 241 F F F 2 040-059 134 134 134 247 247 247 F F F 3 060-079 128 128 128 236 236 236 F F F 4 080-099 135 135 135 248 248 248 F F F 5 100-119 135 135 135 248 248 248 F F F 6 120-139 134 134 134 247 247 247 F F F 7 140-159 133 133 133 245 245 245 F F F 8 160-179 133 133 133 245 245 245 F F F 9 180-199 134 134 134 246 246 246 F F F 10 200-219 134 134 134 247 247 247 F F F 11 220-239 135 135 135 248 248 248 F F F 12 240-259 135 135 135 248 248 248 F F F 13 260-279 132 132 132 244 244 244 F F F 14 280-299 134 134 134 247 247 247 F F F 15 300-319 131 131 131 241 241 241 F F F 16 320-339 123 123 123 227 227 227 F F F 17 340-359 126 126 126 233 233 233 F F F 18 360-379 134 134 134 247 247 247 F F F 19 380-399 134 134 134 247 247 247 F F F 20 400-419 125 125 125 232 232 232 F F F 21 420-439 135 135 135 248 248 248 F F F 22 440-459 134 134 134 247 247 247 F F F 23 460-479 134 134 134 247 247 247 F F F 24 480-499 134 134 134 246 246 246 F F F 25 500-519 134 134 134 246 246 246 F F F 26 520-539 134 134 134 247 247 247 F F F 27 540-559 127 127 127 234 234 234 F F F 28 560-579 134 134 134 247 247 247 F F F 29 580-599 134 134 134 247 247 247 F F F Max gen: 135, Max flush: 248 Begin loop 68 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:42.502117322Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1726,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:16:42.544982366Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1726,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:42.545430858Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1726,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:42.577226031Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1726,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:42.577881644Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1726} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"abd5430f1508af0bc008143ea2566277ef714ec2\",\n git_commit_timestamp: \"2025-01-17T20:55:03.000000000Z\",\n git_branch: \"alan/tokio-to-140\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:42.577916965Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1726} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:42.577932634Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1726} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:42.577950034Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1726} {"msg":"Crucible ecbcaeb8-b606-4379-b4c8-4a0ed02cf90c has session id: 82834f31-f02f-4bf7-8996-1a6cfb0cb7a0","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:42.57861063Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1726,"session_id":"82834f31-f02f-4bf7-8996-1a6cfb0cb7a0"} {"msg":"Upstairs opts: Upstairs UUID: ecbcaeb8-b606-4379-b4c8-4a0ed02cf90c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:42.578641198Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1726,"session_id":"82834f31-f02f-4bf7-8996-1a6cfb0cb7a0"} {"msg":"Crucible stats registered with UUID: ecbcaeb8-b606-4379-b4c8-4a0ed02cf90c","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:42.578656417Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1726,"session_id":"82834f31-f02f-4bf7-8996-1a6cfb0cb7a0"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:16:42.578669325Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1726,"session_id":"82834f31-f02f-4bf7-8996-1a6cfb0cb7a0"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:42.579102848Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1726,"task":"crutest"} {"msg":"The guest has requested activation with gen:136","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:42.579129964Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1726} {"msg":"ecbcaeb8-b606-4379-b4c8-4a0ed02cf90c active request set","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:42.579192831Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1726,"session_id":"82834f31-f02f-4bf7-8996-1a6cfb0cb7a0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:42.579209711Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1726,"client":"0","":"downstairs","session_id":"82834f31-f02f-4bf7-8996-1a6cfb0cb7a0"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:42.579233565Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1726,"client":"0","":"downstairs","session_id":"82834f31-f02f-4bf7-8996-1a6cfb0cb7a0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:42.579247423Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1726,"client":"1","":"downstairs","session_id":"82834f31-f02f-4bf7-8996-1a6cfb0cb7a0"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:42.579261511Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1726,"client":"1","":"downstairs","session_id":"82834f31-f02f-4bf7-8996-1a6cfb0cb7a0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:42.579274298Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1726,"client":"2","":"downstairs","session_id":"82834f31-f02f-4bf7-8996-1a6cfb0cb7a0"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:42.579295471Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1726,"client":"2","":"downstairs","session_id":"82834f31-f02f-4bf7-8996-1a6cfb0cb7a0"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:42.579317984Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1726,"":"io task","client":"2","":"downstairs","session_id":"82834f31-f02f-4bf7-8996-1a6cfb0cb7a0"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:42.579342008Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1726,"":"io task","client":"0","":"downstairs","session_id":"82834f31-f02f-4bf7-8996-1a6cfb0cb7a0"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:42.579365872Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1726,"":"io task","client":"1","":"downstairs","session_id":"82834f31-f02f-4bf7-8996-1a6cfb0cb7a0"} {"msg":"ds_connection connected from Ok(127.0.0.1:50216)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:42.579394689Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1726,"":"io task","client":"2","":"downstairs","session_id":"82834f31-f02f-4bf7-8996-1a6cfb0cb7a0"} {"msg":"ds_connection connected from Ok(127.0.0.1:52228)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:42.579419173Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1726,"":"io task","client":"0","":"downstairs","session_id":"82834f31-f02f-4bf7-8996-1a6cfb0cb7a0"} {"msg":"ds_connection connected from Ok(127.0.0.1:51809)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:42.579656101Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1726,"":"io task","client":"1","":"downstairs","session_id":"82834f31-f02f-4bf7-8996-1a6cfb0cb7a0"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:42.580187472Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1726,"client":"1","":"downstairs","session_id":"82834f31-f02f-4bf7-8996-1a6cfb0cb7a0"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:42.580369909Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1726,"client":"0","":"downstairs","session_id":"82834f31-f02f-4bf7-8996-1a6cfb0cb7a0"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:42.580409602Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1726,"client":"2","":"downstairs","session_id":"82834f31-f02f-4bf7-8996-1a6cfb0cb7a0"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:42.580871632Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1726,"client":"1","":"downstairs","session_id":"82834f31-f02f-4bf7-8996-1a6cfb0cb7a0"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:42.581009332Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1726,"client":"2","":"downstairs","session_id":"82834f31-f02f-4bf7-8996-1a6cfb0cb7a0"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:42.581233754Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1726,"client":"0","":"downstairs","session_id":"82834f31-f02f-4bf7-8996-1a6cfb0cb7a0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:42.58151786Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1726,"session_id":"82834f31-f02f-4bf7-8996-1a6cfb0cb7a0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:42.581542194Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1726,"session_id":"82834f31-f02f-4bf7-8996-1a6cfb0cb7a0"} {"msg":"[0]R flush_numbers[0..12]: [210, 241, 247, 236, 248, 248, 247, 245, 245, 246, 247, 248]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:42.581790239Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1726,"":"downstairs","":"downstairs","session_id":"82834f31-f02f-4bf7-8996-1a6cfb0cb7a0"} {"msg":"[0]R generation[0..12]: [116, 131, 134, 128, 135, 135, 134, 133, 133, 134, 134, 135]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:42.58180952Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1726,"":"downstairs","":"downstairs","session_id":"82834f31-f02f-4bf7-8996-1a6cfb0cb7a0"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:42.581824139Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1726,"":"downstairs","":"downstairs","session_id":"82834f31-f02f-4bf7-8996-1a6cfb0cb7a0"} {"msg":"[1]R flush_numbers[0..12]: [210, 241, 247, 236, 248, 248, 247, 245, 245, 246, 247, 248]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:42.581837947Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1726,"":"downstairs","":"downstairs","session_id":"82834f31-f02f-4bf7-8996-1a6cfb0cb7a0"} {"msg":"[1]R generation[0..12]: [116, 131, 134, 128, 135, 135, 134, 133, 133, 134, 134, 135]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:42.581851405Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1726,"":"downstairs","":"downstairs","session_id":"82834f31-f02f-4bf7-8996-1a6cfb0cb7a0"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:42.581864912Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1726,"":"downstairs","":"downstairs","session_id":"82834f31-f02f-4bf7-8996-1a6cfb0cb7a0"} {"msg":"[2]R flush_numbers[0..12]: [210, 241, 247, 236, 248, 248, 247, 245, 245, 246, 247, 248]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:42.58187821Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1726,"":"downstairs","":"downstairs","session_id":"82834f31-f02f-4bf7-8996-1a6cfb0cb7a0"} {"msg":"[2]R generation[0..12]: [116, 131, 134, 128, 135, 135, 134, 133, 133, 134, 134, 135]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:42.58189557Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1726,"":"downstairs","":"downstairs","session_id":"82834f31-f02f-4bf7-8996-1a6cfb0cb7a0"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:42.581909679Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1726,"":"downstairs","":"downstairs","session_id":"82834f31-f02f-4bf7-8996-1a6cfb0cb7a0"} {"msg":"Max found gen is 136","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:42.581922406Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1726,"":"downstairs","":"downstairs","session_id":"82834f31-f02f-4bf7-8996-1a6cfb0cb7a0"} {"msg":"Generation requested: 136 >= found:136","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:42.581935184Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1726,"":"downstairs","":"downstairs","session_id":"82834f31-f02f-4bf7-8996-1a6cfb0cb7a0"} {"msg":"Next flush: 249","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:42.581947951Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1726,"":"downstairs","":"downstairs","session_id":"82834f31-f02f-4bf7-8996-1a6cfb0cb7a0"} Wait for a query_work_queue command to finish before sending IO {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:42.581968753Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1726,"":"downstairs","":"downstairs","session_id":"82834f31-f02f-4bf7-8996-1a6cfb0cb7a0"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:42.581982191Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1726,"session_id":"82834f31-f02f-4bf7-8996-1a6cfb0cb7a0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:42.581993908Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1726,"client":"0","":"downstairs","session_id":"82834f31-f02f-4bf7-8996-1a6cfb0cb7a0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:42.582006955Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1726,"client":"1","":"downstairs","session_id":"82834f31-f02f-4bf7-8996-1a6cfb0cb7a0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:42.582019903Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1726,"client":"2","":"downstairs","session_id":"82834f31-f02f-4bf7-8996-1a6cfb0cb7a0"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:42.582032841Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1726,"session_id":"82834f31-f02f-4bf7-8996-1a6cfb0cb7a0"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:42.582044868Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1726,"session_id":"82834f31-f02f-4bf7-8996-1a6cfb0cb7a0"} {"msg":"ecbcaeb8-b606-4379-b4c8-4a0ed02cf90c is now active with session: 82834f31-f02f-4bf7-8996-1a6cfb0cb7a0","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:42.582057325Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1726,"session_id":"82834f31-f02f-4bf7-8996-1a6cfb0cb7a0"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:42.582069002Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1726,"session_id":"82834f31-f02f-4bf7-8996-1a6cfb0cb7a0"} {"msg":"The guest has finished waiting for activation with:136","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:42.582080959Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1726} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 560 len 4096 data: 15 02/30 Read block 198 len 8192 03/30 Flush 04/30 Write block 573 len 8192 data: 9 10 05/30 Write block 358 len 32768 data: 14 15 12 11 11 11 8 6 06/30 Write block 419 len 24576 data: 9 9 11 11 11 10 07/30 Read block 272 len 20480 08/30 Read block 467 len 40960 09/30 Read block 187 len 40960 10/30 Read block 291 len 4096 11/30 Read block 24 len 24576 12/30 Flush 13/30 Write block 216 len 32768 data: 11 9 7 7 7 7 9 9 14/30 Write block 54 len 24576 data: 16 15 15 14 14 13 15/30 Write block 415 len 12288 data: 11 10 9 16/30 Write block 434 len 24576 data: 10 8 8 9 7 5 17/30 Read block 146 len 32768 18/30 Read block 362 len 20480 19/30 Read block 390 len 4096 20/30 Read block 129 len 40960 21/30 Write block 566 len 20480 data: 14 9 11 11 11 22/30 Read block 508 len 12288 23/30 Write block 169 len 40960 data: 12 11 12 10 11 10 13 12 13 11 24/30 Write block 259 len 16384 data: 12 9 8 7 25/30 Write block 414 len 32768 data: 11 12 11 10 8 10 10 12 26/30 Read block 76 len 28672 27/30 Write block 448 len 20480 data: 5 8 7 6 7 28/30 Read block 227 len 24576 29/30 Read block 216 len 12288 30/30 Write block 96 len 24576 data: 13 8 12 11 12 13 ---------------------------------------------------------------- Crucible gen:136 GIO:true work queues: Upstairs:0 downstairs:18 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1012 Acked Write 8 Done Done Done false 1013 Acked Write 6 Done Done Done false 1014 Acked Write 3 Done Done Done false 1015 Acked Write 6 Done Done Done false 1016 Acked Read 8 Done Done Done false 1017 Acked Read 5 Done Done Done false 1018 Acked Read 1 Done Done Done false 1019 Acked Read 10 Done Done Done false 1020 Acked Write 5 Done Done Done false 1021 Acked Read 3 Done Done Done false 1022 Acked Write 10 Done Done Done false 1023 Acked Write 4 Done Done Done false 1024 Acked Write 8 Done Done Done false 1025 Acked Read 7 Done Done Done false 1026 Acked Write 5 Done Done Done false 1027 Acked Read 6 Done Done Done false 1028 Acked Read 3 Done Done Done false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 17 17 17 51 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1011 1011 1011 Downstairs last five completed: 1011 1010 1009 1008 1007 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:16:42.600944824Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1726} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 116 116 116 210 210 210 F F F 1 020-039 131 131 131 241 241 241 F F F 2 040-059 134 134 134 247 247 247 T T T 3 060-079 128 128 128 236 236 236 F F F 4 080-099 135 135 135 248 248 248 T T T 5 100-119 135 135 135 248 248 248 T T T 6 120-139 134 134 134 247 247 247 F F F 7 140-159 133 133 133 245 245 245 F F F 8 160-179 133 133 133 245 245 245 T T T 9 180-199 134 134 134 246 246 246 F F F 10 200-219 134 134 134 247 247 247 T T T 11 220-239 135 135 135 248 248 248 T T T 12 240-259 135 135 135 248 248 248 T T T 13 260-279 132 132 132 244 244 244 T T T 14 280-299 134 134 134 247 247 247 F F F 15 300-319 131 131 131 241 241 241 F F F 16 320-339 123 123 123 227 227 227 F F F 17 340-359 136 136 136 250 250 250 F F F 18 360-379 136 136 136 250 250 250 F F F 19 380-399 134 134 134 247 247 247 F F F 20 400-419 136 136 136 250 250 250 T T T 21 420-439 136 136 136 250 250 250 T T T 22 440-459 134 134 134 247 247 247 T T T 23 460-479 134 134 134 247 247 247 F F F 24 480-499 134 134 134 246 246 246 F F F 25 500-519 134 134 134 246 246 246 F F F 26 520-539 134 134 134 247 247 247 F F F 27 540-559 127 127 127 234 234 234 F F F 28 560-579 136 136 136 250 250 250 T T T 29 580-599 134 134 134 247 247 247 F F F Max gen: 136, Max flush: 250 A Difference in extent metadata is expected here On loop 68 Verifying data now No change after verify Loop: 68 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 116 116 116 210 210 210 F F F 1 020-039 131 131 131 241 241 241 F F F 2 040-059 137 137 137 251 251 251 F F F 3 060-079 128 128 128 236 236 236 F F F 4 080-099 137 137 137 251 251 251 F F F 5 100-119 137 137 137 251 251 251 F F F 6 120-139 134 134 134 247 247 247 F F F 7 140-159 133 133 133 245 245 245 F F F 8 160-179 137 137 137 251 251 251 F F F 9 180-199 134 134 134 246 246 246 F F F 10 200-219 137 137 137 251 251 251 F F F 11 220-239 137 137 137 251 251 251 F F F 12 240-259 137 137 137 251 251 251 F F F 13 260-279 137 137 137 251 251 251 F F F 14 280-299 134 134 134 247 247 247 F F F 15 300-319 131 131 131 241 241 241 F F F 16 320-339 123 123 123 227 227 227 F F F 17 340-359 136 136 136 250 250 250 F F F 18 360-379 136 136 136 250 250 250 F F F 19 380-399 134 134 134 247 247 247 F F F 20 400-419 137 137 137 251 251 251 F F F 21 420-439 137 137 137 251 251 251 F F F 22 440-459 137 137 137 251 251 251 F F F 23 460-479 134 134 134 247 247 247 F F F 24 480-499 134 134 134 246 246 246 F F F 25 500-519 134 134 134 246 246 246 F F F 26 520-539 134 134 134 247 247 247 F F F 27 540-559 127 127 127 234 234 234 F F F 28 560-579 137 137 137 251 251 251 F F F 29 580-599 134 134 134 247 247 247 F F F Max gen: 137, Max flush: 251 Begin loop 69 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:46.918495167Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1734,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:46.962576632Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1734,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:46.96320658Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1734} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"abd5430f1508af0bc008143ea2566277ef714ec2\",\n git_commit_timestamp: \"2025-01-17T20:55:03.000000000Z\",\n git_branch: \"alan/tokio-to-140\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:46.963247024Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1734} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:46.963265675Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1734} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:46.963280233Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1734} {"msg":"Crucible 644fe92e-6c9b-4b5b-b6ba-7e6ed071d3f1 has session id: 7af70976-c7dc-4c7c-9606-d0bd9e447603","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:46.963806571Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1734,"session_id":"7af70976-c7dc-4c7c-9606-d0bd9e447603"} {"msg":"Upstairs opts: Upstairs UUID: 644fe92e-6c9b-4b5b-b6ba-7e6ed071d3f1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:46.963847375Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1734,"session_id":"7af70976-c7dc-4c7c-9606-d0bd9e447603"} {"msg":"Crucible stats registered with UUID: 644fe92e-6c9b-4b5b-b6ba-7e6ed071d3f1","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:46.963869327Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1734,"session_id":"7af70976-c7dc-4c7c-9606-d0bd9e447603"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:16:46.963884676Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1734,"session_id":"7af70976-c7dc-4c7c-9606-d0bd9e447603"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:46.964322763Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1734,"task":"crutest"} {"msg":"The guest has requested activation with gen:138","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:46.964343815Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1734} {"msg":"644fe92e-6c9b-4b5b-b6ba-7e6ed071d3f1 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:46.96440303Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1734,"session_id":"7af70976-c7dc-4c7c-9606-d0bd9e447603"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:46.9644205Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1734,"client":"0","":"downstairs","session_id":"7af70976-c7dc-4c7c-9606-d0bd9e447603"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:46.964444383Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1734,"client":"0","":"downstairs","session_id":"7af70976-c7dc-4c7c-9606-d0bd9e447603"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:46.964458062Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1734,"client":"1","":"downstairs","session_id":"7af70976-c7dc-4c7c-9606-d0bd9e447603"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:46.964471699Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1734,"client":"1","":"downstairs","session_id":"7af70976-c7dc-4c7c-9606-d0bd9e447603"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:46.964484707Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1734,"client":"2","":"downstairs","session_id":"7af70976-c7dc-4c7c-9606-d0bd9e447603"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:46.964498385Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1734,"client":"2","":"downstairs","session_id":"7af70976-c7dc-4c7c-9606-d0bd9e447603"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:46.964515485Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1734,"":"io task","client":"2","":"downstairs","session_id":"7af70976-c7dc-4c7c-9606-d0bd9e447603"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:46.964537358Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1734,"":"io task","client":"0","":"downstairs","session_id":"7af70976-c7dc-4c7c-9606-d0bd9e447603"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:46.964562683Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1734,"":"io task","client":"1","":"downstairs","session_id":"7af70976-c7dc-4c7c-9606-d0bd9e447603"} {"msg":"ds_connection connected from Ok(127.0.0.1:34115)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:46.964606198Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1734,"":"io task","client":"2","":"downstairs","session_id":"7af70976-c7dc-4c7c-9606-d0bd9e447603"} {"msg":"ds_connection connected from Ok(127.0.0.1:48364)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:46.964634094Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1734,"":"io task","client":"0","":"downstairs","session_id":"7af70976-c7dc-4c7c-9606-d0bd9e447603"} {"msg":"ds_connection connected from Ok(127.0.0.1:51733)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:46.964658599Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1734,"":"io task","client":"1","":"downstairs","session_id":"7af70976-c7dc-4c7c-9606-d0bd9e447603"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:46.965371766Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1734,"client":"1","":"downstairs","session_id":"7af70976-c7dc-4c7c-9606-d0bd9e447603"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:46.965398341Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1734,"client":"0","":"downstairs","session_id":"7af70976-c7dc-4c7c-9606-d0bd9e447603"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:46.966071855Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1734,"client":"0","":"downstairs","session_id":"7af70976-c7dc-4c7c-9606-d0bd9e447603"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:46.966108356Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1734,"client":"1","":"downstairs","session_id":"7af70976-c7dc-4c7c-9606-d0bd9e447603"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:46.966669544Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1734,"session_id":"7af70976-c7dc-4c7c-9606-d0bd9e447603"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:46.966787263Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1734,"session_id":"7af70976-c7dc-4c7c-9606-d0bd9e447603"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:46.96710674Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1734,"client":"2","":"downstairs","session_id":"7af70976-c7dc-4c7c-9606-d0bd9e447603"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:46.967614246Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1734,"client":"2","":"downstairs","session_id":"7af70976-c7dc-4c7c-9606-d0bd9e447603"} {"msg":"[0]R flush_numbers[0..12]: [210, 241, 251, 236, 251, 251, 247, 245, 251, 246, 251, 251]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:46.968111107Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1734,"":"downstairs","":"downstairs","session_id":"7af70976-c7dc-4c7c-9606-d0bd9e447603"} {"msg":"[0]R generation[0..12]: [116, 131, 137, 128, 137, 137, 134, 133, 137, 134, 137, 137]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:46.96813448Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1734,"":"downstairs","":"downstairs","session_id":"7af70976-c7dc-4c7c-9606-d0bd9e447603"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:46.968149139Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1734,"":"downstairs","":"downstairs","session_id":"7af70976-c7dc-4c7c-9606-d0bd9e447603"} {"msg":"[1]R flush_numbers[0..12]: [210, 241, 251, 236, 251, 251, 247, 245, 251, 246, 251, 251]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:46.968163047Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1734,"":"downstairs","":"downstairs","session_id":"7af70976-c7dc-4c7c-9606-d0bd9e447603"} {"msg":"[1]R generation[0..12]: [116, 131, 137, 128, 137, 137, 134, 133, 137, 134, 137, 137]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:46.968176425Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1734,"":"downstairs","":"downstairs","session_id":"7af70976-c7dc-4c7c-9606-d0bd9e447603"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:46.968190053Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1734,"":"downstairs","":"downstairs","session_id":"7af70976-c7dc-4c7c-9606-d0bd9e447603"} {"msg":"[2]R flush_numbers[0..12]: [210, 241, 251, 236, 251, 251, 247, 245, 251, 246, 251, 251]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:46.9682032Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1734,"":"downstairs","":"downstairs","session_id":"7af70976-c7dc-4c7c-9606-d0bd9e447603"} {"msg":"[2]R generation[0..12]: [116, 131, 137, 128, 137, 137, 134, 133, 137, 134, 137, 137]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:46.968216448Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1734,"":"downstairs","":"downstairs","session_id":"7af70976-c7dc-4c7c-9606-d0bd9e447603"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:46.968229796Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1734,"":"downstairs","":"downstairs","session_id":"7af70976-c7dc-4c7c-9606-d0bd9e447603"} {"msg":"Max found gen is 138","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:46.968242293Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1734,"":"downstairs","":"downstairs","session_id":"7af70976-c7dc-4c7c-9606-d0bd9e447603"} {"msg":"Generation requested: 138 >= found:138","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:46.968255121Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1734,"":"downstairs","":"downstairs","session_id":"7af70976-c7dc-4c7c-9606-d0bd9e447603"} {"msg":"Next flush: 252","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:46.968267848Z","Wait for a query_work_queue command to finish before sending IO hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1734,"":"downstairs","":"downstairs","session_id":"7af70976-c7dc-4c7c-9606-d0bd9e447603"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:46.968288911Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1734,"":"downstairs","":"downstairs","session_id":"7af70976-c7dc-4c7c-9606-d0bd9e447603"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:46.968301738Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1734,"session_id":"7af70976-c7dc-4c7c-9606-d0bd9e447603"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:46.968316157Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1734,"client":"0","":"downstairs","session_id":"7af70976-c7dc-4c7c-9606-d0bd9e447603"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:46.968329244Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1734,"client":"1","":"downstairs","session_id":"7af70976-c7dc-4c7c-9606-d0bd9e447603"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:46.968341732Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1734,"client":"2","":"downstairs","session_id":"7af70976-c7dc-4c7c-9606-d0bd9e447603"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:46.968354499Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1734,"session_id":"7af70976-c7dc-4c7c-9606-d0bd9e447603"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:46.968366436Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1734,"session_id":"7af70976-c7dc-4c7c-9606-d0bd9e447603"} {"msg":"644fe92e-6c9b-4b5b-b6ba-7e6ed071d3f1 is now active with session: 7af70976-c7dc-4c7c-9606-d0bd9e447603","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:46.968378863Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1734,"session_id":"7af70976-c7dc-4c7c-9606-d0bd9e447603"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:46.9683904Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1734,"session_id":"7af70976-c7dc-4c7c-9606-d0bd9e447603"} {"msg":"The guest has finished waiting for activation with:138","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:46.968402347Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1734} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 373 len 40960 data: 14 16 18 17 18 17 17 15 12 10 02/30 Read block 523 len 4096 03/30 Read block 448 len 20480 04/30 Read block 101 len 20480 05/30 Write block 471 len 24576 data: 11 12 10 11 12 11 06/30 Read block 539 len 4096 07/30 Write block 147 len 4096 data: 9 08/30 Read block 377 len 28672 09/30 Read block 313 len 12288 10/30 Read block 254 len 12288 11/30 Read block 211 len 8192 12/30 Flush 13/30 Write block 524 len 16384 data: 15 13 13 11 14/30 Read block 537 len 20480 15/30 Write block 59 len 32768 data: 14 11 11 8 7 7 7 7 16/30 Write block 557 len 12288 data: 10 12 10 17/30 Read block 157 len 32768 18/30 Read block 552 len 20480 19/30 Read block 593 len 20480 20/30 Write block 260 len 16384 data: 10 9 8 8 21/30 Read block 141 len 32768 22/30 Write block 489 len 8192 data: 10 11 23/30 Write block 47 len 32768 data: 9 10 11 11 13 16 16 17 24/30 Read block 578 len 16384 25/30 Read block 116 len 28672 26/30 Read block 127 len 12288 27/30 Write block 13 len 4096 data: 13 28/30 Write block 312 len 28672 data: 6 7 6 5 7 8 9 29/30 Write block 391 len 32768 data: 8 9 8 8 7 9 10 10 30/30 Read block 251 len 12288 ---------------------------------------------------------------- Crucible gen:138 GIO:true work queues: Upstairs:0 downstairs:18 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1012 Acked Write 4 Done Done Done false 1013 Acked Read 5 Done Done Done false 1014 Acked Write 8 Done Done Done false 1015 Acked Write 3 Done Done Done false 1016 Acked Read 8 Done Done Done false 1017 Acked Read 5 Done Done Done false 1018 Acked Read 5 Done Done Done false 1019 Acked Write 4 Done Done Done false 1020 Acked Read 8 Done Done Done false 1021 Acked Write 2 Done Done Done false 1022 Acked Write 8 Done Done Done false 1023 Acked Read 4 Done Done Done false 1024 Acked Read 7 Done Done Done false 1025 Acked Read 3 Done Done Done false 1026 Acked Write 1 Done Done Done false 1027 Acked Write 7 Done Done Done false 1028 Acked Write 8 Done Done Sent false 1029 Acked Read 3 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 2 2 Done 18 18 16 52 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1011 1011 1011 Downstairs last five completed: 1011 1010 1009 1008 1007 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:16:46.988325001Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1734} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 116 116 116 210 210 210 T T T 1 020-039 131 131 131 241 241 241 F F F 2 040-059 137 137 137 251 251 251 T T T 3 060-079 128 128 128 236 236 236 T T T 4 080-099 137 137 137 251 251 251 F F F 5 100-119 137 137 137 251 251 251 F F F 6 120-139 134 134 134 247 247 247 F F F 7 140-159 138 138 138 252 252 252 F F F 8 160-179 137 137 137 251 251 251 F F F 9 180-199 134 134 134 246 246 246 F F F 10 200-219 137 137 137 251 251 251 F F F 11 220-239 137 137 137 251 251 251 F F F 12 240-259 137 137 137 251 251 251 F F F 13 260-279 137 137 137 251 251 251 T T T 14 280-299 134 134 134 247 247 247 F F F 15 300-319 131 131 131 241 241 241 T T T 16 320-339 123 123 123 227 227 227 F F F 17 340-359 136 136 136 250 250 250 F F F 18 360-379 138 138 138 252 252 252 F F F 19 380-399 138 138 138 252 252 252 T T T 20 400-419 137 137 137 251 251 251 F F F 21 420-439 137 137 137 251 251 251 F F F 22 440-459 137 137 137 251 251 251 F F F 23 460-479 138 138 138 252 252 252 F F F 24 480-499 134 134 134 246 246 246 T T T 25 500-519 134 134 134 246 246 246 F F F 26 520-539 134 134 134 247 247 247 T T T 27 540-559 127 127 127 234 234 234 T T T 28 560-579 137 137 137 251 251 251 F F F 29 580-599 134 134 134 247 247 247 F F F Max gen: 138, Max flush: 252 A Difference in extent metadata is expected here On loop 69 Verifying data now No change after verify Loop: 69 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 139 139 139 253 253 253 F F F 1 020-039 131 131 131 241 241 241 F F F 2 040-059 139 139 139 253 253 253 F F F 3 060-079 139 139 139 253 253 253 F F F 4 080-099 137 137 137 251 251 251 F F F 5 100-119 137 137 137 251 251 251 F F F 6 120-139 134 134 134 247 247 247 F F F 7 140-159 138 138 138 252 252 252 F F F 8 160-179 137 137 137 251 251 251 F F F 9 180-199 134 134 134 246 246 246 F F F 10 200-219 137 137 137 251 251 251 F F F 11 220-239 137 137 137 251 251 251 F F F 12 240-259 137 137 137 251 251 251 F F F 13 260-279 139 139 139 253 253 253 F F F 14 280-299 134 134 134 247 247 247 F F F 15 300-319 139 139 139 253 253 253 F F F 16 320-339 123 123 123 227 227 227 F F F 17 340-359 136 136 136 250 250 250 F F F 18 360-379 138 138 138 252 252 252 F F F 19 380-399 139 139 139 253 253 253 F F F 20 400-419 137 137 137 251 251 251 F F F 21 420-439 137 137 137 251 251 251 F F F 22 440-459 137 137 137 251 251 251 F F F 23 460-479 138 138 138 252 252 252 F F F 24 480-499 139 139 139 253 253 253 F F F 25 500-519 134 134 134 246 246 246 F F F 26 520-539 139 139 139 253 253 253 F F F 27 540-559 139 139 139 253 253 253 F F F 28 560-579 137 137 137 251 251 251 F F F 29 580-599 134 134 134 247 247 247 F F F Max gen: 139, Max flush: 253 Begin loop 70 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:51.245547496Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1742,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:51.28964616Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1742,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:51.290264142Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1742} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"abd5430f1508af0bc008143ea2566277ef714ec2\",\n git_commit_timestamp: \"2025-01-17T20:55:03.000000000Z\",\n git_branch: \"alan/tokio-to-140\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:51.290304535Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1742} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:51.290319504Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1742} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:51.290334643Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1742} {"msg":"Crucible d133762a-7ecc-4cf2-ac5d-a96b98e8d845 has session id: dc5ebedc-beff-4f33-9427-185649b485b3","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:51.290864413Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1742,"session_id":"dc5ebedc-beff-4f33-9427-185649b485b3"} {"msg":"Upstairs opts: Upstairs UUID: d133762a-7ecc-4cf2-ac5d-a96b98e8d845, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:51.290904946Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1742,"session_id":"dc5ebedc-beff-4f33-9427-185649b485b3"} {"msg":"Crucible stats registered with UUID: d133762a-7ecc-4cf2-ac5d-a96b98e8d845","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:51.29092879Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1742,"session_id":"dc5ebedc-beff-4f33-9427-185649b485b3"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:16:51.290950613Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1742,"session_id":"dc5ebedc-beff-4f33-9427-185649b485b3"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:51.291339881Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1742,"task":"crutest"} {"msg":"The guest has requested activation with gen:140","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:51.291369478Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1742} {"msg":"d133762a-7ecc-4cf2-ac5d-a96b98e8d845 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:51.291428542Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1742,"session_id":"dc5ebedc-beff-4f33-9427-185649b485b3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:51.291446463Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1742,"client":"0","":"downstairs","session_id":"dc5ebedc-beff-4f33-9427-185649b485b3"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:51.291469096Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1742,"client":"0","":"downstairs","session_id":"dc5ebedc-beff-4f33-9427-185649b485b3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:51.291491939Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1742,"client":"1","":"downstairs","session_id":"dc5ebedc-beff-4f33-9427-185649b485b3"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:51.291518705Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1742,"client":"1","":"downstairs","session_id":"dc5ebedc-beff-4f33-9427-185649b485b3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:51.29154507Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1742,"client":"2","":"downstairs","session_id":"dc5ebedc-beff-4f33-9427-185649b485b3"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:51.291574017Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1742,"client":"2","":"downstairs","session_id":"dc5ebedc-beff-4f33-9427-185649b485b3"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:51.291599973Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1742,"":"io task","client":"2","":"downstairs","session_id":"dc5ebedc-beff-4f33-9427-185649b485b3"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:51.291625708Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1742,"":"io task","client":"0","":"downstairs","session_id":"dc5ebedc-beff-4f33-9427-185649b485b3"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:51.291649752Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1742,"":"io task","client":"1","":"downstairs","session_id":"dc5ebedc-beff-4f33-9427-185649b485b3"} {"msg":"ds_connection connected from Ok(127.0.0.1:44191)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:51.291677148Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1742,"":"io task","client":"0","":"downstairs","session_id":"dc5ebedc-beff-4f33-9427-185649b485b3"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:16:51.291705804Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1742,"":"io task","client":"2","":"downstairs","session_id":"dc5ebedc-beff-4f33-9427-185649b485b3"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:16:51.291735172Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1742,"":"io task","client":"2","":"downstairs","session_id":"dc5ebedc-beff-4f33-9427-185649b485b3"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:16:51.291765859Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1742,"session_id":"dc5ebedc-beff-4f33-9427-185649b485b3"} {"msg":"ds_connection connected from Ok(127.0.0.1:53156)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:51.291797488Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1742,"":"io task","client":"1","":"downstairs","session_id":"dc5ebedc-beff-4f33-9427-185649b485b3"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:51.291893154Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1742,"":"io task","client":"2","":"downstairs","session_id":"dc5ebedc-beff-4f33-9427-185649b485b3"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:51.292405603Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1742,"client":"0","":"downstairs","session_id":"dc5ebedc-beff-4f33-9427-185649b485b3"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:51.29243374Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1742,"client":"1","":"downstairs","session_id":"dc5ebedc-beff-4f33-9427-185649b485b3"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:51.293063938Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1742,"client":"0","":"downstairs","session_id":"dc5ebedc-beff-4f33-9427-185649b485b3"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:51.293089723Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1742,"client":"1","":"downstairs","session_id":"dc5ebedc-beff-4f33-9427-185649b485b3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:51.293599081Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1742,"session_id":"dc5ebedc-beff-4f33-9427-185649b485b3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:16:51.293620594Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1742,"session_id":"dc5ebedc-beff-4f33-9427-185649b485b3"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:01.292289828Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1742,"":"io task","client":"2","":"downstairs","session_id":"dc5ebedc-beff-4f33-9427-185649b485b3"} {"msg":"ds_connection connected from Ok(127.0.0.1:59937)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:01.292611375Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1742,"":"io task","client":"2","":"downstairs","session_id":"dc5ebedc-beff-4f33-9427-185649b485b3"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:01.293677318Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1742,"client":"2","":"downstairs","session_id":"dc5ebedc-beff-4f33-9427-185649b485b3"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:01.294237786Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1742,"client":"2","":"downstairs","session_id":"dc5ebedc-beff-4f33-9427-185649b485b3"} {"msg":"[0]R flush_numbers[0..12]: [253, 241, 253, 253, 251, 251, 247, 252, 251, 246, 251, 251]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:01.294861831Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1742,"":"downstairs","":"downstairs","session_id":"dc5ebedc-beff-4f33-9427-185649b485b3"} {"msg":"[0]R generation[0..12]: [139, 131, 139, 139, 137, 137, 134, 138, 137, 134, 137, 137]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:01.294883793Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1742,"":"downstairs","":"downstairs","session_id":"dc5ebedc-beff-4f33-9427-185649b485b3"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:01.294902765Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1742,"":"downstairs","":"downstairs","session_id":"dc5ebedc-beff-4f33-9427-185649b485b3"} {"msg":"[1]R flush_numbers[0..12]: [253, 241, 253, 253, 251, 251, 247, 252, 251, 246, 251, 251]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:01.294916693Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1742,"":"downstairs","":"downstairs","session_id":"dc5ebedc-beff-4f33-9427-185649b485b3"} {"msg":"[1]R generation[0..12]: [139, 131, 139, 139, 137, 137, 134, 138, 137, 134, 137, 137]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:01.294930811Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1742,"":"downstairs","":"downstairs","session_id":"dc5ebedc-beff-4f33-9427-185649b485b3"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:01.294945039Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1742,"":"downstairs","":"downstairs","session_id":"dc5ebedc-beff-4f33-9427-185649b485b3"} {"msg":"[2]R flush_numbers[0..12]: [253, 241, 253, 253, 251, 251, 247, 252, 251, 246, 251, 251]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:01.294958617Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1742,"":"downstairs","":"downstairs","session_id":"dc5ebedc-beff-4f33-9427-185649b485b3"} {"msg":"[2]R generation[0..12]: [139, 131, 139, 139, 137, 137, 134, 138, 137, 134, 137, 137]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:01.294972025Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1742,"":"downstairs","":"downstairs","session_id":"dc5ebedc-beff-4f33-9427-185649b485b3"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:01.294985603Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1742,"":"downstairs","":"downstairs","session_id":"dc5ebedc-beff-4f33-9427-185649b485b3"} {"msg":"Max found gen is 140","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:01.295002293Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1742,"":"downstairs","":"downstairs","session_id":"dc5ebedc-beff-4f33-9427-185649b485b3"} {"msg":"Generation requested: 140 >= found:140","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:01.295023655Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1742,"":"downstairs","":"downstairs","session_id":"dc5ebedc-beff-4f33-9427-185649b485b3"} {"msg":"Next flush: 254","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:01.295037173Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1742,"":"downstairs","":"downstairs","session_id":"dc5ebedc-beff-4f33-9427-185649b485b3"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:01.295049661Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1742,"":"downstairs","":"downstairs","session_id":"dc5ebedc-beff-4f33-9427-185649b485b3"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:01.295062538Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1742,"session_id":"dc5ebedc-beff-4f33-9427-185649b485b3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-17T21:17:01.295074315Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1742,"client":"0","":"downstairs","session_id":"dc5ebedc-beff-4f33-9427-185649b485b3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:01.29510053Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1742,"client":"1","":"downstairs","session_id":"dc5ebedc-beff-4f33-9427-185649b485b3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:01.295129968Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1742,"client":"2","":"downstairs","session_id":"dc5ebedc-beff-4f33-9427-185649b485b3"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:01.295156473Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1742,"session_id":"dc5ebedc-beff-4f33-9427-185649b485b3"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:01.295180107Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1742,"session_id":"dc5ebedc-beff-4f33-9427-185649b485b3"} {"msg":"d133762a-7ecc-4cf2-ac5d-a96b98e8d845 is now active with session: dc5ebedc-beff-4f33-9427-185649b485b3","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:01.295206212Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1742,"session_id":"dc5ebedc-beff-4f33-9427-185649b485b3"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:01.295229686Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1742,"session_id":"dc5ebedc-beff-4f33-9427-185649b485b3"} {"msg":"The guest has finished waiting for activation with:140","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:01.29525433Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1742} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 119 len 36864 data: 12 11 8 8 7 5 8 6 8 02/30 Write block 511 len 4096 data: 16 03/30 Write block 272 len 40960 data: 8 8 8 9 9 10 8 7 8 11 04/30 Write block 530 len 20480 data: 7 10 11 11 10 05/30 Read block 53 len 28672 06/30 Read block 277 len 40960 07/30 Read block 167 len 4096 08/30 Write block 47 len 28672 data: 10 11 12 12 14 17 17 09/30 Write block 199 len 4096 data: 16 10/30 Write block 84 len 4096 data: 10 11/30 Write block 427 len 12288 data: 12 15 14 12/30 Read block 367 len 16384 13/30 Write block 103 len 16384 data: 15 15 15 12 14/30 Write block 347 len 8192 data: 8 10 15/30 Write block 107 len 36864 data: 10 10 11 12 17 18 15 16 16 16/30 Read block 484 len 28672 17/30 Write block 446 len 24576 data: 6 4 6 9 8 7 18/30 Read block 503 len 16384 19/30 Read block 280 len 28672 20/30 Write block 149 len 12288 data: 8 9 8 21/30 Write block 498 len 32768 data: 17 15 17 19 16 16 14 16 22/30 Read block 93 len 12288 23/30 Write block 397 len 12288 data: 11 11 9 24/30 Write block 570 len 36864 data: 12 11 9 10 11 9 11 11 10 25/30 Read block 382 len 40960 26/30 Read block 123 len 8192 27/30 Read block 263 len 40960 28/30 Write block 94 len 16384 data: 15 15 14 9 29/30 Write block 36 len 24576 data: 13 12 11 10 10 10 30/30 Write block 255 len 4096 data: 10 ---------------------------------------------------------------- Crucible gen:140 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 9 Done Done Done false 1001 Acked Write 1 Done Done Done false 1002 Acked Write 10 Done Done Done false 1003 Acked Write 5 Done Done Done false 1004 Acked Read 7 Done Done Done false 1005 Acked Read 10 Done Done Done false 1006 Acked Read 1 Done Done Done false 1007 Acked Write 7 Done Done Done false 1008 Acked Write 1 Done Done Done false 1009 Acked Write 1 Done Done Done false 1010 Acked Write 3 Done Done Done false 1011 Acked Read 4 Done Done Done false 1012 Acked Write 4 Done Done Done false 1013 Acked Write 2 Done Done Done false 1014 Acked Write 9 Done Done Done false 1015 Acked Read 7 Done Done Done false 1016 Acked Write 6 Done Done Done false 1017 Acked Read 4 Done Done Done false 1018 Acked Read 7 Done Done Done false 1019 Acked Write 3 Done Done Done false 1020 Acked Write 8 Done Done Done false 1021 Acked Read 3 Done Done Done false 1022 Acked Write 3 Done Done Done false 1023 Acked Write 9 Done Done Done false 1024 Acked Read 10 Done Done Done false 1025 Acked Read 2 Done Done Done false 1026 Acked Read 10 Done Done Done false 1027 Acked Write 4 Sent Sent Sent false 1028 Acked Write 6 Sent Sent Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 3 3 9 Done 27 27 27 81 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 0 0 0 Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:17:01.313175564Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1742} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 139 139 139 253 253 253 F F F 1 020-039 131 131 131 241 241 241 T T T 2 040-059 139 139 139 253 253 253 T T T 3 060-079 139 139 139 253 253 253 F F F 4 080-099 137 137 137 251 251 251 T T T 5 100-119 137 137 137 251 251 251 T T T 6 120-139 134 134 134 247 247 247 T T T 7 140-159 138 138 138 252 252 252 T T T 8 160-179 137 137 137 251 251 251 F F F 9 180-199 134 134 134 246 246 246 T T T 10 200-219 137 137 137 251 251 251 F F F 11 220-239 137 137 137 251 251 251 F F F 12 240-259 137 137 137 251 251 251 T T T 13 260-279 139 139 139 253 253 253 T T T 14 280-299 134 134 134 247 247 247 T T T 15 300-319 139 139 139 253 253 253 F F F 16 320-339 123 123 123 227 227 227 F F F 17 340-359 136 136 136 250 250 250 T T T 18 360-379 138 138 138 252 252 252 F F F 19 380-399 139 139 139 253 253 253 T T T 20 400-419 137 137 137 251 251 251 F F F 21 420-439 137 137 137 251 251 251 T T T 22 440-459 137 137 137 251 251 251 T T T 23 460-479 138 138 138 252 252 252 F F F 24 480-499 139 139 139 253 253 253 T T T 25 500-519 134 134 134 246 246 246 T T T 26 520-539 139 139 139 253 253 253 T T T 27 540-559 139 139 139 253 253 253 F F F 28 560-579 137 137 137 251 251 251 T T T 29 580-599 134 134 134 247 247 247 F F F Max gen: 139, Max flush: 253 A Difference in extent metadata is expected here On loop 70 Verifying data now No change after verify Loop: 70 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 139 139 139 253 253 253 F F F 1 020-039 140 140 140 254 254 254 F F F 2 040-059 140 140 140 254 254 254 F F F 3 060-079 139 139 139 253 253 253 F F F 4 080-099 140 140 140 254 254 254 F F F 5 100-119 140 140 140 254 254 254 F F F 6 120-139 140 140 140 254 254 254 F F F 7 140-159 140 140 140 254 254 254 F F F 8 160-179 137 137 137 251 251 251 F F F 9 180-199 140 140 140 254 254 254 F F F 10 200-219 137 137 137 251 251 251 F F F 11 220-239 137 137 137 251 251 251 F F F 12 240-259 140 140 140 254 254 254 F F F 13 260-279 140 140 140 254 254 254 F F F 14 280-299 140 140 140 254 254 254 F F F 15 300-319 139 139 139 253 253 253 F F F 16 320-339 123 123 123 227 227 227 F F F 17 340-359 140 140 140 254 254 254 F F F 18 360-379 138 138 138 252 252 252 F F F 19 380-399 140 140 140 254 254 254 F F F 20 400-419 137 137 137 251 251 251 F F F 21 420-439 140 140 140 254 254 254 F F F 22 440-459 140 140 140 254 254 254 F F F 23 460-479 138 138 138 252 252 252 F F F 24 480-499 140 140 140 254 254 254 F F F 25 500-519 140 140 140 254 254 254 F F F 26 520-539 140 140 140 254 254 254 F F F 27 540-559 139 139 139 253 253 253 F F F 28 560-579 140 140 140 254 254 254 F F F 29 580-599 134 134 134 247 247 247 F F F Max gen: 140, Max flush: 254 Begin loop 71 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:15.620279313Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1750,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:15.665024415Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1750,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:15.665677307Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1750} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"abd5430f1508af0bc008143ea2566277ef714ec2\",\n git_commit_timestamp: \"2025-01-17T20:55:03.000000000Z\",\n git_branch: \"alan/tokio-to-140\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:15.6657176Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1750} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:15.665731999Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1750} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:15.665746277Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1750} {"msg":"Crucible bea70c31-b0f6-496c-9976-2fa8931651db has session id: e571237c-a28d-4ae1-a070-b43fd8d3cd4e","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:15.666382549Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1750,"session_id":"e571237c-a28d-4ae1-a070-b43fd8d3cd4e"} {"msg":"Upstairs opts: Upstairs UUID: bea70c31-b0f6-496c-9976-2fa8931651db, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:15.666422632Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1750,"session_id":"e571237c-a28d-4ae1-a070-b43fd8d3cd4e"} {"msg":"Crucible stats registered with UUID: bea70c31-b0f6-496c-9976-2fa8931651db","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:15.666446186Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1750,"session_id":"e571237c-a28d-4ae1-a070-b43fd8d3cd4e"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:17:15.666467919Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1750,"session_id":"e571237c-a28d-4ae1-a070-b43fd8d3cd4e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:15.666916391Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1750,"task":"crutest"} {"msg":"The guest has requested activation with gen:142","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:15.666950121Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1750} {"msg":"bea70c31-b0f6-496c-9976-2fa8931651db active request set","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:15.667022223Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1750,"session_id":"e571237c-a28d-4ae1-a070-b43fd8d3cd4e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:15.667050519Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1750,"client":"0","":"downstairs","session_id":"e571237c-a28d-4ae1-a070-b43fd8d3cd4e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:15.66708614Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1750,"client":"0","":"downstairs","session_id":"e571237c-a28d-4ae1-a070-b43fd8d3cd4e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:15.667110154Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1750,"client":"1","":"downstairs","session_id":"e571237c-a28d-4ae1-a070-b43fd8d3cd4e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:15.667134779Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1750,"client":"1","":"downstairs","session_id":"e571237c-a28d-4ae1-a070-b43fd8d3cd4e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:15.667160164Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1750,"client":"2","":"downstairs","session_id":"e571237c-a28d-4ae1-a070-b43fd8d3cd4e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:15.667185428Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1750,"client":"2","":"downstairs","session_id":"e571237c-a28d-4ae1-a070-b43fd8d3cd4e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:15.667210563Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1750,"":"io task","client":"2","":"downstairs","session_id":"e571237c-a28d-4ae1-a070-b43fd8d3cd4e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:15.667234917Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1750,"":"io task","client":"0","":"downstairs","session_id":"e571237c-a28d-4ae1-a070-b43fd8d3cd4e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:15.66725785Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1750,"":"io task","client":"1","":"downstairs","session_id":"e571237c-a28d-4ae1-a070-b43fd8d3cd4e"} {"msg":"ds_connection connected from Ok(127.0.0.1:37714)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:15.667282395Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1750,"":"io task","client":"2","":"downstairs","session_id":"e571237c-a28d-4ae1-a070-b43fd8d3cd4e"} {"msg":"ds_connection connected from Ok(127.0.0.1:61100)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:15.66730778Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1750,"":"io task","client":"1","":"downstairs","session_id":"e571237c-a28d-4ae1-a070-b43fd8d3cd4e"} {"msg":"ds_connection connected from Ok(127.0.0.1:60682)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:15.667454836Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1750,"":"io task","client":"0","":"downstairs","session_id":"e571237c-a28d-4ae1-a070-b43fd8d3cd4e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:15.667953277Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1750,"client":"2","":"downstairs","session_id":"e571237c-a28d-4ae1-a070-b43fd8d3cd4e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:15.667980583Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1750,"client":"0","":"downstairs","session_id":"e571237c-a28d-4ae1-a070-b43fd8d3cd4e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:15.668602967Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1750,"client":"0","":"downstairs","session_id":"e571237c-a28d-4ae1-a070-b43fd8d3cd4e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:15.668641529Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1750,"client":"2","":"downstairs","session_id":"e571237c-a28d-4ae1-a070-b43fd8d3cd4e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:15.669215425Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1750,"session_id":"e571237c-a28d-4ae1-a070-b43fd8d3cd4e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:15.669252276Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1750,"session_id":"e571237c-a28d-4ae1-a070-b43fd8d3cd4e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:15.669501102Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1750,"client":"1","":"downstairs","session_id":"e571237c-a28d-4ae1-a070-b43fd8d3cd4e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:15.670041798Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1750,"client":"1","":"downstairs","session_id":"e571237c-a28d-4ae1-a070-b43fd8d3cd4e"} {"msg":"[0]R flush_numbers[0..12]: [253, 254, 254, 253, 254, 254, 254, 254, 251, 254, 251, 251]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:15.670530694Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1750,"":"downstairs","":"downstairs","session_id":"e571237c-a28d-4ae1-a070-b43fd8d3cd4e"} {"msg":"[0]R generation[0..12]: [139, 140, 140, 139, 140, 140, 140, 140, 137, 140, 137, 137]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:15.670553577Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1750,"":"downstairs","":"downstairs","session_id":"e571237c-a28d-4ae1-a070-b43fd8d3cd4e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:15.670568336Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1750,"":"downstairs","":"downstairs","session_id":"e571237c-a28d-4ae1-a070-b43fd8d3cd4e"} {"msg":"[1]R flush_numbers[0..12]: [253, 254, 254, 253, 254, 254, 254, 254, 251, 254, 251, 251]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:15.670582324Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1750,"":"downstairs","":"downstairs","session_id":"e571237c-a28d-4ae1-a070-b43fd8d3cd4e"} {"msg":"[1]R generation[0..12]: [139, 140, 140, 139, 140, 140, 140, 140, 137, 140, 137, 137]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:15.670595852Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1750,"":"downstairs","":"downstairs","session_id":"e571237c-a28d-4ae1-a070-b43fd8d3cd4e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:15.67060974Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1750,"":"downstairs","":"downstairs","session_id":"e571237c-a28d-4ae1-a070-b43fd8d3cd4e"} {"msg":"[2]R flush_numbers[0..12]: [253, 254, 254, 253, 254, 254, 254, 254, 251, 254, 251, 251]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:15.67062785Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1750,"":"downstairs","":"downstairs","session_id":"e571237c-a28d-4ae1-a070-b43fd8d3cd4e"} {"msg":"[2]R generation[0..12]: [139, 140, 140, 139, 140, 140, 140, 140, 137, 140, 137, 137]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:15.670641738Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1750,"":"downstairs","":"downstairs","session_id":"e571237c-a28d-4ae1-a070-b43fd8d3cd4e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:15.670655426Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1750,"":"downstairs","":"downstairs","session_id":"e571237c-a28d-4ae1-a070-b43fd8d3cd4e"} {"msg":"Max found gen is 141","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:15.670667863Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1750,"":"downstairs","":"downstairs","session_id":"e571237c-a28d-4ae1-a070-b43fd8d3cd4e"} {"msg":"Generation requested: 142 >= found:141","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:15.670680471Z","hostname":"Wait for a query_work_queue command to finish before sending IO w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1750,"":"downstairs","":"downstairs","session_id":"e571237c-a28d-4ae1-a070-b43fd8d3cd4e"} {"msg":"Next flush: 255","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:15.670704835Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1750,"":"downstairs","":"downstairs","session_id":"e571237c-a28d-4ae1-a070-b43fd8d3cd4e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:15.67072921Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1750,"":"downstairs","":"downstairs","session_id":"e571237c-a28d-4ae1-a070-b43fd8d3cd4e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:15.67074797Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1750,"session_id":"e571237c-a28d-4ae1-a070-b43fd8d3cd4e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:15.670760008Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1750,"client":"0","":"downstairs","session_id":"e571237c-a28d-4ae1-a070-b43fd8d3cd4e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:15.670772875Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1750,"client":"1","":"downstairs","session_id":"e571237c-a28d-4ae1-a070-b43fd8d3cd4e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:15.670785482Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1750,"client":"2","":"downstairs","session_id":"e571237c-a28d-4ae1-a070-b43fd8d3cd4e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:15.67079814Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1750,"session_id":"e571237c-a28d-4ae1-a070-b43fd8d3cd4e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:15.670809786Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1750,"session_id":"e571237c-a28d-4ae1-a070-b43fd8d3cd4e"} {"msg":"bea70c31-b0f6-496c-9976-2fa8931651db is now active with session: e571237c-a28d-4ae1-a070-b43fd8d3cd4e","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:15.670822164Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1750,"session_id":"e571237c-a28d-4ae1-a070-b43fd8d3cd4e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:15.6708336Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1750,"session_id":"e571237c-a28d-4ae1-a070-b43fd8d3cd4e"} {"msg":"The guest has finished waiting for activation with:142","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:15.670845558Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1750} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 429 len 24576 data: 15 13 12 11 8 11 02/30 Write block 378 len 40960 data: 18 18 16 13 11 11 11 9 9 9 03/30 Read block 438 len 32768 04/30 Read block 442 len 12288 05/30 Flush 06/30 Flush 07/30 Flush 08/30 Write block 549 len 32768 data: 11 12 9 7 5 6 7 8 09/30 Write block 576 len 24576 data: 12 12 11 12 11 12 10/30 Write block 517 len 16384 data: 15 13 14 12 11/30 Write block 49 len 4096 data: 13 12/30 Read block 344 len 28672 13/30 Read block 157 len 12288 14/30 Read block 23 len 36864 15/30 Write block 473 len 12288 data: 11 12 13 16/30 Read block 259 len 36864 17/30 Flush 18/30 Write block 142 len 32768 data: 11 9 8 8 7 10 8 9 19/30 Read block 207 len 4096 20/30 Read block 78 len 32768 21/30 Write block 64 len 40960 data: 8 8 8 7 8 8 8 12 11 10 22/30 Read block 318 len 8192 23/30 Write block 188 len 24576 data: 7 7 10 10 12 14 24/30 Write block 552 len 16384 data: 8 6 7 8 25/30 Write block 402 len 40960 data: 8 8 8 8 7 5 5 6 7 6 26/30 Flush 27/30 Write block 117 len 24576 data: 15 16 13 12 9 9 28/30 Write block 33 len 12288 data: 15 15 19 29/30 Read block 187 len 28672 30/30 Write block 488 len 4096 data: 8 ---------------------------------------------------------------- Crucible gen:142 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 6 Done Done Done false 1027 Acked Write 3 Done Done Done false 1028 Acked Read 7 Done Done Done false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:17:15.690133891Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1750} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 139 139 139 253 253 253 F F F 1 020-039 140 140 140 254 254 254 T T T 2 040-059 142 142 142 258 258 258 F F F 3 060-079 142 142 142 259 259 259 F F F 4 080-099 140 140 140 254 254 254 F F F 5 100-119 140 140 140 254 254 254 T T T 6 120-139 140 140 140 254 254 254 T T T 7 140-159 142 142 142 259 259 259 F F F 8 160-179 137 137 137 251 251 251 F F F 9 180-199 142 142 142 259 259 259 F F F 10 200-219 137 137 137 251 251 251 F F F 11 220-239 137 137 137 251 251 251 F F F 12 240-259 140 140 140 254 254 254 F F F 13 260-279 140 140 140 254 254 254 F F F 14 280-299 140 140 140 254 254 254 F F F 15 300-319 139 139 139 253 253 253 F F F 16 320-339 123 123 123 227 227 227 F F F 17 340-359 140 140 140 254 254 254 F F F 18 360-379 142 142 142 255 255 255 F F F 19 380-399 142 142 142 255 255 255 F F F 20 400-419 142 142 142 259 259 259 F F F 21 420-439 142 142 142 255 255 255 F F F 22 440-459 140 140 140 254 254 254 F F F 23 460-479 142 142 142 258 258 258 F F F 24 480-499 140 140 140 254 254 254 T T T 25 500-519 142 142 142 258 258 258 F F F 26 520-539 142 142 142 258 258 258 F F F 27 540-559 142 142 142 259 259 259 F F F 28 560-579 142 142 142 258 258 258 F F F 29 580-599 142 142 142 258 258 258 F F F Max gen: 142, Max flush: 259 A Difference in extent metadata is expected here On loop 71 Verifying data now No change after verify Loop: 71 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 139 139 139 253 253 253 F F F 1 020-039 143 143 143 260 260 260 F F F 2 040-059 142 142 142 258 258 258 F F F 3 060-079 142 142 142 259 259 259 F F F 4 080-099 140 140 140 254 254 254 F F F 5 100-119 143 143 143 260 260 260 F F F 6 120-139 143 143 143 260 260 260 F F F 7 140-159 142 142 142 259 259 259 F F F 8 160-179 137 137 137 251 251 251 F F F 9 180-199 142 142 142 259 259 259 F F F 10 200-219 137 137 137 251 251 251 F F F 11 220-239 137 137 137 251 251 251 F F F 12 240-259 140 140 140 254 254 254 F F F 13 260-279 140 140 140 254 254 254 F F F 14 280-299 140 140 140 254 254 254 F F F 15 300-319 139 139 139 253 253 253 F F F 16 320-339 123 123 123 227 227 227 F F F 17 340-359 140 140 140 254 254 254 F F F 18 360-379 142 142 142 255 255 255 F F F 19 380-399 142 142 142 255 255 255 F F F 20 400-419 142 142 142 259 259 259 F F F 21 420-439 142 142 142 255 255 255 F F F 22 440-459 140 140 140 254 254 254 F F F 23 460-479 142 142 142 258 258 258 F F F 24 480-499 143 143 143 260 260 260 F F F 25 500-519 142 142 142 258 258 258 F F F 26 520-539 142 142 142 258 258 258 F F F 27 540-559 142 142 142 259 259 259 F F F 28 560-579 142 142 142 258 258 258 F F F 29 580-599 142 142 142 258 258 258 F F F Max gen: 143, Max flush: 260 Begin loop 72 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:19.922263729Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1758,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:19.96669991Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1758,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:19.967368261Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1758} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"abd5430f1508af0bc008143ea2566277ef714ec2\",\n git_commit_timestamp: \"2025-01-17T20:55:03.000000000Z\",\n git_branch: \"alan/tokio-to-140\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:19.967409705Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1758} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:19.967424974Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1758} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:19.967440163Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1758} {"msg":"Crucible 60f2d3dd-2562-4762-a4e5-6fb1ef64ef5c has session id: ee14bd2b-bd3e-4111-9b77-3a0c4a20308a","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:19.967969102Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1758,"session_id":"ee14bd2b-bd3e-4111-9b77-3a0c4a20308a"} {"msg":"Upstairs opts: Upstairs UUID: 60f2d3dd-2562-4762-a4e5-6fb1ef64ef5c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:19.968007765Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1758,"session_id":"ee14bd2b-bd3e-4111-9b77-3a0c4a20308a"} {"msg":"Crucible stats registered with UUID: 60f2d3dd-2562-4762-a4e5-6fb1ef64ef5c","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:19.968032299Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1758,"session_id":"ee14bd2b-bd3e-4111-9b77-3a0c4a20308a"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:17:19.968051881Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1758,"session_id":"ee14bd2b-bd3e-4111-9b77-3a0c4a20308a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:19.968528839Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1758,"task":"crutest"} {"msg":"The guest has requested activation with gen:144","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:19.968552893Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1758} {"msg":"60f2d3dd-2562-4762-a4e5-6fb1ef64ef5c active request set","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:19.968619352Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1758,"session_id":"ee14bd2b-bd3e-4111-9b77-3a0c4a20308a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:19.968636652Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1758,"client":"0","":"downstairs","session_id":"ee14bd2b-bd3e-4111-9b77-3a0c4a20308a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:19.9686665Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1758,"client":"0","":"downstairs","session_id":"ee14bd2b-bd3e-4111-9b77-3a0c4a20308a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:19.968691825Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1758,"client":"1","":"downstairs","session_id":"ee14bd2b-bd3e-4111-9b77-3a0c4a20308a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:19.968720331Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1758,"client":"1","":"downstairs","session_id":"ee14bd2b-bd3e-4111-9b77-3a0c4a20308a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:19.968746967Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1758,"client":"2","":"downstairs","session_id":"ee14bd2b-bd3e-4111-9b77-3a0c4a20308a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:19.968773682Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1758,"client":"2","":"downstairs","session_id":"ee14bd2b-bd3e-4111-9b77-3a0c4a20308a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:19.968794855Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1758,"":"io task","client":"2","":"downstairs","session_id":"ee14bd2b-bd3e-4111-9b77-3a0c4a20308a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:19.968815937Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1758,"":"io task","client":"0","":"downstairs","session_id":"ee14bd2b-bd3e-4111-9b77-3a0c4a20308a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:19.968845865Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1758,"":"io task","client":"1","":"downstairs","session_id":"ee14bd2b-bd3e-4111-9b77-3a0c4a20308a"} {"msg":"ds_connection connected from Ok(127.0.0.1:43598)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:19.968873931Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1758,"":"io task","client":"2","":"downstairs","session_id":"ee14bd2b-bd3e-4111-9b77-3a0c4a20308a"} {"msg":"ds_connection connected from Ok(127.0.0.1:51552)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:19.969000655Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1758,"":"io task","client":"0","":"downstairs","session_id":"ee14bd2b-bd3e-4111-9b77-3a0c4a20308a"} {"msg":"ds_connection connected from Ok(127.0.0.1:61216)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:19.96904277Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1758,"":"io task","client":"1","":"downstairs","session_id":"ee14bd2b-bd3e-4111-9b77-3a0c4a20308a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:19.969498616Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1758,"client":"2","":"downstairs","session_id":"ee14bd2b-bd3e-4111-9b77-3a0c4a20308a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:19.969534417Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1758,"client":"0","":"downstairs","session_id":"ee14bd2b-bd3e-4111-9b77-3a0c4a20308a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:19.969748823Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1758,"client":"1","":"downstairs","session_id":"ee14bd2b-bd3e-4111-9b77-3a0c4a20308a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:19.970228353Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1758,"client":"0","":"downstairs","session_id":"ee14bd2b-bd3e-4111-9b77-3a0c4a20308a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:19.970315494Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1758,"client":"2","":"downstairs","session_id":"ee14bd2b-bd3e-4111-9b77-3a0c4a20308a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:19.970349494Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1758,"client":"1","":"downstairs","session_id":"ee14bd2b-bd3e-4111-9b77-3a0c4a20308a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:19.970861253Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1758,"session_id":"ee14bd2b-bd3e-4111-9b77-3a0c4a20308a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:19.970885427Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1758,"session_id":"ee14bd2b-bd3e-4111-9b77-3a0c4a20308a"} {"msg":"[0]R flush_numbers[0..12]: [253, 260, 258, 259, 254, 260, 260, 259, 251, 259, 251, 251]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:19.970906249Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1758,"":"downstairs","":"downstairs","session_id":"ee14bd2b-bd3e-4111-9b77-3a0c4a20308a"} {"msg":"[0]R generation[0..12]: [139, 143, 142, 142, 140, 143, 143, 142, 137, 142, 137, 137]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:19.970920757Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1758,"":"downstairs","":"downstairs","session_id":"ee14bd2b-bd3e-4111-9b77-3a0c4a20308a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:19.970938418Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1758,"":"downstairs","":"downstairs","session_id":"ee14bd2b-bd3e-4111-9b77-3a0c4a20308a"} {"msg":"[1]R flush_numbers[0..12]: [253, 260, 258, 259, 254, 260, 260, 259, 251, 259, 251, 251]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:19.970952816Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1758,"":"downstairs","":"downstairs","session_id":"ee14bd2b-bd3e-4111-9b77-3a0c4a20308a"} {"msg":"[1]R generation[0..12]: [139, 143, 142, 142, 140, 143, 143, 142, 137, 142, 137, 137]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:19.970966484Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1758,"":"downstairs","":"downstairs","session_id":"ee14bd2b-bd3e-4111-9b77-3a0c4a20308a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:19.970980202Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1758,"":"downstairs","":"downstairs","session_id":"ee14bd2b-bd3e-4111-9b77-3a0c4a20308a"} {"msg":"[2]R flush_numbers[0..12]: [253, 260, 258, 259, 254, 260, 260, 259, 251, 259, 251, 251]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:19.97099374Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1758,"":"downstairs","":"downstairs","session_id":"ee14bd2b-bd3e-4111-9b77-3a0c4a20308a"} {"msg":"[2]R generation[0..12]: [139, 143, 142, 142, 140, 143, 143, 142, 137, 142, 137, 137]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:19.971007398Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1758,"":"downstairs","":"downstairs","session_id":"ee14bd2b-bd3e-4111-9b77-3a0c4a20308a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:19.971021286Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1758,"":"downstairs","":"downstairs","session_id":"ee14bd2b-bd3e-4111-9b77-3a0c4a20308a"} {"msg":"Max found gen is 144","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:19.971033823Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1758,"":"downstairs","":"downstairs","session_id":"ee14bd2b-bd3e-4111-9b77-3a0c4a20308a"} {"msg":"Generation requested: 144 >= found:144","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:19.971046631Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1758,"":"downstairs","":"downstairs","session_id":"ee14bd2b-bd3e-4111-9b77-3a0c4a20308a"} {"msg":"Next flush: 261","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:19.971059088Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1758,"":"downstairs","":"downstairs","session_id":"ee14bd2b-bd3e-4111-9b77-3a0c4a20308a"} {"msg":"All extents match","v":0,"name":"crucible","Wait for a query_work_queue command to finish before sending IO level":30,"time":"2025-01-17T21:17:19.971078019Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1758,"":"downstairs","":"downstairs","session_id":"ee14bd2b-bd3e-4111-9b77-3a0c4a20308a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:19.971090867Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1758,"session_id":"ee14bd2b-bd3e-4111-9b77-3a0c4a20308a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:19.971102554Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1758,"client":"0","":"downstairs","session_id":"ee14bd2b-bd3e-4111-9b77-3a0c4a20308a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:19.971115511Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1758,"client":"1","":"downstairs","session_id":"ee14bd2b-bd3e-4111-9b77-3a0c4a20308a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:19.971128469Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1758,"client":"2","":"downstairs","session_id":"ee14bd2b-bd3e-4111-9b77-3a0c4a20308a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:19.971143207Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1758,"session_id":"ee14bd2b-bd3e-4111-9b77-3a0c4a20308a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:19.971156966Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1758,"session_id":"ee14bd2b-bd3e-4111-9b77-3a0c4a20308a"} {"msg":"60f2d3dd-2562-4762-a4e5-6fb1ef64ef5c is now active with session: ee14bd2b-bd3e-4111-9b77-3a0c4a20308a","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:19.971169623Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1758,"session_id":"ee14bd2b-bd3e-4111-9b77-3a0c4a20308a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:19.97118137Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1758,"session_id":"ee14bd2b-bd3e-4111-9b77-3a0c4a20308a"} {"msg":"The guest has finished waiting for activation with:144","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:19.971193477Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1758} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 88 len 12288 data: 12 15 18 02/30 Write block 286 len 4096 data: 9 03/30 Read block 34 len 16384 04/30 Read block 256 len 24576 05/30 Read block 61 len 40960 06/30 Write block 509 len 40960 data: 18 17 17 16 17 19 21 20 16 14 07/30 Write block 180 len 40960 data: 10 10 10 7 5 4 3 4 8 8 08/30 Read block 410 len 12288 09/30 Read block 254 len 12288 10/30 Write block 313 len 12288 data: 8 7 6 11/30 Write block 37 len 16384 data: 13 12 11 11 12/30 Read block 118 len 36864 13/30 Write block 379 len 4096 data: 19 14/30 Read block 54 len 32768 15/30 Read block 39 len 20480 16/30 Write block 331 len 20480 data: 9 8 7 11 11 17/30 Read block 336 len 20480 18/30 Write block 189 len 16384 data: 9 11 11 13 19/30 Read block 482 len 20480 20/30 Write block 280 len 36864 data: 9 12 11 8 9 9 10 10 12 21/30 Read block 47 len 4096 22/30 Read block 482 len 24576 23/30 Read block 290 len 20480 24/30 Read block 433 len 32768 25/30 Write block 2 len 28672 data: 9 8 9 10 9 10 9 26/30 Write block 481 len 24576 data: 11 10 8 9 9 9 27/30 Write block 286 len 12288 data: 11 11 13 28/30 Write block 259 len 28672 data: 13 11 10 9 9 7 8 29/30 Read block 557 len 36864 30/30 Write block 339 len 32768 data: 16 15 15 12 15 9 9 7 ---------------------------------------------------------------- Crucible gen:144 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 3 Done Done Done false 1001 Acked Write 1 Done Done Done false 1002 Acked Read 4 Done Done Done false 1003 Acked Read 6 Done Done Done false 1004 Acked Read 10 Done Done Done false 1005 Acked Write 10 Done Done Done false 1006 Acked Write 10 Done Done Done false 1007 Acked Read 3 Done Done Done false 1008 Acked Read 3 Done Done Done false 1009 Acked Write 3 Done Done Done false 1010 Acked Write 4 Done Done Done false 1011 Acked Read 9 Done Done Done false 1012 Acked Write 1 Done Done Done false 1013 Acked Read 8 Done Done Done false 1014 Acked Read 5 Done Done Done false 1015 Acked Write 5 Done Done Done false 1016 Acked Read 5 Done Done Done false 1017 Acked Write 4 Done Done Done false 1018 Acked Read 5 Done Done Done false 1019 Acked Write 9 Done Done Done false 1020 Acked Read 1 Done Done Done false 1021 Acked Read 6 Done Done Done false 1022 Acked Read 5 Done Done Done false 1023 Acked Read 8 Done Done Done false 1024 Acked Write 7 Done Done Done false 1025 Acked Write 6 Done Done Done false 1026 Acked Write 3 Done Done Done false 1027 Acked Write 7 Done Done Done false 1028 Acked Read 9 Done Done Done false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 29 29 29 87 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 0 0 0 Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:17:19.99070359Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1758} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 139 139 139 253 253 253 T T T 1 020-039 143 143 143 260 260 260 T T T 2 040-059 142 142 142 258 258 258 T T T 3 060-079 142 142 142 259 259 259 F F F 4 080-099 140 140 140 254 254 254 T T T 5 100-119 143 143 143 260 260 260 F F F 6 120-139 143 143 143 260 260 260 F F F 7 140-159 142 142 142 259 259 259 F F F 8 160-179 137 137 137 251 251 251 F F F 9 180-199 142 142 142 259 259 259 T T T 10 200-219 137 137 137 251 251 251 F F F 11 220-239 137 137 137 251 251 251 F F F 12 240-259 140 140 140 254 254 254 T T T 13 260-279 140 140 140 254 254 254 T T T 14 280-299 140 140 140 254 254 254 T T T 15 300-319 139 139 139 253 253 253 T T T 16 320-339 123 123 123 227 227 227 T T T 17 340-359 140 140 140 254 254 254 T T T 18 360-379 142 142 142 255 255 255 T T T 19 380-399 142 142 142 255 255 255 F F F 20 400-419 142 142 142 259 259 259 F F F 21 420-439 142 142 142 255 255 255 F F F 22 440-459 140 140 140 254 254 254 F F F 23 460-479 142 142 142 258 258 258 F F F 24 480-499 143 143 143 260 260 260 T T T 25 500-519 142 142 142 258 258 258 T T T 26 520-539 142 142 142 258 258 258 F F F 27 540-559 142 142 142 259 259 259 F F F 28 560-579 142 142 142 258 258 258 F F F 29 580-599 142 142 142 258 258 258 F F F Max gen: 143, Max flush: 260 A Difference in extent metadata is expected here On loop 72 Verifying data now No change after verify Loop: 72 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 144 144 144 261 261 261 F F F 1 020-039 144 144 144 261 261 261 F F F 2 040-059 144 144 144 261 261 261 F F F 3 060-079 142 142 142 259 259 259 F F F 4 080-099 144 144 144 261 261 261 F F F 5 100-119 143 143 143 260 260 260 F F F 6 120-139 143 143 143 260 260 260 F F F 7 140-159 142 142 142 259 259 259 F F F 8 160-179 137 137 137 251 251 251 F F F 9 180-199 144 144 144 261 261 261 F F F 10 200-219 137 137 137 251 251 251 F F F 11 220-239 137 137 137 251 251 251 F F F 12 240-259 144 144 144 261 261 261 F F F 13 260-279 144 144 144 261 261 261 F F F 14 280-299 144 144 144 261 261 261 F F F 15 300-319 144 144 144 261 261 261 F F F 16 320-339 144 144 144 261 261 261 F F F 17 340-359 144 144 144 261 261 261 F F F 18 360-379 144 144 144 261 261 261 F F F 19 380-399 142 142 142 255 255 255 F F F 20 400-419 142 142 142 259 259 259 F F F 21 420-439 142 142 142 255 255 255 F F F 22 440-459 140 140 140 254 254 254 F F F 23 460-479 142 142 142 258 258 258 F F F 24 480-499 144 144 144 261 261 261 F F F 25 500-519 144 144 144 261 261 261 F F F 26 520-539 142 142 142 258 258 258 F F F 27 540-559 142 142 142 259 259 259 F F F 28 560-579 142 142 142 258 258 258 F F F 29 580-599 142 142 142 258 258 258 F F F Max gen: 144, Max flush: 261 Begin loop 73 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:24.284874699Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1766,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:24.328703786Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1766,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:24.329331153Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1766} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"abd5430f1508af0bc008143ea2566277ef714ec2\",\n git_commit_timestamp: \"2025-01-17T20:55:03.000000000Z\",\n git_branch: \"alan/tokio-to-140\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:24.329372046Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1766} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:24.329386395Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1766} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:24.329401854Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1766} {"msg":"Crucible 8c4f80f0-e9e9-4d91-aba2-8c0a4ae52767 has session id: 735f635a-5c75-42eb-8380-7b60b3fcad64","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:24.329982273Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1766,"session_id":"735f635a-5c75-42eb-8380-7b60b3fcad64"} {"msg":"Upstairs opts: Upstairs UUID: 8c4f80f0-e9e9-4d91-aba2-8c0a4ae52767, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:24.330022797Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1766,"session_id":"735f635a-5c75-42eb-8380-7b60b3fcad64"} {"msg":"Crucible stats registered with UUID: 8c4f80f0-e9e9-4d91-aba2-8c0a4ae52767","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:24.330046441Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1766,"session_id":"735f635a-5c75-42eb-8380-7b60b3fcad64"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:17:24.33006274Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1766,"session_id":"735f635a-5c75-42eb-8380-7b60b3fcad64"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:24.330630502Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1766,"task":"crutest"} {"msg":"The guest has requested activation with gen:146","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:24.330664062Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1766} {"msg":"8c4f80f0-e9e9-4d91-aba2-8c0a4ae52767 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:24.330736254Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1766,"session_id":"735f635a-5c75-42eb-8380-7b60b3fcad64"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:24.330761358Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1766,"client":"0","":"downstairs","session_id":"735f635a-5c75-42eb-8380-7b60b3fcad64"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:24.330787074Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1766,"client":"0","":"downstairs","session_id":"735f635a-5c75-42eb-8380-7b60b3fcad64"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:24.330800902Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1766,"client":"1","":"downstairs","session_id":"735f635a-5c75-42eb-8380-7b60b3fcad64"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:24.3308149Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1766,"client":"1","":"downstairs","session_id":"735f635a-5c75-42eb-8380-7b60b3fcad64"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:24.330891805Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1766,"client":"2","":"downstairs","session_id":"735f635a-5c75-42eb-8380-7b60b3fcad64"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:24.331005181Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1766,"client":"2","":"downstairs","session_id":"735f635a-5c75-42eb-8380-7b60b3fcad64"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:24.331035539Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1766,"":"io task","client":"2","":"downstairs","session_id":"735f635a-5c75-42eb-8380-7b60b3fcad64"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:24.331062675Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1766,"":"io task","client":"0","":"downstairs","session_id":"735f635a-5c75-42eb-8380-7b60b3fcad64"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:24.33108945Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1766,"":"io task","client":"1","":"downstairs","session_id":"735f635a-5c75-42eb-8380-7b60b3fcad64"} {"msg":"ds_connection connected from Ok(127.0.0.1:61419)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:24.331118027Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1766,"":"io task","client":"2","":"downstairs","session_id":"735f635a-5c75-42eb-8380-7b60b3fcad64"} {"msg":"ds_connection connected from Ok(127.0.0.1:55085)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:24.331192441Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1766,"":"io task","client":"1","":"downstairs","session_id":"735f635a-5c75-42eb-8380-7b60b3fcad64"} {"msg":"ds_connection connected from Ok(127.0.0.1:63996)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:24.33122528Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1766,"":"io task","client":"0","":"downstairs","session_id":"735f635a-5c75-42eb-8380-7b60b3fcad64"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:24.331701648Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1766,"client":"0","":"downstairs","session_id":"735f635a-5c75-42eb-8380-7b60b3fcad64"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:24.331845602Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1766,"client":"1","":"downstairs","session_id":"735f635a-5c75-42eb-8380-7b60b3fcad64"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:24.332186802Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1766,"client":"0","":"downstairs","session_id":"735f635a-5c75-42eb-8380-7b60b3fcad64"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:24.332322611Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1766,"client":"1","":"downstairs","session_id":"735f635a-5c75-42eb-8380-7b60b3fcad64"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:24.332744228Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1766,"session_id":"735f635a-5c75-42eb-8380-7b60b3fcad64"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:24.332824144Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1766,"session_id":"735f635a-5c75-42eb-8380-7b60b3fcad64"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:24.333074921Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1766,"client":"2","":"downstairs","session_id":"735f635a-5c75-42eb-8380-7b60b3fcad64"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:24.333575083Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1766,"client":"2","":"downstairs","session_id":"735f635a-5c75-42eb-8380-7b60b3fcad64"} {"msg":"[0]R flush_numbers[0..12]: [261, 261, 261, 259, 261, 260, 260, 259, 251, 261, 251, 251]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:24.334061858Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1766,"":"downstairs","":"downstairs","session_id":"735f635a-5c75-42eb-8380-7b60b3fcad64"} {"msg":"[0]R generation[0..12]: [144, 144, 144, 142, 144, 143, 143, 142, 137, 144, 137, 137]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:24.334080799Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1766,"":"downstairs","":"downstairs","session_id":"735f635a-5c75-42eb-8380-7b60b3fcad64"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:24.334095547Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1766,"":"downstairs","":"downstairs","session_id":"735f635a-5c75-42eb-8380-7b60b3fcad64"} {"msg":"[1]R flush_numbers[0..12]: [261, 261, 261, 259, 261, 260, 260, 259, 251, 261, 251, 251]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:24.334109305Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1766,"":"downstairs","":"downstairs","session_id":"735f635a-5c75-42eb-8380-7b60b3fcad64"} {"msg":"[1]R generation[0..12]: [144, 144, 144, 142, 144, 143, 143, 142, 137, 144, 137, 137]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:24.334122733Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1766,"":"downstairs","":"downstairs","session_id":"735f635a-5c75-42eb-8380-7b60b3fcad64"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:24.334136371Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1766,"":"downstairs","":"downstairs","session_id":"735f635a-5c75-42eb-8380-7b60b3fcad64"} {"msg":"[2]R flush_numbers[0..12]: [261, 261, 261, 259, 261, 260, 260, 259, 251, 261, 251, 251]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:24.334149779Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1766,"":"downstairs","":"downstairs","session_id":"735f635a-5c75-42eb-8380-7b60b3fcad64"} {"msg":"[2]R generation[0..12]: [144, 144, 144, 142, 144, 143, 143, 142, 137, 144, 137, 137]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:24.334166339Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1766,"":"downstairs","":"downstairs","session_id":"735f635a-5c75-42eb-8380-7b60b3fcad64"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:24.334180137Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1766,"":"downstairs","":"downstairs","session_id":"735f635a-5c75-42eb-8380-7b60b3fcad64"} {"msg":"Max found gen is 145","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:24.334192574Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1766,"":"downstairs","":"downstairs","session_id":"735f635a-5c75-42eb-8380-7b60b3fcad64"} {"msg":"Generation requested: 146 >= found:145","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:24.334205271Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1766,"":"downstairs","":"downstairs","session_id":"735f635a-5c75-42eb-8380-7b60b3fcad64"} {"msg":"Next flush: 262","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:24.334217709Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1766,"":"downstairs","Wait for a query_work_queue command to finish before sending IO ":"downstairs","session_id":"735f635a-5c75-42eb-8380-7b60b3fcad64"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:24.33423688Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1766,"":"downstairs","":"downstairs","session_id":"735f635a-5c75-42eb-8380-7b60b3fcad64"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:24.334250818Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1766,"session_id":"735f635a-5c75-42eb-8380-7b60b3fcad64"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:24.334262495Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1766,"client":"0","":"downstairs","session_id":"735f635a-5c75-42eb-8380-7b60b3fcad64"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:24.334275443Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1766,"client":"1","":"downstairs","session_id":"735f635a-5c75-42eb-8380-7b60b3fcad64"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:24.33428807Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1766,"client":"2","":"downstairs","session_id":"735f635a-5c75-42eb-8380-7b60b3fcad64"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:24.334300777Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1766,"session_id":"735f635a-5c75-42eb-8380-7b60b3fcad64"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:24.334312774Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1766,"session_id":"735f635a-5c75-42eb-8380-7b60b3fcad64"} {"msg":"8c4f80f0-e9e9-4d91-aba2-8c0a4ae52767 is now active with session: 735f635a-5c75-42eb-8380-7b60b3fcad64","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:24.334325001Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1766,"session_id":"735f635a-5c75-42eb-8380-7b60b3fcad64"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:24.334337249Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1766,"session_id":"735f635a-5c75-42eb-8380-7b60b3fcad64"} {"msg":"The guest has finished waiting for activation with:146","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:24.334349155Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1766} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 395 len 36864 data: 8 10 12 12 10 9 10 9 9 02/30 Read block 472 len 24576 03/30 Write block 596 len 4096 data: 5 04/30 Write block 318 len 20480 data: 10 10 11 10 8 05/30 Write block 63 len 36864 data: 8 9 9 9 8 9 9 9 13 06/30 Write block 157 len 24576 data: 12 11 11 10 12 12 07/30 Read block 513 len 8192 08/30 Write block 174 len 4096 data: 11 09/30 Read block 583 len 28672 10/30 Read block 136 len 40960 11/30 Write block 4 len 32768 data: 10 11 10 11 10 9 11 13 12/30 Read block 111 len 8192 13/30 Read block 59 len 12288 14/30 Read block 171 len 12288 15/30 Read block 360 len 40960 16/30 Read block 177 len 20480 17/30 Write block 497 len 8192 data: 15 18 18/30 Read block 74 len 24576 19/30 Write block 235 len 4096 data: 15 20/30 Read block 284 len 8192 21/30 Write block 125 len 24576 data: 9 7 9 9 10 11 22/30 Read block 146 len 16384 23/30 Read block 392 len 20480 24/30 Write block 14 len 4096 data: 12 25/30 Write block 205 len 8192 data: 8 7 26/30 Write block 187 len 4096 data: 5 27/30 Read block 341 len 32768 28/30 Write block 540 len 8192 data: 10 10 29/30 Write block 312 len 12288 data: 7 9 8 30/30 Write block 363 len 40960 data: 12 9 7 6 5 5 5 9 10 14 ---------------------------------------------------------------- Crucible gen:146 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 9 Done Done Done false 1001 Acked Read 6 Done Done Done false 1002 Acked Write 1 Done Done Done false 1003 Acked Write 5 Done Done Done false 1004 Acked Write 9 Done Done Done false 1005 Acked Write 6 Done Done Done false 1006 Acked Read 2 Done Done Done false 1007 Acked Write 1 Done Done Done false 1008 Acked Read 7 Done Done Done false 1009 Acked Read 10 Done Done Done false 1010 Acked Write 8 Done Done Done false 1011 Acked Read 2 Done Done Done false 1012 Acked Read 3 Done Done Done false 1013 Acked Read 3 Done Done Done false 1014 Acked Read 10 Done Done Done false 1015 Acked Read 5 Done Done Done false 1016 Acked Write 2 Done Done Done false 1017 Acked Read 6 Done Done Done false 1018 Acked Write 1 Done Done Done false 1019 Acked Read 2 Done Done Done false 1020 Acked Write 6 Done Done Done false 1021 Acked Read 4 Done Done Done false 1022 Acked Read 5 Done Done Done false 1023 Acked Write 1 Done Done Done false 1024 Acked Write 2 Done Done Done false 1025 Acked Write 1 Done Done Done false 1026 Acked Read 8 Done Done Done false 1027 Acked Write 2 Done Done Done false 1028 Acked Write 3 Done Done Done false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 29 29 29 87 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 0 0 0 Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:17:24.351887917Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1766} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 144 144 144 261 261 261 T T T 1 020-039 144 144 144 261 261 261 F F F 2 040-059 144 144 144 261 261 261 F F F 3 060-079 142 142 142 259 259 259 T T T 4 080-099 144 144 144 261 261 261 F F F 5 100-119 143 143 143 260 260 260 F F F 6 120-139 143 143 143 260 260 260 T T T 7 140-159 142 142 142 259 259 259 T T T 8 160-179 137 137 137 251 251 251 T T T 9 180-199 144 144 144 261 261 261 T T T 10 200-219 137 137 137 251 251 251 T T T 11 220-239 137 137 137 251 251 251 T T T 12 240-259 144 144 144 261 261 261 F F F 13 260-279 144 144 144 261 261 261 F F F 14 280-299 144 144 144 261 261 261 F F F 15 300-319 144 144 144 261 261 261 T T T 16 320-339 144 144 144 261 261 261 T T T 17 340-359 144 144 144 261 261 261 F F F 18 360-379 144 144 144 261 261 261 T T T 19 380-399 142 142 142 255 255 255 T T T 20 400-419 142 142 142 259 259 259 T T T 21 420-439 142 142 142 255 255 255 F F F 22 440-459 140 140 140 254 254 254 F F F 23 460-479 142 142 142 258 258 258 F F F 24 480-499 144 144 144 261 261 261 T T T 25 500-519 144 144 144 261 261 261 F F F 26 520-539 142 142 142 258 258 258 F F F 27 540-559 142 142 142 259 259 259 T T T 28 560-579 142 142 142 258 258 258 F F F 29 580-599 142 142 142 258 258 258 T T T Max gen: 144, Max flush: 261 A Difference in extent metadata is expected here On loop 73 Verifying data now No change after verify Loop: 73 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 145 145 145 262 262 262 F F F 1 020-039 144 144 144 261 261 261 F F F 2 040-059 144 144 144 261 261 261 F F F 3 060-079 145 145 145 262 262 262 F F F 4 080-099 144 144 144 261 261 261 F F F 5 100-119 143 143 143 260 260 260 F F F 6 120-139 145 145 145 262 262 262 F F F 7 140-159 145 145 145 262 262 262 F F F 8 160-179 145 145 145 262 262 262 F F F 9 180-199 145 145 145 262 262 262 F F F 10 200-219 145 145 145 262 262 262 F F F 11 220-239 145 145 145 262 262 262 F F F 12 240-259 144 144 144 261 261 261 F F F 13 260-279 144 144 144 261 261 261 F F F 14 280-299 144 144 144 261 261 261 F F F 15 300-319 145 145 145 262 262 262 F F F 16 320-339 145 145 145 262 262 262 F F F 17 340-359 144 144 144 261 261 261 F F F 18 360-379 145 145 145 262 262 262 F F F 19 380-399 145 145 145 262 262 262 F F F 20 400-419 145 145 145 262 262 262 F F F 21 420-439 142 142 142 255 255 255 F F F 22 440-459 140 140 140 254 254 254 F F F 23 460-479 142 142 142 258 258 258 F F F 24 480-499 145 145 145 262 262 262 F F F 25 500-519 144 144 144 261 261 261 F F F 26 520-539 142 142 142 258 258 258 F F F 27 540-559 145 145 145 262 262 262 F F F 28 560-579 142 142 142 258 258 258 F F F 29 580-599 145 145 145 262 262 262 F F F Max gen: 145, Max flush: 262 Begin loop 74 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:28.657585804Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1774,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:17:28.701264683Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1774,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:28.701675143Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1774,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:28.734310697Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1774,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:28.734933991Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1774} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"abd5430f1508af0bc008143ea2566277ef714ec2\",\n git_commit_timestamp: \"2025-01-17T20:55:03.000000000Z\",\n git_branch: \"alan/tokio-to-140\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:28.734967731Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1774} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:28.734981169Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1774} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:28.734996058Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1774} {"msg":"Crucible 77b4e6ee-22ad-4cbc-a2e1-a3a376bf689c has session id: 9eeaddbf-01cc-45ab-9f20-ff5b3c56c7b4","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:28.735578098Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1774,"session_id":"9eeaddbf-01cc-45ab-9f20-ff5b3c56c7b4"} {"msg":"Upstairs opts: Upstairs UUID: 77b4e6ee-22ad-4cbc-a2e1-a3a376bf689c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:28.735606304Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1774,"session_id":"9eeaddbf-01cc-45ab-9f20-ff5b3c56c7b4"} {"msg":"Crucible stats registered with UUID: 77b4e6ee-22ad-4cbc-a2e1-a3a376bf689c","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:28.735620653Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1774,"session_id":"9eeaddbf-01cc-45ab-9f20-ff5b3c56c7b4"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:17:28.73563313Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1774,"session_id":"9eeaddbf-01cc-45ab-9f20-ff5b3c56c7b4"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:28.73607763Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1774,"task":"crutest"} {"msg":"The guest has requested activation with gen:148","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:28.736110239Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1774} {"msg":"77b4e6ee-22ad-4cbc-a2e1-a3a376bf689c active request set","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:28.736164231Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1774,"session_id":"9eeaddbf-01cc-45ab-9f20-ff5b3c56c7b4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:28.736190676Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1774,"client":"0","":"downstairs","session_id":"9eeaddbf-01cc-45ab-9f20-ff5b3c56c7b4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:28.736217001Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1774,"client":"0","":"downstairs","session_id":"9eeaddbf-01cc-45ab-9f20-ff5b3c56c7b4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:28.736234612Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1774,"client":"1","":"downstairs","session_id":"9eeaddbf-01cc-45ab-9f20-ff5b3c56c7b4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:28.736250211Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1774,"client":"1","":"downstairs","session_id":"9eeaddbf-01cc-45ab-9f20-ff5b3c56c7b4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:28.736263839Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1774,"client":"2","":"downstairs","session_id":"9eeaddbf-01cc-45ab-9f20-ff5b3c56c7b4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:28.736277627Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1774,"client":"2","":"downstairs","session_id":"9eeaddbf-01cc-45ab-9f20-ff5b3c56c7b4"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:28.736290234Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1774,"":"io task","client":"2","":"downstairs","session_id":"9eeaddbf-01cc-45ab-9f20-ff5b3c56c7b4"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:28.736312868Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1774,"":"io task","client":"0","":"downstairs","session_id":"9eeaddbf-01cc-45ab-9f20-ff5b3c56c7b4"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:28.736339633Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1774,"":"io task","client":"1","":"downstairs","session_id":"9eeaddbf-01cc-45ab-9f20-ff5b3c56c7b4"} {"msg":"ds_connection connected from Ok(127.0.0.1:45470)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:28.736365358Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1774,"":"io task","client":"2","":"downstairs","session_id":"9eeaddbf-01cc-45ab-9f20-ff5b3c56c7b4"} {"msg":"ds_connection connected from Ok(127.0.0.1:63673)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:28.736392104Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1774,"":"io task","client":"0","":"downstairs","session_id":"9eeaddbf-01cc-45ab-9f20-ff5b3c56c7b4"} {"msg":"ds_connection connected from Ok(127.0.0.1:40108)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:28.736416708Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1774,"":"io task","client":"1","":"downstairs","session_id":"9eeaddbf-01cc-45ab-9f20-ff5b3c56c7b4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:28.737095185Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1774,"client":"1","":"downstairs","session_id":"9eeaddbf-01cc-45ab-9f20-ff5b3c56c7b4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:28.737122931Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1774,"client":"2","":"downstairs","session_id":"9eeaddbf-01cc-45ab-9f20-ff5b3c56c7b4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:28.737217406Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1774,"client":"0","":"downstairs","session_id":"9eeaddbf-01cc-45ab-9f20-ff5b3c56c7b4"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:28.737733348Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1774,"client":"1","":"downstairs","session_id":"9eeaddbf-01cc-45ab-9f20-ff5b3c56c7b4"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:28.737853648Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1774,"client":"0","":"downstairs","session_id":"9eeaddbf-01cc-45ab-9f20-ff5b3c56c7b4"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:28.73789057Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1774,"client":"2","":"downstairs","session_id":"9eeaddbf-01cc-45ab-9f20-ff5b3c56c7b4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:28.738241955Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1774,"session_id":"9eeaddbf-01cc-45ab-9f20-ff5b3c56c7b4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:28.73840489Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1774,"session_id":"9eeaddbf-01cc-45ab-9f20-ff5b3c56c7b4"} {"msg":"[0]R flush_numbers[0..12]: [262, 261, 261, 262, 261, 260, 262, 262, 262, 262, 262, 262]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:28.738520218Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1774,"":"downstairs","":"downstairs","session_id":"9eeaddbf-01cc-45ab-9f20-ff5b3c56c7b4"} {"msg":"[0]R generation[0..12]: [145, 144, 144, 145, 144, 143, 145, 145, 145, 145, 145, 145]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:28.738539929Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1774,"":"downstairs","":"downstairs","session_id":"9eeaddbf-01cc-45ab-9f20-ff5b3c56c7b4"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:28.738554458Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1774,"":"downstairs","":"downstairs","session_id":"9eeaddbf-01cc-45ab-9f20-ff5b3c56c7b4"} {"msg":"[1]R flush_numbers[0..12]: [262, 261, 261, 262, 261, 260, 262, 262, 262, 262, 262, 262]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:28.738568246Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1774,"":"downstairs","":"downstairs","session_id":"9eeaddbf-01cc-45ab-9f20-ff5b3c56c7b4"} {"msg":"[1]R generation[0..12]: [145, 144, 144, 145, 144, 143, 145, 145, 145, 145, 145, 145]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:28.738581683Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1774,"":"downstairs","":"downstairs","session_id":"9eeaddbf-01cc-45ab-9f20-ff5b3c56c7b4"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:28.738595191Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1774,"":"downstairs","":"downstairs","session_id":"9eeaddbf-01cc-45ab-9f20-ff5b3c56c7b4"} {"msg":"[2]R flush_numbers[0..12]: [262, 261, 261, 262, 261, 260, 262, 262, 262, 262, 262, 262]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:28.738608409Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1774,"":"downstairs","":"downstairs","session_id":"9eeaddbf-01cc-45ab-9f20-ff5b3c56c7b4"} {"msg":"[2]R generation[0..12]: [145, 144, 144, 145, 144, 143, 145, 145, 145, 145, 145, 145]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:28.738621727Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1774,"":"downstairs","":"downstairs","session_id":"9eeaddbf-01cc-45ab-9f20-ff5b3c56c7b4"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:28.738635175Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1774,"":"downstairs","":"downstairs","session_id":"9eeaddbf-01cc-45ab-9f20-ff5b3c56c7b4"} {"msg":"Max found gen is 146","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:28.738647632Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1774,"":"downstairs","":"downstairs","session_id":"9eeaddbf-01cc-45ab-9f20-ff5b3c56c7b4"} {"msg":"Generation requested: 148 >= found:146","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:28.738663221Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1774,"":"downstairs","":"downstairs","session_id":"9eeaddbf-01cc-45ab-9f20-ff5b3c56c7b4"} {"msg":"Next flush: 263","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-17T21:17:28.738676069Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1774,"":"downstairs","":"downstairs","session_id":"9eeaddbf-01cc-45ab-9f20-ff5b3c56c7b4"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:28.73869525Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1774,"":"downstairs","":"downstairs","session_id":"9eeaddbf-01cc-45ab-9f20-ff5b3c56c7b4"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:28.738708268Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1774,"session_id":"9eeaddbf-01cc-45ab-9f20-ff5b3c56c7b4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:28.738719744Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1774,"client":"0","":"downstairs","session_id":"9eeaddbf-01cc-45ab-9f20-ff5b3c56c7b4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:28.738733072Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1774,"client":"1","":"downstairs","session_id":"9eeaddbf-01cc-45ab-9f20-ff5b3c56c7b4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:28.73874581Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1774,"client":"2","":"downstairs","session_id":"9eeaddbf-01cc-45ab-9f20-ff5b3c56c7b4"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:28.738758707Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1774,"session_id":"9eeaddbf-01cc-45ab-9f20-ff5b3c56c7b4"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:28.738770494Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1774,"session_id":"9eeaddbf-01cc-45ab-9f20-ff5b3c56c7b4"} {"msg":"77b4e6ee-22ad-4cbc-a2e1-a3a376bf689c is now active with session: 9eeaddbf-01cc-45ab-9f20-ff5b3c56c7b4","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:28.738782961Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1774,"session_id":"9eeaddbf-01cc-45ab-9f20-ff5b3c56c7b4"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:28.738794488Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1774,"session_id":"9eeaddbf-01cc-45ab-9f20-ff5b3c56c7b4"} {"msg":"The guest has finished waiting for activation with:148","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:28.738806505Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1774} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 523 len 36864 data: 16 16 14 14 12 10 8 8 11 03/30 Write block 48 len 20480 data: 12 14 13 15 18 04/30 Flush 05/30 Write block 233 len 24576 data: 13 14 16 13 14 9 06/30 Read block 251 len 32768 07/30 Read block 273 len 16384 08/30 Read block 143 len 4096 09/30 Read block 228 len 20480 10/30 Read block 279 len 4096 11/30 Write block 116 len 36864 data: 14 16 17 14 13 10 10 8 6 12/30 Flush 13/30 Write block 490 len 12288 data: 12 11 9 14/30 Write block 15 len 4096 data: 10 15/30 Read block 222 len 8192 16/30 Write block 470 len 40960 data: 12 12 13 12 13 14 12 10 9 8 17/30 Read block 144 len 24576 18/30 Flush 19/30 Write block 68 len 4096 data: 10 20/30 Flush 21/30 Write block 466 len 40960 data: 12 15 13 14 13 13 14 13 14 15 22/30 Read block 258 len 24576 23/30 Read block 549 len 24576 24/30 Flush 25/30 Write block 423 len 20480 data: 12 11 10 11 13 26/30 Flush 27/30 Write block 561 len 16384 data: 15 17 17 11 28/30 Read block 493 len 28672 29/30 Read block 544 len 20480 30/30 Write block 506 len 12288 data: 14 16 16 ---------------------------------------------------------------- Crucible gen:148 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 4 Done Done Done false 1027 Acked Read 7 Done Done Done false 1028 Acked Read 5 Done Done Done false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:17:28.758552336Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1774} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 148 148 148 266 266 266 F F F 1 020-039 144 144 144 261 261 261 F F F 2 040-059 148 148 148 264 264 264 F F F 3 060-079 148 148 148 267 267 267 F F F 4 080-099 144 144 144 261 261 261 F F F 5 100-119 148 148 148 265 265 265 F F F 6 120-139 148 148 148 265 265 265 F F F 7 140-159 145 145 145 262 262 262 F F F 8 160-179 145 145 145 262 262 262 F F F 9 180-199 145 145 145 262 262 262 F F F 10 200-219 145 145 145 262 262 262 F F F 11 220-239 148 148 148 265 265 265 F F F 12 240-259 144 144 144 261 261 261 F F F 13 260-279 144 144 144 261 261 261 F F F 14 280-299 144 144 144 261 261 261 F F F 15 300-319 145 145 145 262 262 262 F F F 16 320-339 145 145 145 262 262 262 F F F 17 340-359 144 144 144 261 261 261 F F F 18 360-379 145 145 145 262 262 262 F F F 19 380-399 145 145 145 262 262 262 F F F 20 400-419 145 145 145 262 262 262 F F F 21 420-439 148 148 148 269 269 269 F F F 22 440-459 140 140 140 254 254 254 F F F 23 460-479 148 148 148 268 268 268 F F F 24 480-499 148 148 148 266 266 266 F F F 25 500-519 144 144 144 261 261 261 T T T 26 520-539 148 148 148 264 264 264 F F F 27 540-559 145 145 145 262 262 262 F F F 28 560-579 142 142 142 258 258 258 T T T 29 580-599 145 145 145 262 262 262 F F F Max gen: 148, Max flush: 269 A Difference in extent metadata is expected here On loop 74 Verifying data now No change after verify Loop: 74 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 148 148 148 266 266 266 F F F 1 020-039 144 144 144 261 261 261 F F F 2 040-059 148 148 148 264 264 264 F F F 3 060-079 148 148 148 267 267 267 F F F 4 080-099 144 144 144 261 261 261 F F F 5 100-119 148 148 148 265 265 265 F F F 6 120-139 148 148 148 265 265 265 F F F 7 140-159 145 145 145 262 262 262 F F F 8 160-179 145 145 145 262 262 262 F F F 9 180-199 145 145 145 262 262 262 F F F 10 200-219 145 145 145 262 262 262 F F F 11 220-239 148 148 148 265 265 265 F F F 12 240-259 144 144 144 261 261 261 F F F 13 260-279 144 144 144 261 261 261 F F F 14 280-299 144 144 144 261 261 261 F F F 15 300-319 145 145 145 262 262 262 F F F 16 320-339 145 145 145 262 262 262 F F F 17 340-359 144 144 144 261 261 261 F F F 18 360-379 145 145 145 262 262 262 F F F 19 380-399 145 145 145 262 262 262 F F F 20 400-419 145 145 145 262 262 262 F F F 21 420-439 148 148 148 269 269 269 F F F 22 440-459 140 140 140 254 254 254 F F F 23 460-479 148 148 148 268 268 268 F F F 24 480-499 148 148 148 266 266 266 F F F 25 500-519 149 149 149 270 270 270 F F F 26 520-539 148 148 148 264 264 264 F F F 27 540-559 145 145 145 262 262 262 F F F 28 560-579 149 149 149 270 270 270 F F F 29 580-599 145 145 145 262 262 262 F F F Max gen: 149, Max flush: 270 Begin loop 75 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:33.016028178Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1782,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:17:33.05960978Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1782,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:33.060179443Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1782,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:33.092434155Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1782,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:33.093242888Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1782} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"abd5430f1508af0bc008143ea2566277ef714ec2\",\n git_commit_timestamp: \"2025-01-17T20:55:03.000000000Z\",\n git_branch: \"alan/tokio-to-140\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:33.093277388Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1782} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:33.093291946Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1782} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:33.093306045Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1782} {"msg":"Crucible 5efc5166-5c52-4d2c-9c6e-15b5b7618eed has session id: 21771cdd-03ba-4529-afc7-fe3c844107e8","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:33.09396463Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1782,"session_id":"21771cdd-03ba-4529-afc7-fe3c844107e8"} {"msg":"Upstairs opts: Upstairs UUID: 5efc5166-5c52-4d2c-9c6e-15b5b7618eed, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:33.093998559Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1782,"session_id":"21771cdd-03ba-4529-afc7-fe3c844107e8"} {"msg":"Crucible stats registered with UUID: 5efc5166-5c52-4d2c-9c6e-15b5b7618eed","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:33.094020102Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1782,"session_id":"21771cdd-03ba-4529-afc7-fe3c844107e8"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:17:33.09403338Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1782,"session_id":"21771cdd-03ba-4529-afc7-fe3c844107e8"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:33.094485724Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1782,"task":"crutest"} {"msg":"The guest has requested activation with gen:150","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:33.094522325Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1782} {"msg":"5efc5166-5c52-4d2c-9c6e-15b5b7618eed active request set","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:33.094580159Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1782,"session_id":"21771cdd-03ba-4529-afc7-fe3c844107e8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:33.094605174Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1782,"client":"0","":"downstairs","session_id":"21771cdd-03ba-4529-afc7-fe3c844107e8"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:33.094630299Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1782,"client":"0","":"downstairs","session_id":"21771cdd-03ba-4529-afc7-fe3c844107e8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:33.094644907Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1782,"client":"1","":"downstairs","session_id":"21771cdd-03ba-4529-afc7-fe3c844107e8"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:33.094658895Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1782,"client":"1","":"downstairs","session_id":"21771cdd-03ba-4529-afc7-fe3c844107e8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:33.094672153Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1782,"client":"2","":"downstairs","session_id":"21771cdd-03ba-4529-afc7-fe3c844107e8"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:33.094686281Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1782,"client":"2","":"downstairs","session_id":"21771cdd-03ba-4529-afc7-fe3c844107e8"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:33.094699009Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1782,"":"io task","client":"2","":"downstairs","session_id":"21771cdd-03ba-4529-afc7-fe3c844107e8"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:33.094712236Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1782,"":"io task","client":"0","":"downstairs","session_id":"21771cdd-03ba-4529-afc7-fe3c844107e8"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:33.094725394Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1782,"":"io task","client":"1","":"downstairs","session_id":"21771cdd-03ba-4529-afc7-fe3c844107e8"} {"msg":"ds_connection connected from Ok(127.0.0.1:45219)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:33.095109589Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1782,"":"io task","client":"1","":"downstairs","session_id":"21771cdd-03ba-4529-afc7-fe3c844107e8"} {"msg":"ds_connection connected from Ok(127.0.0.1:59275)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:33.095150543Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1782,"":"io task","client":"0","":"downstairs","session_id":"21771cdd-03ba-4529-afc7-fe3c844107e8"} {"msg":"ds_connection connected from Ok(127.0.0.1:47178)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:33.095186204Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1782,"":"io task","client":"2","":"downstairs","session_id":"21771cdd-03ba-4529-afc7-fe3c844107e8"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:33.095716003Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1782,"client":"1","":"downstairs","session_id":"21771cdd-03ba-4529-afc7-fe3c844107e8"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:33.095742238Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1782,"client":"2","":"downstairs","session_id":"21771cdd-03ba-4529-afc7-fe3c844107e8"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:33.095888834Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1782,"client":"0","":"downstairs","session_id":"21771cdd-03ba-4529-afc7-fe3c844107e8"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:33.096352595Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1782,"client":"2","":"downstairs","session_id":"21771cdd-03ba-4529-afc7-fe3c844107e8"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:33.09639458Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1782,"client":"1","":"downstairs","session_id":"21771cdd-03ba-4529-afc7-fe3c844107e8"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:33.096591305Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1782,"client":"0","":"downstairs","session_id":"21771cdd-03ba-4529-afc7-fe3c844107e8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:33.097060629Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1782,"session_id":"21771cdd-03ba-4529-afc7-fe3c844107e8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:33.097095009Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1782,"session_id":"21771cdd-03ba-4529-afc7-fe3c844107e8"} {"msg":"[0]R flush_numbers[0..12]: [266, 261, 264, 267, 261, 265, 265, 262, 262, 262, 262, 265]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:33.097191906Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1782,"":"downstairs","":"downstairs","session_id":"21771cdd-03ba-4529-afc7-fe3c844107e8"} {"msg":"[0]R generation[0..12]: [148, 144, 148, 148, 144, 148, 148, 145, 145, 145, 145, 148]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:33.097219392Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1782,"":"downstairs","":"downstairs","session_id":"21771cdd-03ba-4529-afc7-fe3c844107e8"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:33.097234491Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1782,"":"downstairs","":"downstairs","session_id":"21771cdd-03ba-4529-afc7-fe3c844107e8"} {"msg":"[1]R flush_numbers[0..12]: [266, 261, 264, 267, 261, 265, 265, 262, 262, 262, 262, 265]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:33.097248339Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1782,"":"downstairs","":"downstairs","session_id":"21771cdd-03ba-4529-afc7-fe3c844107e8"} {"msg":"[1]R generation[0..12]: [148, 144, 148, 148, 144, 148, 148, 145, 145, 145, 145, 148]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:33.097262107Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1782,"":"downstairs","":"downstairs","session_id":"21771cdd-03ba-4529-afc7-fe3c844107e8"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:33.097279287Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1782,"":"downstairs","":"downstairs","session_id":"21771cdd-03ba-4529-afc7-fe3c844107e8"} {"msg":"[2]R flush_numbers[0..12]: [266, 261, 264, 267, 261, 265, 265, 262, 262, 262, 262, 265]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:33.097293035Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1782,"":"downstairs","":"downstairs","session_id":"21771cdd-03ba-4529-afc7-fe3c844107e8"} {"msg":"[2]R generation[0..12]: [148, 144, 148, 148, 144, 148, 148, 145, 145, 145, 145, 148]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:33.097306313Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1782,"":"downstairs","":"downstairs","session_id":"21771cdd-03ba-4529-afc7-fe3c844107e8"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:33.097320011Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1782,"":"downstairs","":"downstairs","session_id":"21771cdd-03ba-4529-afc7-fe3c844107e8"} {"msg":"Max found gen is 150","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:33.097333048Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1782,"":"downstairs","":"downstairs","session_id":"21771cdd-03ba-4529-afc7-fe3c844107e8"} {"msg":"Generation requested: 150 >= found:150","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-17T21:17:33.097345866Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1782,"":"downstairs","":"downstairs","session_id":"21771cdd-03ba-4529-afc7-fe3c844107e8"} {"msg":"Next flush: 271","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:33.097365157Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1782,"":"downstairs","":"downstairs","session_id":"21771cdd-03ba-4529-afc7-fe3c844107e8"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:33.097377865Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1782,"":"downstairs","":"downstairs","session_id":"21771cdd-03ba-4529-afc7-fe3c844107e8"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:33.097390642Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1782,"session_id":"21771cdd-03ba-4529-afc7-fe3c844107e8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:33.097402209Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1782,"client":"0","":"downstairs","session_id":"21771cdd-03ba-4529-afc7-fe3c844107e8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:33.097415296Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1782,"client":"1","":"downstairs","session_id":"21771cdd-03ba-4529-afc7-fe3c844107e8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:33.097428084Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1782,"client":"2","":"downstairs","session_id":"21771cdd-03ba-4529-afc7-fe3c844107e8"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:33.097440781Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1782,"session_id":"21771cdd-03ba-4529-afc7-fe3c844107e8"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:33.097452568Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1782,"session_id":"21771cdd-03ba-4529-afc7-fe3c844107e8"} {"msg":"5efc5166-5c52-4d2c-9c6e-15b5b7618eed is now active with session: 21771cdd-03ba-4529-afc7-fe3c844107e8","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:33.097465096Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1782,"session_id":"21771cdd-03ba-4529-afc7-fe3c844107e8"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:33.097478303Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1782,"session_id":"21771cdd-03ba-4529-afc7-fe3c844107e8"} {"msg":"The guest has finished waiting for activation with:150","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:33.09749032Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1782} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 569 len 36864 data: 12 13 12 10 11 12 10 13 13 02/30 Read block 521 len 12288 03/30 Read block 146 len 40960 04/30 Read block 582 len 8192 05/30 Read block 120 len 36864 06/30 Write block 563 len 4096 data: 18 07/30 Write block 486 len 12288 data: 10 8 9 08/30 Read block 506 len 12288 09/30 Read block 28 len 24576 10/30 Read block 478 len 20480 11/30 Read block 388 len 16384 12/30 Read block 467 len 28672 13/30 Read block 43 len 8192 14/30 Read block 7 len 12288 15/30 Write block 152 len 40960 data: 9 9 11 12 11 13 12 12 11 13 16/30 Write block 471 len 28672 data: 14 15 14 15 16 13 11 17/30 Write block 407 len 12288 data: 6 6 7 18/30 Read block 266 len 32768 19/30 Read block 411 len 8192 20/30 Write block 311 len 40960 data: 5 8 10 9 7 8 9 11 11 12 21/30 Read block 74 len 36864 22/30 Read block 127 len 36864 23/30 Read block 24 len 8192 24/30 Flush 25/30 Write block 510 len 36864 data: 18 18 17 18 20 22 21 17 15 26/30 Write block 487 len 20480 data: 9 10 11 13 12 27/30 Read block 461 len 20480 28/30 Write block 558 len 8192 data: 13 11 29/30 Write block 106 len 12288 data: 13 11 11 30/30 Write block 8 len 4096 data: 11 ---------------------------------------------------------------- Crucible gen:150 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 9 Done Done Done false 1025 Acked Write 5 Done Done Done false 1026 Acked Read 5 Done Done Done false 1027 Acked Write 2 Done Done Sent false 1028 Acked Write 3 Sent Sent Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 3 7 Done 4 4 3 11 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:17:33.117437929Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1782} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 148 148 148 266 266 266 T T T 1 020-039 144 144 144 261 261 261 F F F 2 040-059 148 148 148 264 264 264 F F F 3 060-079 148 148 148 267 267 267 F F F 4 080-099 144 144 144 261 261 261 F F F 5 100-119 148 148 148 265 265 265 T T T 6 120-139 148 148 148 265 265 265 F F F 7 140-159 150 150 150 271 271 271 F F F 8 160-179 150 150 150 271 271 271 F F F 9 180-199 145 145 145 262 262 262 F F F 10 200-219 145 145 145 262 262 262 F F F 11 220-239 148 148 148 265 265 265 F F F 12 240-259 144 144 144 261 261 261 F F F 13 260-279 144 144 144 261 261 261 F F F 14 280-299 144 144 144 261 261 261 F F F 15 300-319 150 150 150 271 271 271 F F F 16 320-339 150 150 150 271 271 271 F F F 17 340-359 144 144 144 261 261 261 F F F 18 360-379 145 145 145 262 262 262 F F F 19 380-399 145 145 145 262 262 262 F F F 20 400-419 150 150 150 271 271 271 F F F 21 420-439 148 148 148 269 269 269 F F F 22 440-459 140 140 140 254 254 254 F F F 23 460-479 150 150 150 271 271 271 F F F 24 480-499 150 150 150 271 271 271 T T T 25 500-519 149 149 149 270 270 270 T T T 26 520-539 148 148 148 264 264 264 F F F 27 540-559 145 145 145 262 262 262 T T T 28 560-579 150 150 150 271 271 271 F F F 29 580-599 145 145 145 262 262 262 F F F Max gen: 150, Max flush: 271 A Difference in extent metadata is expected here On loop 75 Verifying data now No change after verify Loop: 75 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 151 151 151 272 272 272 F F F 1 020-039 144 144 144 261 261 261 F F F 2 040-059 148 148 148 264 264 264 F F F 3 060-079 148 148 148 267 267 267 F F F 4 080-099 144 144 144 261 261 261 F F F 5 100-119 151 151 151 272 272 272 F F F 6 120-139 148 148 148 265 265 265 F F F 7 140-159 150 150 150 271 271 271 F F F 8 160-179 150 150 150 271 271 271 F F F 9 180-199 145 145 145 262 262 262 F F F 10 200-219 145 145 145 262 262 262 F F F 11 220-239 148 148 148 265 265 265 F F F 12 240-259 144 144 144 261 261 261 F F F 13 260-279 144 144 144 261 261 261 F F F 14 280-299 144 144 144 261 261 261 F F F 15 300-319 150 150 150 271 271 271 F F F 16 320-339 150 150 150 271 271 271 F F F 17 340-359 144 144 144 261 261 261 F F F 18 360-379 145 145 145 262 262 262 F F F 19 380-399 145 145 145 262 262 262 F F F 20 400-419 150 150 150 271 271 271 F F F 21 420-439 148 148 148 269 269 269 F F F 22 440-459 140 140 140 254 254 254 F F F 23 460-479 150 150 150 271 271 271 F F F 24 480-499 151 151 151 272 272 272 F F F 25 500-519 151 151 151 272 272 272 F F F 26 520-539 148 148 148 264 264 264 F F F 27 540-559 151 151 151 272 272 272 F F F 28 560-579 150 150 150 271 271 271 F F F 29 580-599 145 145 145 262 262 262 F F F Max gen: 151, Max flush: 272 Begin loop 76 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:37.396476844Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1790,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:37.440901659Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1790,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:37.44157041Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1790} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"abd5430f1508af0bc008143ea2566277ef714ec2\",\n git_commit_timestamp: \"2025-01-17T20:55:03.000000000Z\",\n git_branch: \"alan/tokio-to-140\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:37.441610344Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1790} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:37.441629715Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1790} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:37.441645254Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1790} {"msg":"Crucible 6f6abe80-8fc4-4f1e-ab23-4893abe08e15 has session id: d280c3ea-4e57-456b-9d6e-b1571280e239","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:37.442234578Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1790,"session_id":"d280c3ea-4e57-456b-9d6e-b1571280e239"} {"msg":"Upstairs opts: Upstairs UUID: 6f6abe80-8fc4-4f1e-ab23-4893abe08e15, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:37.442273081Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1790,"session_id":"d280c3ea-4e57-456b-9d6e-b1571280e239"} {"msg":"Crucible stats registered with UUID: 6f6abe80-8fc4-4f1e-ab23-4893abe08e15","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:37.442295894Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1790,"session_id":"d280c3ea-4e57-456b-9d6e-b1571280e239"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:17:37.442316936Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1790,"session_id":"d280c3ea-4e57-456b-9d6e-b1571280e239"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:37.44283735Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1790,"task":"crutest"} {"msg":"The guest has requested activation with gen:152","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:37.44286997Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1790} {"msg":"6f6abe80-8fc4-4f1e-ab23-4893abe08e15 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:37.442952388Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1790,"session_id":"d280c3ea-4e57-456b-9d6e-b1571280e239"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:37.442980314Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1790,"client":"0","":"downstairs","session_id":"d280c3ea-4e57-456b-9d6e-b1571280e239"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:37.443013664Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1790,"client":"0","":"downstairs","session_id":"d280c3ea-4e57-456b-9d6e-b1571280e239"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:37.443039159Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1790,"client":"1","":"downstairs","session_id":"d280c3ea-4e57-456b-9d6e-b1571280e239"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:37.443067245Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1790,"client":"1","":"downstairs","session_id":"d280c3ea-4e57-456b-9d6e-b1571280e239"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:37.44309376Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1790,"client":"2","":"downstairs","session_id":"d280c3ea-4e57-456b-9d6e-b1571280e239"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:37.443123057Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1790,"client":"2","":"downstairs","session_id":"d280c3ea-4e57-456b-9d6e-b1571280e239"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:37.443156267Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1790,"":"io task","client":"2","":"downstairs","session_id":"d280c3ea-4e57-456b-9d6e-b1571280e239"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:37.443184323Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1790,"":"io task","client":"0","":"downstairs","session_id":"d280c3ea-4e57-456b-9d6e-b1571280e239"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:37.44321307Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1790,"":"io task","client":"1","":"downstairs","session_id":"d280c3ea-4e57-456b-9d6e-b1571280e239"} {"msg":"ds_connection connected from Ok(127.0.0.1:54488)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:37.443240696Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1790,"":"io task","client":"2","":"downstairs","session_id":"d280c3ea-4e57-456b-9d6e-b1571280e239"} {"msg":"ds_connection connected from Ok(127.0.0.1:64382)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:37.443269703Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1790,"":"io task","client":"1","":"downstairs","session_id":"d280c3ea-4e57-456b-9d6e-b1571280e239"} {"msg":"ds_connection connected from Ok(127.0.0.1:51279)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:37.44329902Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1790,"":"io task","client":"0","":"downstairs","session_id":"d280c3ea-4e57-456b-9d6e-b1571280e239"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:37.443827679Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1790,"client":"1","":"downstairs","session_id":"d280c3ea-4e57-456b-9d6e-b1571280e239"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:37.443954714Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1790,"client":"0","":"downstairs","session_id":"d280c3ea-4e57-456b-9d6e-b1571280e239"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:37.444216447Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1790,"client":"2","":"downstairs","session_id":"d280c3ea-4e57-456b-9d6e-b1571280e239"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:37.444335906Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1790,"client":"1","":"downstairs","session_id":"d280c3ea-4e57-456b-9d6e-b1571280e239"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:37.444658125Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1790,"client":"0","":"downstairs","session_id":"d280c3ea-4e57-456b-9d6e-b1571280e239"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:37.444797476Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1790,"client":"2","":"downstairs","session_id":"d280c3ea-4e57-456b-9d6e-b1571280e239"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:37.444953227Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1790,"session_id":"d280c3ea-4e57-456b-9d6e-b1571280e239"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:37.445216971Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1790,"session_id":"d280c3ea-4e57-456b-9d6e-b1571280e239"} {"msg":"[0]R flush_numbers[0..12]: [272, 261, 264, 267, 261, 272, 265, 271, 271, 262, 262, 265]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:37.445336451Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1790,"":"downstairs","":"downstairs","session_id":"d280c3ea-4e57-456b-9d6e-b1571280e239"} {"msg":"[0]R generation[0..12]: [151, 144, 148, 148, 144, 151, 148, 150, 150, 145, 145, 148]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:37.445360445Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1790,"":"downstairs","":"downstairs","session_id":"d280c3ea-4e57-456b-9d6e-b1571280e239"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:37.445375644Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1790,"":"downstairs","":"downstairs","session_id":"d280c3ea-4e57-456b-9d6e-b1571280e239"} {"msg":"[1]R flush_numbers[0..12]: [272, 261, 264, 267, 261, 272, 265, 271, 271, 262, 262, 265]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:37.445389282Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1790,"":"downstairs","":"downstairs","session_id":"d280c3ea-4e57-456b-9d6e-b1571280e239"} {"msg":"[1]R generation[0..12]: [151, 144, 148, 148, 144, 151, 148, 150, 150, 145, 145, 148]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:37.44540277Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1790,"":"downstairs","":"downstairs","session_id":"d280c3ea-4e57-456b-9d6e-b1571280e239"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:37.445416768Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1790,"":"downstairs","":"downstairs","session_id":"d280c3ea-4e57-456b-9d6e-b1571280e239"} {"msg":"[2]R flush_numbers[0..12]: [272, 261, 264, 267, 261, 272, 265, 271, 271, 262, 262, 265]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:37.445430396Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1790,"":"downstairs","":"downstairs","session_id":"d280c3ea-4e57-456b-9d6e-b1571280e239"} {"msg":"[2]R generation[0..12]: [151, 144, 148, 148, 144, 151, 148, 150, 150, 145, 145, 148]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:37.445443884Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1790,"":"downstairs","":"downstairs","session_id":"d280c3ea-4e57-456b-9d6e-b1571280e239"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:37.445457372Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1790,"":"downstairs","":"downstairs","session_id":"d280c3ea-4e57-456b-9d6e-b1571280e239"} {"msg":"Max found gen is 152","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:37.445469889Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1790,"":"downstairs","":"downstairs","session_id":"d280c3ea-4e57-456b-9d6e-b1571280e239"} {"msg":"Generation requested: 152 >= found:152","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:37.445482516Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1790,"":"downstairs","":"downstairs","session_id":"d280c3ea-4e57-456b-9d6e-b1571280e239"} {"msg":"Next flush: 273","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:37.445495024Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid"Wait for a query_work_queue command to finish before sending IO :1790,"":"downstairs","":"downstairs","session_id":"d280c3ea-4e57-456b-9d6e-b1571280e239"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:37.445513945Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1790,"":"downstairs","":"downstairs","session_id":"d280c3ea-4e57-456b-9d6e-b1571280e239"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:37.445526842Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1790,"session_id":"d280c3ea-4e57-456b-9d6e-b1571280e239"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:37.445546214Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1790,"client":"0","":"downstairs","session_id":"d280c3ea-4e57-456b-9d6e-b1571280e239"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:37.445562513Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1790,"client":"1","":"downstairs","session_id":"d280c3ea-4e57-456b-9d6e-b1571280e239"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:37.445575711Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1790,"client":"2","":"downstairs","session_id":"d280c3ea-4e57-456b-9d6e-b1571280e239"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:37.445588779Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1790,"session_id":"d280c3ea-4e57-456b-9d6e-b1571280e239"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:37.445600636Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1790,"session_id":"d280c3ea-4e57-456b-9d6e-b1571280e239"} {"msg":"6f6abe80-8fc4-4f1e-ab23-4893abe08e15 is now active with session: d280c3ea-4e57-456b-9d6e-b1571280e239","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:37.445612873Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1790,"session_id":"d280c3ea-4e57-456b-9d6e-b1571280e239"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:37.44562427Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1790,"session_id":"d280c3ea-4e57-456b-9d6e-b1571280e239"} {"msg":"The guest has finished waiting for activation with:152","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:37.445636126Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1790} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 121 len 20480 data: 11 11 9 7 10 02/30 Flush 03/30 Write block 151 len 4096 data: 9 04/30 Write block 335 len 40960 data: 12 11 14 15 17 16 16 13 16 10 05/30 Write block 142 len 40960 data: 12 10 9 9 8 11 9 10 10 10 06/30 Write block 448 len 8192 data: 7 10 07/30 Flush 08/30 Write block 189 len 36864 data: 10 12 12 14 15 15 13 13 10 09/30 Flush 10/30 Write block 68 len 24576 data: 11 10 10 14 12 11 11/30 Read block 589 len 8192 12/30 Flush 13/30 Write block 274 len 12288 data: 9 10 10 14/30 Read block 372 len 4096 15/30 Flush 16/30 Write block 80 len 12288 data: 11 11 8 17/30 Write block 160 len 4096 data: 12 18/30 Write block 167 len 8192 data: 13 14 19/30 Flush 20/30 Write block 391 len 24576 data: 9 10 9 9 9 11 21/30 Flush 22/30 Write block 385 len 16384 data: 10 10 10 9 23/30 Write block 275 len 16384 data: 11 11 11 9 24/30 Write block 188 len 20480 data: 9 11 13 13 15 25/30 Read block 41 len 8192 26/30 Read block 137 len 40960 27/30 Write block 314 len 12288 data: 10 8 9 28/30 Write block 329 len 4096 data: 9 29/30 Read block 372 len 4096 30/30 Read block 465 len 24576 ---------------------------------------------------------------- Crucible gen:152 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 4 Done Done Done false 1022 Acked Write 4 Done Done Done false 1023 Acked Write 5 Done Done Done false 1024 Acked Read 2 Done Done Done false 1025 Acked Read 10 Done Done Done false 1026 Acked Write 3 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Read 1 Done Done Done false 1029 Acked Read 6 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 9 9 9 27 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1020 1020 1020 Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:17:37.463092829Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1790} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 151 151 151 272 272 272 F F F 1 020-039 144 144 144 261 261 261 F F F 2 040-059 148 148 148 264 264 264 F F F 3 060-079 152 152 152 276 276 276 F F F 4 080-099 152 152 152 278 278 278 F F F 5 100-119 151 151 151 272 272 272 F F F 6 120-139 152 152 152 273 273 273 F F F 7 140-159 152 152 152 274 274 274 F F F 8 160-179 152 152 152 278 278 278 F F F 9 180-199 152 152 152 275 275 275 T T T 10 200-219 145 145 145 262 262 262 F F F 11 220-239 148 148 148 265 265 265 F F F 12 240-259 144 144 144 261 261 261 F F F 13 260-279 152 152 152 277 277 277 T T T 14 280-299 144 144 144 261 261 261 F F F 15 300-319 150 150 150 271 271 271 T T T 16 320-339 152 152 152 274 274 274 T T T 17 340-359 152 152 152 274 274 274 F F F 18 360-379 145 145 145 262 262 262 F F F 19 380-399 152 152 152 279 279 279 T T T 20 400-419 150 150 150 271 271 271 F F F 21 420-439 148 148 148 269 269 269 F F F 22 440-459 152 152 152 274 274 274 F F F 23 460-479 150 150 150 271 271 271 F F F 24 480-499 151 151 151 272 272 272 F F F 25 500-519 151 151 151 272 272 272 F F F 26 520-539 148 148 148 264 264 264 F F F 27 540-559 151 151 151 272 272 272 F F F 28 560-579 150 150 150 271 271 271 F F F 29 580-599 145 145 145 262 262 262 F F F Max gen: 152, Max flush: 279 A Difference in extent metadata is expected here On loop 76 Verifying data now No change after verify Loop: 76 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 151 151 151 272 272 272 F F F 1 020-039 144 144 144 261 261 261 F F F 2 040-059 148 148 148 264 264 264 F F F 3 060-079 152 152 152 276 276 276 F F F 4 080-099 152 152 152 278 278 278 F F F 5 100-119 151 151 151 272 272 272 F F F 6 120-139 152 152 152 273 273 273 F F F 7 140-159 152 152 152 274 274 274 F F F 8 160-179 152 152 152 278 278 278 F F F 9 180-199 153 153 153 280 280 280 F F F 10 200-219 145 145 145 262 262 262 F F F 11 220-239 148 148 148 265 265 265 F F F 12 240-259 144 144 144 261 261 261 F F F 13 260-279 153 153 153 280 280 280 F F F 14 280-299 144 144 144 261 261 261 F F F 15 300-319 153 153 153 280 280 280 F F F 16 320-339 153 153 153 280 280 280 F F F 17 340-359 152 152 152 274 274 274 F F F 18 360-379 145 145 145 262 262 262 F F F 19 380-399 153 153 153 280 280 280 F F F 20 400-419 150 150 150 271 271 271 F F F 21 420-439 148 148 148 269 269 269 F F F 22 440-459 152 152 152 274 274 274 F F F 23 460-479 150 150 150 271 271 271 F F F 24 480-499 151 151 151 272 272 272 F F F 25 500-519 151 151 151 272 272 272 F F F 26 520-539 148 148 148 264 264 264 F F F 27 540-559 151 151 151 272 272 272 F F F 28 560-579 150 150 150 271 271 271 F F F 29 580-599 145 145 145 262 262 262 F F F Max gen: 153, Max flush: 280 Begin loop 77 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:41.703981867Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1798,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:41.749174231Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1798,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:41.749844463Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1798} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"abd5430f1508af0bc008143ea2566277ef714ec2\",\n git_commit_timestamp: \"2025-01-17T20:55:03.000000000Z\",\n git_branch: \"alan/tokio-to-140\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:41.749900176Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1798} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:41.749926051Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1798} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:41.749952356Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1798} {"msg":"Crucible 999abeca-8b27-41fc-b672-9d4a752f7ecf has session id: 2b9da683-4391-4714-bb40-f7dc935be4ae","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:41.750549225Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1798,"session_id":"2b9da683-4391-4714-bb40-f7dc935be4ae"} {"msg":"Upstairs opts: Upstairs UUID: 999abeca-8b27-41fc-b672-9d4a752f7ecf, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:41.750579413Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1798,"session_id":"2b9da683-4391-4714-bb40-f7dc935be4ae"} {"msg":"Crucible stats registered with UUID: 999abeca-8b27-41fc-b672-9d4a752f7ecf","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:41.750594591Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1798,"session_id":"2b9da683-4391-4714-bb40-f7dc935be4ae"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:17:41.750607469Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1798,"session_id":"2b9da683-4391-4714-bb40-f7dc935be4ae"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:41.751058062Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1798,"task":"crutest"} {"msg":"The guest has requested activation with gen:154","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:41.751081436Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1798} {"msg":"999abeca-8b27-41fc-b672-9d4a752f7ecf active request set","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:41.751148895Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1798,"session_id":"2b9da683-4391-4714-bb40-f7dc935be4ae"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:41.751165465Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1798,"client":"0","":"downstairs","session_id":"2b9da683-4391-4714-bb40-f7dc935be4ae"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:41.751192931Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1798,"client":"0","":"downstairs","session_id":"2b9da683-4391-4714-bb40-f7dc935be4ae"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:41.751206829Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1798,"client":"1","":"downstairs","session_id":"2b9da683-4391-4714-bb40-f7dc935be4ae"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:41.751221088Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1798,"client":"1","":"downstairs","session_id":"2b9da683-4391-4714-bb40-f7dc935be4ae"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:41.751234095Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1798,"client":"2","":"downstairs","session_id":"2b9da683-4391-4714-bb40-f7dc935be4ae"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:41.751247583Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1798,"client":"2","":"downstairs","session_id":"2b9da683-4391-4714-bb40-f7dc935be4ae"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:41.75126026Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1798,"":"io task","client":"2","":"downstairs","session_id":"2b9da683-4391-4714-bb40-f7dc935be4ae"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:41.751273698Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1798,"":"io task","client":"0","":"downstairs","session_id":"2b9da683-4391-4714-bb40-f7dc935be4ae"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:41.751290858Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1798,"":"io task","client":"1","":"downstairs","session_id":"2b9da683-4391-4714-bb40-f7dc935be4ae"} {"msg":"ds_connection connected from Ok(127.0.0.1:37083)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:41.751313582Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1798,"":"io task","client":"2","":"downstairs","session_id":"2b9da683-4391-4714-bb40-f7dc935be4ae"} {"msg":"ds_connection connected from Ok(127.0.0.1:39548)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:41.751406046Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1798,"":"io task","client":"1","":"downstairs","session_id":"2b9da683-4391-4714-bb40-f7dc935be4ae"} {"msg":"ds_connection connected from Ok(127.0.0.1:39201)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:41.751445008Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1798,"":"io task","client":"0","":"downstairs","session_id":"2b9da683-4391-4714-bb40-f7dc935be4ae"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:41.752083902Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1798,"client":"0","":"downstairs","session_id":"2b9da683-4391-4714-bb40-f7dc935be4ae"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:41.752111868Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1798,"client":"2","":"downstairs","session_id":"2b9da683-4391-4714-bb40-f7dc935be4ae"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:41.752127798Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1798,"client":"1","":"downstairs","session_id":"2b9da683-4391-4714-bb40-f7dc935be4ae"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:41.752800641Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1798,"client":"1","":"downstairs","session_id":"2b9da683-4391-4714-bb40-f7dc935be4ae"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:41.752829277Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1798,"client":"0","":"downstairs","session_id":"2b9da683-4391-4714-bb40-f7dc935be4ae"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:41.75285094Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1798,"client":"2","":"downstairs","session_id":"2b9da683-4391-4714-bb40-f7dc935be4ae"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:41.753359838Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1798,"session_id":"2b9da683-4391-4714-bb40-f7dc935be4ae"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:41.753383381Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1798,"session_id":"2b9da683-4391-4714-bb40-f7dc935be4ae"} {"msg":"[0]R flush_numbers[0..12]: [272, 261, 264, 276, 278, 272, 273, 274, 278, 280, 262, 265]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:41.753454873Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1798,"":"downstairs","":"downstairs","session_id":"2b9da683-4391-4714-bb40-f7dc935be4ae"} {"msg":"[0]R generation[0..12]: [151, 144, 148, 152, 152, 151, 152, 152, 152, 153, 145, 148]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:41.753474775Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1798,"":"downstairs","":"downstairs","session_id":"2b9da683-4391-4714-bb40-f7dc935be4ae"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:41.753489063Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1798,"":"downstairs","":"downstairs","session_id":"2b9da683-4391-4714-bb40-f7dc935be4ae"} {"msg":"[1]R flush_numbers[0..12]: [272, 261, 264, 276, 278, 272, 273, 274, 278, 280, 262, 265]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:41.753502711Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1798,"":"downstairs","":"downstairs","session_id":"2b9da683-4391-4714-bb40-f7dc935be4ae"} {"msg":"[1]R generation[0..12]: [151, 144, 148, 152, 152, 151, 152, 152, 152, 153, 145, 148]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:41.753516339Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1798,"":"downstairs","":"downstairs","session_id":"2b9da683-4391-4714-bb40-f7dc935be4ae"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:41.753530317Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1798,"":"downstairs","":"downstairs","session_id":"2b9da683-4391-4714-bb40-f7dc935be4ae"} {"msg":"[2]R flush_numbers[0..12]: [272, 261, 264, 276, 278, 272, 273, 274, 278, 280, 262, 265]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:41.753543875Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1798,"":"downstairs","":"downstairs","session_id":"2b9da683-4391-4714-bb40-f7dc935be4ae"} {"msg":"[2]R generation[0..12]: [151, 144, 148, 152, 152, 151, 152, 152, 152, 153, 145, 148]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:41.753557193Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1798,"":"downstairs","":"downstairs","session_id":"2b9da683-4391-4714-bb40-f7dc935be4ae"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:41.753570621Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1798,"":"downstairs","":"downstairs","session_id":"2b9da683-4391-4714-bb40-f7dc935be4ae"} {"msg":"Max found gen is 154","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:41.75358629Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1798,"":"downstairs","":"downstairs","session_id":"2b9da683-4391-4714-bb40-f7dc935be4ae"} {"msg":"Generation requested: 154 >= found:154","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:41.753599418Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1798,"":"downstairs","":"downstairs","session_id":"2b9da683-4391-4714-bb40-f7dc935be4ae"} {"msg":"Next flush: 281","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:41.753611955Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1798,"":"downstairs","":"downstairs","session_id":"2b9da683-4391-4714-bb40-f7dc935be4ae"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-17T21:17:41.753624372Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1798,"":"downstairs","":"downstairs","session_id":"2b9da683-4391-4714-bb40-f7dc935be4ae"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:41.753642343Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1798,"session_id":"2b9da683-4391-4714-bb40-f7dc935be4ae"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:41.75365399Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1798,"client":"0","":"downstairs","session_id":"2b9da683-4391-4714-bb40-f7dc935be4ae"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:41.753667818Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1798,"client":"1","":"downstairs","session_id":"2b9da683-4391-4714-bb40-f7dc935be4ae"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:41.753680495Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1798,"client":"2","":"downstairs","session_id":"2b9da683-4391-4714-bb40-f7dc935be4ae"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:41.753693242Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1798,"session_id":"2b9da683-4391-4714-bb40-f7dc935be4ae"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:41.753704929Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1798,"session_id":"2b9da683-4391-4714-bb40-f7dc935be4ae"} {"msg":"999abeca-8b27-41fc-b672-9d4a752f7ecf is now active with session: 2b9da683-4391-4714-bb40-f7dc935be4ae","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:41.753717356Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1798,"session_id":"2b9da683-4391-4714-bb40-f7dc935be4ae"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:41.753728893Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1798,"session_id":"2b9da683-4391-4714-bb40-f7dc935be4ae"} {"msg":"The guest has finished waiting for activation with:154","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:41.75374067Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1798} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 298 len 4096 data: 10 02/30 Read block 163 len 24576 03/30 Write block 225 len 24576 data: 8 10 11 12 13 14 04/30 Read block 445 len 28672 05/30 Write block 226 len 24576 data: 11 12 13 14 15 13 06/30 Read block 202 len 36864 07/30 Read block 525 len 40960 08/30 Write block 276 len 36864 data: 12 12 10 8 10 13 12 9 10 09/30 Write block 197 len 40960 data: 11 12 17 14 12 13 9 8 9 8 10/30 Write block 38 len 16384 data: 13 12 12 11 11/30 Read block 486 len 40960 12/30 Read block 477 len 24576 13/30 Write block 263 len 4096 data: 10 14/30 Read block 518 len 16384 15/30 Read block 156 len 28672 16/30 Write block 96 len 16384 data: 15 10 13 12 17/30 Read block 532 len 20480 18/30 Read block 62 len 12288 19/30 Read block 126 len 32768 20/30 Read block 556 len 28672 21/30 Write block 56 len 28672 data: 16 15 15 15 12 12 9 22/30 Read block 127 len 20480 23/30 Read block 225 len 32768 24/30 Read block 357 len 36864 25/30 Read block 463 len 4096 26/30 Flush 27/30 Write block 370 len 16384 data: 10 11 15 15 28/30 Write block 242 len 4096 data: 11 29/30 Read block 269 len 8192 30/30 Read block 340 len 24576 ---------------------------------------------------------------- Crucible gen:154 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 4 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Read 6 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 4 3 3 10 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:17:41.774072733Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1798} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 151 151 151 272 272 272 F F F 1 020-039 154 154 154 281 281 281 F F F 2 040-059 154 154 154 281 281 281 F F F 3 060-079 154 154 154 281 281 281 F F F 4 080-099 154 154 154 281 281 281 F F F 5 100-119 151 151 151 272 272 272 F F F 6 120-139 152 152 152 273 273 273 F F F 7 140-159 152 152 152 274 274 274 F F F 8 160-179 152 152 152 278 278 278 F F F 9 180-199 154 154 154 281 281 281 F F F 10 200-219 154 154 154 281 281 281 F F F 11 220-239 154 154 154 281 281 281 F F F 12 240-259 144 144 144 261 261 261 T T T 13 260-279 154 154 154 281 281 281 F F F 14 280-299 154 154 154 281 281 281 F F F 15 300-319 153 153 153 280 280 280 F F F 16 320-339 153 153 153 280 280 280 F F F 17 340-359 152 152 152 274 274 274 F F F 18 360-379 145 145 145 262 262 262 T T T 19 380-399 153 153 153 280 280 280 F F F 20 400-419 150 150 150 271 271 271 F F F 21 420-439 148 148 148 269 269 269 F F F 22 440-459 152 152 152 274 274 274 F F F 23 460-479 150 150 150 271 271 271 F F F 24 480-499 151 151 151 272 272 272 F F F 25 500-519 151 151 151 272 272 272 F F F 26 520-539 148 148 148 264 264 264 F F F 27 540-559 151 151 151 272 272 272 F F F 28 560-579 150 150 150 271 271 271 F F F 29 580-599 145 145 145 262 262 262 F F F Max gen: 154, Max flush: 281 A Difference in extent metadata is expected here On loop 77 Verifying data now No change after verify Loop: 77 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 151 151 151 272 272 272 F F F 1 020-039 154 154 154 281 281 281 F F F 2 040-059 154 154 154 281 281 281 F F F 3 060-079 154 154 154 281 281 281 F F F 4 080-099 154 154 154 281 281 281 F F F 5 100-119 151 151 151 272 272 272 F F F 6 120-139 152 152 152 273 273 273 F F F 7 140-159 152 152 152 274 274 274 F F F 8 160-179 152 152 152 278 278 278 F F F 9 180-199 154 154 154 281 281 281 F F F 10 200-219 154 154 154 281 281 281 F F F 11 220-239 154 154 154 281 281 281 F F F 12 240-259 155 155 155 282 282 282 F F F 13 260-279 154 154 154 281 281 281 F F F 14 280-299 154 154 154 281 281 281 F F F 15 300-319 153 153 153 280 280 280 F F F 16 320-339 153 153 153 280 280 280 F F F 17 340-359 152 152 152 274 274 274 F F F 18 360-379 155 155 155 282 282 282 F F F 19 380-399 153 153 153 280 280 280 F F F 20 400-419 150 150 150 271 271 271 F F F 21 420-439 148 148 148 269 269 269 F F F 22 440-459 152 152 152 274 274 274 F F F 23 460-479 150 150 150 271 271 271 F F F 24 480-499 151 151 151 272 272 272 F F F 25 500-519 151 151 151 272 272 272 F F F 26 520-539 148 148 148 264 264 264 F F F 27 540-559 151 151 151 272 272 272 F F F 28 560-579 150 150 150 271 271 271 F F F 29 580-599 145 145 145 262 262 262 F F F Max gen: 155, Max flush: 282 Begin loop 78 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:46.001046837Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1806,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:46.047233732Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1806,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:46.04793074Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1806} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"abd5430f1508af0bc008143ea2566277ef714ec2\",\n git_commit_timestamp: \"2025-01-17T20:55:03.000000000Z\",\n git_branch: \"alan/tokio-to-140\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:46.047978238Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1806} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:46.047995458Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1806} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:46.048013028Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1806} {"msg":"Crucible a3317f56-3e6b-4da2-bd87-1d58c27c853f has session id: c41e812d-cc43-4ed3-97cb-a859445d713d","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:46.048544128Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1806,"session_id":"c41e812d-cc43-4ed3-97cb-a859445d713d"} {"msg":"Upstairs opts: Upstairs UUID: a3317f56-3e6b-4da2-bd87-1d58c27c853f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:46.048572155Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1806,"session_id":"c41e812d-cc43-4ed3-97cb-a859445d713d"} {"msg":"Crucible stats registered with UUID: a3317f56-3e6b-4da2-bd87-1d58c27c853f","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:46.048586573Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1806,"session_id":"c41e812d-cc43-4ed3-97cb-a859445d713d"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:17:46.048603033Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1806,"session_id":"c41e812d-cc43-4ed3-97cb-a859445d713d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:46.04909484Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1806,"task":"crutest"} {"msg":"The guest has requested activation with gen:156","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:46.049118494Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1806} {"msg":"a3317f56-3e6b-4da2-bd87-1d58c27c853f active request set","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:46.049199681Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1806,"session_id":"c41e812d-cc43-4ed3-97cb-a859445d713d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:46.049217202Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1806,"client":"0","":"downstairs","session_id":"c41e812d-cc43-4ed3-97cb-a859445d713d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:46.049241616Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1806,"client":"0","":"downstairs","session_id":"c41e812d-cc43-4ed3-97cb-a859445d713d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:46.049255304Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1806,"client":"1","":"downstairs","session_id":"c41e812d-cc43-4ed3-97cb-a859445d713d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:46.049280219Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1806,"client":"1","":"downstairs","session_id":"c41e812d-cc43-4ed3-97cb-a859445d713d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:46.0493Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1806,"client":"2","":"downstairs","session_id":"c41e812d-cc43-4ed3-97cb-a859445d713d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:46.049326055Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1806,"client":"2","":"downstairs","session_id":"c41e812d-cc43-4ed3-97cb-a859445d713d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:46.049340794Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1806,"":"io task","client":"2","":"downstairs","session_id":"c41e812d-cc43-4ed3-97cb-a859445d713d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:46.049363968Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1806,"":"io task","client":"0","":"downstairs","session_id":"c41e812d-cc43-4ed3-97cb-a859445d713d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:46.049390993Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1806,"":"io task","client":"1","":"downstairs","session_id":"c41e812d-cc43-4ed3-97cb-a859445d713d"} {"msg":"ds_connection connected from Ok(127.0.0.1:38744)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:46.049454831Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1806,"":"io task","client":"2","":"downstairs","session_id":"c41e812d-cc43-4ed3-97cb-a859445d713d"} {"msg":"ds_connection connected from Ok(127.0.0.1:58289)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:46.049499236Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1806,"":"io task","client":"1","":"downstairs","session_id":"c41e812d-cc43-4ed3-97cb-a859445d713d"} {"msg":"ds_connection connected from Ok(127.0.0.1:60724)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:46.049599345Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1806,"":"io task","client":"0","":"downstairs","session_id":"c41e812d-cc43-4ed3-97cb-a859445d713d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:46.050160273Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1806,"client":"0","":"downstairs","session_id":"c41e812d-cc43-4ed3-97cb-a859445d713d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:46.05018877Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1806,"client":"1","":"downstairs","session_id":"c41e812d-cc43-4ed3-97cb-a859445d713d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:46.050810803Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1806,"client":"0","":"downstairs","session_id":"c41e812d-cc43-4ed3-97cb-a859445d713d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:46.05084052Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1806,"client":"1","":"downstairs","session_id":"c41e812d-cc43-4ed3-97cb-a859445d713d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:46.051384729Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1806,"session_id":"c41e812d-cc43-4ed3-97cb-a859445d713d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:46.051466847Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1806,"session_id":"c41e812d-cc43-4ed3-97cb-a859445d713d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:46.051748651Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1806,"client":"2","":"downstairs","session_id":"c41e812d-cc43-4ed3-97cb-a859445d713d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:46.052255117Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1806,"client":"2","":"downstairs","session_id":"c41e812d-cc43-4ed3-97cb-a859445d713d"} {"msg":"[0]R flush_numbers[0..12]: [272, 281, 281, 281, 281, 272, 273, 274, 278, 281, 281, 281]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:46.052733657Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1806,"":"downstairs","":"downstairs","session_id":"c41e812d-cc43-4ed3-97cb-a859445d713d"} {"msg":"[0]R generation[0..12]: [151, 154, 154, 154, 154, 151, 152, 152, 152, 154, 154, 154]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:46.052762254Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1806,"":"downstairs","":"downstairs","session_id":"c41e812d-cc43-4ed3-97cb-a859445d713d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:46.052781755Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1806,"":"downstairs","":"downstairs","session_id":"c41e812d-cc43-4ed3-97cb-a859445d713d"} {"msg":"[1]R flush_numbers[0..12]: [272, 281, 281, 281, 281, 272, 273, 274, 278, 281, 281, 281]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:46.052799356Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1806,"":"downstairs","":"downstairs","session_id":"c41e812d-cc43-4ed3-97cb-a859445d713d"} {"msg":"[1]R generation[0..12]: [151, 154, 154, 154, 154, 151, 152, 152, 152, 154, 154, 154]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:46.052817246Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1806,"":"downstairs","":"downstairs","session_id":"c41e812d-cc43-4ed3-97cb-a859445d713d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:46.052838759Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1806,"":"downstairs","":"downstairs","session_id":"c41e812d-cc43-4ed3-97cb-a859445d713d"} {"msg":"[2]R flush_numbers[0..12]: [272, 281, 281, 281, 281, 272, 273, 274, 278, 281, 281, 281]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:46.052857029Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1806,"":"downstairs","":"downstairs","session_id":"c41e812d-cc43-4ed3-97cb-a859445d713d"} {"msg":"[2]R generation[0..12]: [151, 154, 154, 154, 154, 151, 152, 152, 152, 154, 154, 154]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:46.052874769Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1806,"":"downstairs","":"downstairs","session_id":"c41e812d-cc43-4ed3-97cb-a859445d713d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-17T21:17:46.05289181Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1806,"":"downstairs","":"downstairs","session_id":"c41e812d-cc43-4ed3-97cb-a859445d713d"} {"msg":"Max found gen is 156","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:46.052914123Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1806,"":"downstairs","":"downstairs","session_id":"c41e812d-cc43-4ed3-97cb-a859445d713d"} {"msg":"Generation requested: 156 >= found:156","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:46.052930152Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1806,"":"downstairs","":"downstairs","session_id":"c41e812d-cc43-4ed3-97cb-a859445d713d"} {"msg":"Next flush: 283","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:46.052947032Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1806,"":"downstairs","":"downstairs","session_id":"c41e812d-cc43-4ed3-97cb-a859445d713d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:46.052963861Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1806,"":"downstairs","":"downstairs","session_id":"c41e812d-cc43-4ed3-97cb-a859445d713d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:46.052980951Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1806,"session_id":"c41e812d-cc43-4ed3-97cb-a859445d713d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:46.052996441Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1806,"client":"0","":"downstairs","session_id":"c41e812d-cc43-4ed3-97cb-a859445d713d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:46.05301287Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1806,"client":"1","":"downstairs","session_id":"c41e812d-cc43-4ed3-97cb-a859445d713d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:46.0530298Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1806,"client":"2","":"downstairs","session_id":"c41e812d-cc43-4ed3-97cb-a859445d713d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:46.05304666Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1806,"session_id":"c41e812d-cc43-4ed3-97cb-a859445d713d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:46.053062429Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1806,"session_id":"c41e812d-cc43-4ed3-97cb-a859445d713d"} {"msg":"a3317f56-3e6b-4da2-bd87-1d58c27c853f is now active with session: c41e812d-cc43-4ed3-97cb-a859445d713d","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2025-01-17T21:17:46.053078799Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1806,"session_id":"c41e812d-cc43-4ed3-97cb-a859445d713d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:46.053096309Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1806,"session_id":"c41e812d-cc43-4ed3-97cb-a859445d713d"} {"msg":"The guest has finished waiting for activation with:156","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:46.053113139Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1806} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 40 len 40960 data: 13 12 9 7 8 7 9 11 13 15 02/30 Write block 196 len 36864 data: 14 12 13 18 15 13 14 10 9 03/30 Write block 438 len 16384 data: 8 6 5 7 04/30 Read block 36 len 24576 05/30 Flush 06/30 Write block 432 len 40960 data: 12 9 12 9 9 10 9 7 6 8 07/30 Write block 102 len 12288 data: 13 16 16 08/30 Read block 43 len 32768 09/30 Read block 28 len 8192 10/30 Read block 340 len 8192 11/30 Read block 289 len 28672 12/30 Read block 290 len 16384 13/30 Flush 14/30 Flush 15/30 Write block 347 len 32768 data: 9 11 9 11 8 10 10 11 16/30 Read block 430 len 4096 17/30 Read block 86 len 28672 18/30 Write block 373 len 16384 data: 16 17 19 18 19/30 Write block 25 len 40960 data: 9 12 13 11 10 14 11 14 16 16 20/30 Read block 28 len 20480 21/30 Read block 203 len 24576 22/30 Read block 503 len 36864 23/30 Write block 332 len 28672 data: 9 8 12 13 12 15 16 24/30 Flush 25/30 Write block 316 len 16384 data: 10 10 12 12 26/30 Read block 128 len 20480 27/30 Write block 249 len 28672 data: 9 8 7 8 7 9 11 28/30 Write block 21 len 12288 data: 8 9 8 29/30 Write block 17 len 20480 data: 11 10 9 7 9 30/30 Write block 320 len 20480 data: 13 11 9 11 10 ---------------------------------------------------------------- Crucible gen:156 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 4 Done Done Done false 1025 Acked Read 5 Done Done Done false 1026 Acked Write 7 Done Done Done false 1027 Acked Write 3 Done Done Done false 1028 Acked Write 5 Sent Sent Sent false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:17:46.07232935Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1806} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 151 151 151 272 272 272 T T T 1 020-039 156 156 156 286 286 286 T T T 2 040-059 156 156 156 283 283 283 F F F 3 060-079 154 154 154 281 281 281 F F F 4 080-099 154 154 154 281 281 281 F F F 5 100-119 156 156 156 284 284 284 F F F 6 120-139 152 152 152 273 273 273 F F F 7 140-159 152 152 152 274 274 274 F F F 8 160-179 152 152 152 278 278 278 F F F 9 180-199 156 156 156 283 283 283 F F F 10 200-219 156 156 156 283 283 283 F F F 11 220-239 154 154 154 281 281 281 F F F 12 240-259 155 155 155 282 282 282 T T T 13 260-279 154 154 154 281 281 281 F F F 14 280-299 154 154 154 281 281 281 F F F 15 300-319 153 153 153 280 280 280 T T T 16 320-339 156 156 156 286 286 286 T T T 17 340-359 156 156 156 286 286 286 F F F 18 360-379 156 156 156 286 286 286 F F F 19 380-399 153 153 153 280 280 280 F F F 20 400-419 150 150 150 271 271 271 F F F 21 420-439 156 156 156 284 284 284 F F F 22 440-459 156 156 156 284 284 284 F F F 23 460-479 150 150 150 271 271 271 F F F 24 480-499 151 151 151 272 272 272 F F F 25 500-519 151 151 151 272 272 272 F F F 26 520-539 148 148 148 264 264 264 F F F 27 540-559 151 151 151 272 272 272 F F F 28 560-579 150 150 150 271 271 271 F F F 29 580-599 145 145 145 262 262 262 F F F Max gen: 156, Max flush: 286 A Difference in extent metadata is expected here On loop 78 Verifying data now No change after verify Loop: 78 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 157 157 157 287 287 287 F F F 1 020-039 157 157 157 287 287 287 F F F 2 040-059 156 156 156 283 283 283 F F F 3 060-079 154 154 154 281 281 281 F F F 4 080-099 154 154 154 281 281 281 F F F 5 100-119 156 156 156 284 284 284 F F F 6 120-139 152 152 152 273 273 273 F F F 7 140-159 152 152 152 274 274 274 F F F 8 160-179 152 152 152 278 278 278 F F F 9 180-199 156 156 156 283 283 283 F F F 10 200-219 156 156 156 283 283 283 F F F 11 220-239 154 154 154 281 281 281 F F F 12 240-259 157 157 157 287 287 287 F F F 13 260-279 154 154 154 281 281 281 F F F 14 280-299 154 154 154 281 281 281 F F F 15 300-319 157 157 157 287 287 287 F F F 16 320-339 157 157 157 287 287 287 F F F 17 340-359 156 156 156 286 286 286 F F F 18 360-379 156 156 156 286 286 286 F F F 19 380-399 153 153 153 280 280 280 F F F 20 400-419 150 150 150 271 271 271 F F F 21 420-439 156 156 156 284 284 284 F F F 22 440-459 156 156 156 284 284 284 F F F 23 460-479 150 150 150 271 271 271 F F F 24 480-499 151 151 151 272 272 272 F F F 25 500-519 151 151 151 272 272 272 F F F 26 520-539 148 148 148 264 264 264 F F F 27 540-559 151 151 151 272 272 272 F F F 28 560-579 150 150 150 271 271 271 F F F 29 580-599 145 145 145 262 262 262 F F F Max gen: 157, Max flush: 287 Begin loop 79 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:50.316348968Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1814,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:17:50.361312088Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1814,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:50.361766613Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1814,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:50.39361807Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1814,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:50.394214929Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1814} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"abd5430f1508af0bc008143ea2566277ef714ec2\",\n git_commit_timestamp: \"2025-01-17T20:55:03.000000000Z\",\n git_branch: \"alan/tokio-to-140\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:50.394249239Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1814} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:50.394265959Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1814} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:50.394280167Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1814} {"msg":"Crucible a69083cb-bc0a-428e-af14-d67b87c8ec69 has session id: 671d5754-cd2a-4014-a1c5-a83276510c61","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:50.394830208Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1814,"session_id":"671d5754-cd2a-4014-a1c5-a83276510c61"} {"msg":"Upstairs opts: Upstairs UUID: a69083cb-bc0a-428e-af14-d67b87c8ec69, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:50.394870542Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1814,"session_id":"671d5754-cd2a-4014-a1c5-a83276510c61"} {"msg":"Crucible stats registered with UUID: a69083cb-bc0a-428e-af14-d67b87c8ec69","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:50.394894055Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1814,"session_id":"671d5754-cd2a-4014-a1c5-a83276510c61"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:17:50.394913877Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1814,"session_id":"671d5754-cd2a-4014-a1c5-a83276510c61"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:50.39538086Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1814,"task":"crutest"} {"msg":"The guest has requested activation with gen:158","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:50.395403753Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1814} {"msg":"a69083cb-bc0a-428e-af14-d67b87c8ec69 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:50.395476506Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1814,"session_id":"671d5754-cd2a-4014-a1c5-a83276510c61"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:50.395493876Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1814,"client":"0","":"downstairs","session_id":"671d5754-cd2a-4014-a1c5-a83276510c61"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:50.395516139Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1814,"client":"0","":"downstairs","session_id":"671d5754-cd2a-4014-a1c5-a83276510c61"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:50.395529427Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1814,"client":"1","":"downstairs","session_id":"671d5754-cd2a-4014-a1c5-a83276510c61"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:50.395543365Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1814,"client":"1","":"downstairs","session_id":"671d5754-cd2a-4014-a1c5-a83276510c61"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:50.395556093Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1814,"client":"2","":"downstairs","session_id":"671d5754-cd2a-4014-a1c5-a83276510c61"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:50.395581117Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1814,"client":"2","":"downstairs","session_id":"671d5754-cd2a-4014-a1c5-a83276510c61"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:50.395604571Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1814,"":"io task","client":"2","":"downstairs","session_id":"671d5754-cd2a-4014-a1c5-a83276510c61"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:50.395663515Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1814,"":"io task","client":"0","":"downstairs","session_id":"671d5754-cd2a-4014-a1c5-a83276510c61"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:50.395690941Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1814,"":"io task","client":"1","":"downstairs","session_id":"671d5754-cd2a-4014-a1c5-a83276510c61"} {"msg":"ds_connection connected from Ok(127.0.0.1:33194)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:50.395715936Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1814,"":"io task","client":"2","":"downstairs","session_id":"671d5754-cd2a-4014-a1c5-a83276510c61"} {"msg":"ds_connection connected from Ok(127.0.0.1:48007)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:50.395759741Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1814,"":"io task","client":"1","":"downstairs","session_id":"671d5754-cd2a-4014-a1c5-a83276510c61"} {"msg":"ds_connection connected from Ok(127.0.0.1:49930)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:50.395788078Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1814,"":"io task","client":"0","":"downstairs","session_id":"671d5754-cd2a-4014-a1c5-a83276510c61"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:50.396363574Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1814,"client":"1","":"downstairs","session_id":"671d5754-cd2a-4014-a1c5-a83276510c61"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:50.396520816Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1814,"client":"0","":"downstairs","session_id":"671d5754-cd2a-4014-a1c5-a83276510c61"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:50.39657799Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1814,"client":"2","":"downstairs","session_id":"671d5754-cd2a-4014-a1c5-a83276510c61"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:50.397079053Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1814,"client":"1","":"downstairs","session_id":"671d5754-cd2a-4014-a1c5-a83276510c61"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:50.397107619Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1814,"client":"2","":"downstairs","session_id":"671d5754-cd2a-4014-a1c5-a83276510c61"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:50.39736576Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1814,"client":"0","":"downstairs","session_id":"671d5754-cd2a-4014-a1c5-a83276510c61"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:50.397660723Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1814,"session_id":"671d5754-cd2a-4014-a1c5-a83276510c61"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:50.397688889Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1814,"session_id":"671d5754-cd2a-4014-a1c5-a83276510c61"} {"msg":"[0]R flush_numbers[0..12]: [287, 287, 283, 281, 281, 284, 273, 274, 278, 283, 283, 281]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:50.397919084Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1814,"":"downstairs","":"downstairs","session_id":"671d5754-cd2a-4014-a1c5-a83276510c61"} {"msg":"[0]R generation[0..12]: [157, 157, 156, 154, 154, 156, 152, 152, 152, 156, 156, 154]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:50.397942247Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1814,"":"downstairs","":"downstairs","session_id":"671d5754-cd2a-4014-a1c5-a83276510c61"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:50.397957106Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1814,"":"downstairs","":"downstairs","session_id":"671d5754-cd2a-4014-a1c5-a83276510c61"} {"msg":"[1]R flush_numbers[0..12]: [287, 287, 283, 281, 281, 284, 273, 274, 278, 283, 283, 281]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:50.397970764Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1814,"":"downstairs","":"downstairs","session_id":"671d5754-cd2a-4014-a1c5-a83276510c61"} {"msg":"[1]R generation[0..12]: [157, 157, 156, 154, 154, 156, 152, 152, 152, 156, 156, 154]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:50.397984112Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1814,"":"downstairs","":"downstairs","session_id":"671d5754-cd2a-4014-a1c5-a83276510c61"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:50.39799793Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1814,"":"downstairs","":"downstairs","session_id":"671d5754-cd2a-4014-a1c5-a83276510c61"} {"msg":"[2]R flush_numbers[0..12]: [287, 287, 283, 281, 281, 284, 273, 274, 278, 283, 283, 281]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:50.398011908Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1814,"":"downstairs","":"downstairs","session_id":"671d5754-cd2a-4014-a1c5-a83276510c61"} {"msg":"[2]R generation[0..12]: [157, 157, 156, 154, 154, 156, 152, 152, 152, 156, 156, 154]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:50.398025436Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1814,"":"downstairs","":"downstairs","session_id":"671d5754-cd2a-4014-a1c5-a83276510c61"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:50.398038834Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1814,"":"downstairs","":"downstairs","session_id":"671d5754-cd2a-4014-a1c5-a83276510c61"} {"msg":"Max found gen is 158","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:50.398051311Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":Wait for a query_work_queue command to finish before sending IO 1814,"":"downstairs","":"downstairs","session_id":"671d5754-cd2a-4014-a1c5-a83276510c61"} {"msg":"Generation requested: 158 >= found:158","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:50.398115398Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1814,"":"downstairs","":"downstairs","session_id":"671d5754-cd2a-4014-a1c5-a83276510c61"} {"msg":"Next flush: 288","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:50.398137611Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1814,"":"downstairs","":"downstairs","session_id":"671d5754-cd2a-4014-a1c5-a83276510c61"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:50.398151599Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1814,"":"downstairs","":"downstairs","session_id":"671d5754-cd2a-4014-a1c5-a83276510c61"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:50.398168379Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1814,"session_id":"671d5754-cd2a-4014-a1c5-a83276510c61"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:50.398180507Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1814,"client":"0","":"downstairs","session_id":"671d5754-cd2a-4014-a1c5-a83276510c61"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:50.398194024Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1814,"client":"1","":"downstairs","session_id":"671d5754-cd2a-4014-a1c5-a83276510c61"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:50.398206982Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1814,"client":"2","":"downstairs","session_id":"671d5754-cd2a-4014-a1c5-a83276510c61"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:50.398219679Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1814,"session_id":"671d5754-cd2a-4014-a1c5-a83276510c61"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:50.398231957Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1814,"session_id":"671d5754-cd2a-4014-a1c5-a83276510c61"} {"msg":"a69083cb-bc0a-428e-af14-d67b87c8ec69 is now active with session: 671d5754-cd2a-4014-a1c5-a83276510c61","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:50.398244834Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1814,"session_id":"671d5754-cd2a-4014-a1c5-a83276510c61"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:50.398256541Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1814,"session_id":"671d5754-cd2a-4014-a1c5-a83276510c61"} {"msg":"The guest has finished waiting for activation with:158","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:50.398268368Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1814} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 146 len 16384 data: 9 12 10 11 02/30 Write block 241 len 28672 data: 12 12 11 11 9 9 11 03/30 Write block 333 len 4096 data: 9 04/30 Flush 05/30 Write block 272 len 24576 data: 9 9 10 12 13 13 06/30 Flush 07/30 Write block 94 len 32768 data: 16 16 16 11 14 13 13 14 08/30 Read block 166 len 36864 09/30 Read block 457 len 28672 10/30 Read block 130 len 28672 11/30 Read block 462 len 40960 12/30 Read block 385 len 12288 13/30 Read block 320 len 32768 14/30 Flush 15/30 Write block 394 len 40960 data: 10 10 12 13 13 11 10 11 10 10 16/30 Read block 82 len 4096 17/30 Write block 197 len 4096 data: 13 18/30 Write block 9 len 20480 data: 10 12 14 12 14 19/30 Write block 580 len 36864 data: 12 13 11 11 12 11 11 10 10 20/30 Write block 32 len 40960 data: 15 17 17 20 14 14 14 13 14 13 21/30 Read block 268 len 28672 22/30 Write block 515 len 40960 data: 23 22 18 16 15 13 12 9 17 17 23/30 Read block 324 len 20480 24/30 Write block 594 len 20480 data: 8 5 6 5 6 25/30 Read block 536 len 28672 26/30 Read block 519 len 36864 27/30 Write block 82 len 40960 data: 9 9 11 9 10 11 13 16 19 18 28/30 Write block 148 len 32768 data: 11 12 11 11 10 10 12 13 29/30 Write block 496 len 28672 data: 15 16 19 16 18 20 17 30/30 Read block 33 len 20480 ---------------------------------------------------------------- Crucible gen:158 GIO:true work queues: Upstairs:0 downstairs:16 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1014 Acked Write 10 Done Done Done false 1015 Acked Read 1 Done Done Done false 1016 Acked Write 1 Done Done Done false 1017 Acked Write 5 Done Done Done false 1018 Acked Write 9 Done Done Done false 1019 Acked Write 10 Done Done Done false 1020 Acked Read 7 Done Done Done false 1021 Acked Write 10 Done Done Done false 1022 Acked Read 5 Done Done Done false 1023 Acked Write 5 Done Done Done false 1024 Acked Read 7 Done Done Done false 1025 Acked Read 9 Done Done Done false 1026 Acked Write 10 Done Done Done false 1027 Acked Write 8 Done Done Done false 1028 Acked Write 7 Done Done Done false 1029 Acked Read 5 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 0 1 Done 15 16 16 47 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1013 1013 1013 Downstairs last five completed: 1013 1012 1011 1010 1009 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:17:50.418742474Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1814} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 157 157 157 287 287 287 T T T 1 020-039 157 157 157 287 287 287 T T T 2 040-059 156 156 156 283 283 283 T T T 3 060-079 154 154 154 281 281 281 F F F 4 080-099 158 158 158 290 290 290 T T T 5 100-119 158 158 158 290 290 290 F F F 6 120-139 152 152 152 273 273 273 F F F 7 140-159 158 158 158 288 288 288 T T T 8 160-179 152 152 152 278 278 278 F F F 9 180-199 156 156 156 283 283 283 T T T 10 200-219 156 156 156 283 283 283 F F F 11 220-239 154 154 154 281 281 281 F F F 12 240-259 158 158 158 288 288 288 F F F 13 260-279 158 158 158 289 289 289 F F F 14 280-299 154 154 154 281 281 281 F F F 15 300-319 157 157 157 287 287 287 F F F 16 320-339 158 158 158 288 288 288 F F F 17 340-359 156 156 156 286 286 286 F F F 18 360-379 156 156 156 286 286 286 F F F 19 380-399 153 153 153 280 280 280 T T T 20 400-419 150 150 150 271 271 271 T T T 21 420-439 156 156 156 284 284 284 F F F 22 440-459 156 156 156 284 284 284 F F F 23 460-479 150 150 150 271 271 271 F F F 24 480-499 151 151 151 272 272 272 T T T 25 500-519 151 151 151 272 272 272 T T T 26 520-539 148 148 148 264 264 264 T T T 27 540-559 151 151 151 272 272 272 F F F 28 560-579 150 150 150 271 271 271 F F F 29 580-599 145 145 145 262 262 262 T T T Max gen: 158, Max flush: 290 A Difference in extent metadata is expected here On loop 79 Verifying data now No change after verify Loop: 79 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 159 159 159 291 291 291 F F F 1 020-039 159 159 159 291 291 291 F F F 2 040-059 159 159 159 291 291 291 F F F 3 060-079 154 154 154 281 281 281 F F F 4 080-099 159 159 159 291 291 291 F F F 5 100-119 158 158 158 290 290 290 F F F 6 120-139 152 152 152 273 273 273 F F F 7 140-159 159 159 159 291 291 291 F F F 8 160-179 152 152 152 278 278 278 F F F 9 180-199 159 159 159 291 291 291 F F F 10 200-219 156 156 156 283 283 283 F F F 11 220-239 154 154 154 281 281 281 F F F 12 240-259 158 158 158 288 288 288 F F F 13 260-279 158 158 158 289 289 289 F F F 14 280-299 154 154 154 281 281 281 F F F 15 300-319 157 157 157 287 287 287 F F F 16 320-339 158 158 158 288 288 288 F F F 17 340-359 156 156 156 286 286 286 F F F 18 360-379 156 156 156 286 286 286 F F F 19 380-399 159 159 159 291 291 291 F F F 20 400-419 159 159 159 291 291 291 F F F 21 420-439 156 156 156 284 284 284 F F F 22 440-459 156 156 156 284 284 284 F F F 23 460-479 150 150 150 271 271 271 F F F 24 480-499 159 159 159 291 291 291 F F F 25 500-519 159 159 159 291 291 291 F F F 26 520-539 159 159 159 291 291 291 F F F 27 540-559 151 151 151 272 272 272 F F F 28 560-579 150 150 150 271 271 271 F F F 29 580-599 159 159 159 291 291 291 F F F Max gen: 159, Max flush: 291 Begin loop 80 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:54.73314587Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1822,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:54.777551263Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1822,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:54.778458793Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1822} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"abd5430f1508af0bc008143ea2566277ef714ec2\",\n git_commit_timestamp: \"2025-01-17T20:55:03.000000000Z\",\n git_branch: \"alan/tokio-to-140\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:54.778505Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1822} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:54.778523131Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1822} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:54.778542042Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1822} {"msg":"Crucible 4931c821-6898-44cf-9c5b-df691561a9a5 has session id: 0bc650c1-4e22-4630-bb34-0d1346992c7b","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:54.779092004Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1822,"session_id":"0bc650c1-4e22-4630-bb34-0d1346992c7b"} {"msg":"Upstairs opts: Upstairs UUID: 4931c821-6898-44cf-9c5b-df691561a9a5, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:54.779145485Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1822,"session_id":"0bc650c1-4e22-4630-bb34-0d1346992c7b"} {"msg":"Crucible stats registered with UUID: 4931c821-6898-44cf-9c5b-df691561a9a5","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:54.779170149Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1822,"session_id":"0bc650c1-4e22-4630-bb34-0d1346992c7b"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:17:54.779193183Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1822,"session_id":"0bc650c1-4e22-4630-bb34-0d1346992c7b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:54.779728565Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1822,"task":"crutest"} {"msg":"The guest has requested activation with gen:160","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:54.779758683Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1822} {"msg":"4931c821-6898-44cf-9c5b-df691561a9a5 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:54.779829584Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1822,"session_id":"0bc650c1-4e22-4630-bb34-0d1346992c7b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:54.779855099Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1822,"client":"0","":"downstairs","session_id":"0bc650c1-4e22-4630-bb34-0d1346992c7b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:54.779877973Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1822,"client":"0","":"downstairs","session_id":"0bc650c1-4e22-4630-bb34-0d1346992c7b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:54.779892011Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1822,"client":"1","":"downstairs","session_id":"0bc650c1-4e22-4630-bb34-0d1346992c7b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:54.779906089Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1822,"client":"1","":"downstairs","session_id":"0bc650c1-4e22-4630-bb34-0d1346992c7b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:54.779919047Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1822,"client":"2","":"downstairs","session_id":"0bc650c1-4e22-4630-bb34-0d1346992c7b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:54.779932675Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1822,"client":"2","":"downstairs","session_id":"0bc650c1-4e22-4630-bb34-0d1346992c7b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:54.779945362Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1822,"":"io task","client":"2","":"downstairs","session_id":"0bc650c1-4e22-4630-bb34-0d1346992c7b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:54.77995869Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1822,"":"io task","client":"0","":"downstairs","session_id":"0bc650c1-4e22-4630-bb34-0d1346992c7b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:54.779977981Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1822,"":"io task","client":"1","":"downstairs","session_id":"0bc650c1-4e22-4630-bb34-0d1346992c7b"} {"msg":"ds_connection connected from Ok(127.0.0.1:53422)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:54.780121105Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1822,"":"io task","client":"0","":"downstairs","session_id":"0bc650c1-4e22-4630-bb34-0d1346992c7b"} {"msg":"ds_connection connected from Ok(127.0.0.1:58863)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:54.780161409Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1822,"":"io task","client":"1","":"downstairs","session_id":"0bc650c1-4e22-4630-bb34-0d1346992c7b"} {"msg":"ds_connection connected from Ok(127.0.0.1:58974)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:54.780252122Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1822,"":"io task","client":"2","":"downstairs","session_id":"0bc650c1-4e22-4630-bb34-0d1346992c7b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:54.780808216Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1822,"client":"2","":"downstairs","session_id":"0bc650c1-4e22-4630-bb34-0d1346992c7b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:54.780971282Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1822,"client":"0","":"downstairs","session_id":"0bc650c1-4e22-4630-bb34-0d1346992c7b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:54.781354406Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1822,"client":"2","":"downstairs","session_id":"0bc650c1-4e22-4630-bb34-0d1346992c7b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:54.781460888Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1822,"client":"0","":"downstairs","session_id":"0bc650c1-4e22-4630-bb34-0d1346992c7b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:54.781676784Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1822,"client":"1","":"downstairs","session_id":"0bc650c1-4e22-4630-bb34-0d1346992c7b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:54.781900805Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1822,"session_id":"0bc650c1-4e22-4630-bb34-0d1346992c7b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:54.78206248Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1822,"session_id":"0bc650c1-4e22-4630-bb34-0d1346992c7b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:54.782244186Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1822,"client":"1","":"downstairs","session_id":"0bc650c1-4e22-4630-bb34-0d1346992c7b"} {"msg":"[0]R flush_numbers[0..12]: [291, 291, 291, 281, 291, 290, 273, 291, 278, 291, 283, 281]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:54.782732121Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1822,"":"downstairs","":"downstairs","session_id":"0bc650c1-4e22-4630-bb34-0d1346992c7b"} {"msg":"[0]R generation[0..12]: [159, 159, 159, 154, 159, 158, 152, 159, 152, 159, 156, 154]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:54.782752373Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1822,"":"downstairs","":"downstairs","session_id":"0bc650c1-4e22-4630-bb34-0d1346992c7b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:54.782767101Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1822,"":"downstairs","":"downstairs","session_id":"0bc650c1-4e22-4630-bb34-0d1346992c7b"} {"msg":"[1]R flush_numbers[0..12]: [291, 291, 291, 281, 291, 290, 273, 291, 278, 291, 283, 281]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:54.782780739Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1822,"":"downstairs","":"downstairs","session_id":"0bc650c1-4e22-4630-bb34-0d1346992c7b"} {"msg":"[1]R generation[0..12]: [159, 159, 159, 154, 159, 158, 152, 159, 152, 159, 156, 154]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:54.782794147Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1822,"":"downstairs","":"downstairs","session_id":"0bc650c1-4e22-4630-bb34-0d1346992c7b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:54.782808055Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1822,"":"downstairs","":"downstairs","session_id":"0bc650c1-4e22-4630-bb34-0d1346992c7b"} {"msg":"[2]R flush_numbers[0..12]: [291, 291, 291, 281, 291, 290, 273, 291, 278, 291, 283, 281]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:54.782825335Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1822,"":"downstairs","":"downstairs","session_id":"0bc650c1-4e22-4630-bb34-0d1346992c7b"} {"msg":"[2]R generation[0..12]: [159, 159, 159, 154, 159, 158, 152, 159, 152, 159, 156, 154]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:54.782839163Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1822,"":"downstairs","":"downstairs","session_id":"0bc650c1-4e22-4630-bb34-0d1346992c7b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:54.782852711Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1822,"":"downstairs","":"downstairs","session_id":"0bc650c1-4e22-4630-bb34-0d1346992c7b"} {"msg":"Max found gen is 160","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:54.782865289Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1822,"":"downstairs","":"downstairs","session_id":"0bc650c1-4e22-4630-bb34-0d1346992c7b"} {"msg":"Generation requested: 160 >= found:160","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:54.782878176Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1822,"":"downstairs","":"downstairs","session_id":"0bc650c1-4e22-4630-bb34-0d1346992c7b"} {"msg":"Next flush: 292","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:54.782890644Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1822,"":"downstairs","":"downstairs","session_id":"0bc650c1-4e22-4630-bb34-0d1346992c7b"} {"msg":"All extents match","v":Wait for a query_work_queue command to finish before sending IO 0,"name":"crucible","level":30,"time":"2025-01-17T21:17:54.782909175Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1822,"":"downstairs","":"downstairs","session_id":"0bc650c1-4e22-4630-bb34-0d1346992c7b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:54.782921922Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1822,"session_id":"0bc650c1-4e22-4630-bb34-0d1346992c7b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:54.782933439Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1822,"client":"0","":"downstairs","session_id":"0bc650c1-4e22-4630-bb34-0d1346992c7b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:54.782946286Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1822,"client":"1","":"downstairs","session_id":"0bc650c1-4e22-4630-bb34-0d1346992c7b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:54.782958924Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1822,"client":"2","":"downstairs","session_id":"0bc650c1-4e22-4630-bb34-0d1346992c7b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:54.782971711Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1822,"session_id":"0bc650c1-4e22-4630-bb34-0d1346992c7b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:54.782983428Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1822,"session_id":"0bc650c1-4e22-4630-bb34-0d1346992c7b"} {"msg":"4931c821-6898-44cf-9c5b-df691561a9a5 is now active with session: 0bc650c1-4e22-4630-bb34-0d1346992c7b","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:54.783009223Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1822,"session_id":"0bc650c1-4e22-4630-bb34-0d1346992c7b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:54.78302229Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1822,"session_id":"0bc650c1-4e22-4630-bb34-0d1346992c7b"} {"msg":"The guest has finished waiting for activation with:160","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:54.783034688Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1822} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 244 len 8192 data: 12 10 02/30 Read block 529 len 16384 03/30 Write block 268 len 36864 data: 8 10 11 10 10 10 11 13 14 04/30 Write block 129 len 12288 data: 11 12 10 05/30 Read block 496 len 24576 06/30 Read block 49 len 36864 07/30 Read block 402 len 16384 08/30 Write block 348 len 20480 data: 12 10 12 9 11 09/30 Write block 145 len 4096 data: 10 10/30 Write block 312 len 24576 data: 9 11 11 9 11 11 11/30 Read block 584 len 32768 12/30 Read block 545 len 16384 13/30 Write block 305 len 24576 data: 12 10 10 9 6 5 14/30 Read block 77 len 4096 15/30 Write block 195 len 28672 data: 14 15 14 14 19 16 14 16/30 Write block 39 len 32768 data: 14 15 14 10 8 9 8 10 17/30 Read block 174 len 16384 18/30 Write block 221 len 40960 data: 8 10 10 8 9 12 13 14 15 16 19/30 Read block 231 len 28672 20/30 Write block 303 len 20480 data: 12 11 13 11 11 21/30 Read block 412 len 32768 22/30 Write block 66 len 20480 data: 10 9 12 11 11 23/30 Read block 95 len 12288 24/30 Read block 351 len 12288 25/30 Write block 532 len 24576 data: 12 12 11 10 9 10 26/30 Write block 542 len 28672 data: 10 12 9 9 11 11 12 27/30 Write block 382 len 16384 data: 12 12 12 11 28/30 Read block 474 len 24576 29/30 Read block 491 len 40960 30/30 Read block 134 len 40960 ---------------------------------------------------------------- Crucible gen:160 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 2 Done Done Done false 1001 Acked Read 4 Done Done Done false 1002 Acked Write 9 Done Done Done false 1003 Acked Write 3 Done Done Done false 1004 Acked Read 6 Done Done Done false 1005 Acked Read 9 Done Done Done false 1006 Acked Read 4 Done Done Done false 1007 Acked Write 5 Done Done Done false 1008 Acked Write 1 Done Done Done false 1009 Acked Write 6 Done Done Done false 1010 Acked Read 8 Done Done Done false 1011 Acked Read 4 Done Done Done false 1012 Acked Write 6 Done Done Done false 1013 Acked Read 1 Done Done Done false 1014 Acked Write 7 Done Done Done false 1015 Acked Write 8 Done Done Done false 1016 Acked Read 4 Done Done Done false 1017 Acked Write 10 Done Done Done false 1018 Acked Read 7 Done Done Done false 1019 Acked Write 5 Done Done Done false 1020 Acked Read 8 Done Done Done false 1021 Acked Write 5 Done Done Done false 1022 Acked Read 3 Done Done Done false 1023 Acked Read 3 Done Done Done false 1024 Acked Write 6 Done Done Done false 1025 Acked Write 7 Done Done Done false 1026 Acked Write 4 Done Done Done false 1027 Acked Read 6 Done Done Done false 1028 Acked Read 10 Done Done Done false 1029 Acked Read 10 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 30 30 30 90 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 0 0 0 Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:17:54.803157279Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1822} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 159 159 159 291 291 291 F F F 1 020-039 159 159 159 291 291 291 T T T 2 040-059 159 159 159 291 291 291 T T T 3 060-079 154 154 154 281 281 281 T T T 4 080-099 159 159 159 291 291 291 F F F 5 100-119 158 158 158 290 290 290 F F F 6 120-139 152 152 152 273 273 273 T T T 7 140-159 159 159 159 291 291 291 T T T 8 160-179 152 152 152 278 278 278 F F F 9 180-199 159 159 159 291 291 291 T T T 10 200-219 156 156 156 283 283 283 T T T 11 220-239 154 154 154 281 281 281 T T T 12 240-259 158 158 158 288 288 288 T T T 13 260-279 158 158 158 289 289 289 T T T 14 280-299 154 154 154 281 281 281 F F F 15 300-319 157 157 157 287 287 287 T T T 16 320-339 158 158 158 288 288 288 F F F 17 340-359 156 156 156 286 286 286 T T T 18 360-379 156 156 156 286 286 286 F F F 19 380-399 159 159 159 291 291 291 T T T 20 400-419 159 159 159 291 291 291 F F F 21 420-439 156 156 156 284 284 284 F F F 22 440-459 156 156 156 284 284 284 F F F 23 460-479 150 150 150 271 271 271 F F F 24 480-499 159 159 159 291 291 291 F F F 25 500-519 159 159 159 291 291 291 F F F 26 520-539 159 159 159 291 291 291 T T T 27 540-559 151 151 151 272 272 272 T T T 28 560-579 150 150 150 271 271 271 F F F 29 580-599 159 159 159 291 291 291 F F F Max gen: 159, Max flush: 291 A Difference in extent metadata is expected here On loop 80 Verifying data now No change after verify Loop: 80 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 159 159 159 291 291 291 F F F 1 020-039 160 160 160 292 292 292 F F F 2 040-059 160 160 160 292 292 292 F F F 3 060-079 160 160 160 292 292 292 F F F 4 080-099 159 159 159 291 291 291 F F F 5 100-119 158 158 158 290 290 290 F F F 6 120-139 160 160 160 292 292 292 F F F 7 140-159 160 160 160 292 292 292 F F F 8 160-179 152 152 152 278 278 278 F F F 9 180-199 160 160 160 292 292 292 F F F 10 200-219 160 160 160 292 292 292 F F F 11 220-239 160 160 160 292 292 292 F F F 12 240-259 160 160 160 292 292 292 F F F 13 260-279 160 160 160 292 292 292 F F F 14 280-299 154 154 154 281 281 281 F F F 15 300-319 160 160 160 292 292 292 F F F 16 320-339 158 158 158 288 288 288 F F F 17 340-359 160 160 160 292 292 292 F F F 18 360-379 156 156 156 286 286 286 F F F 19 380-399 160 160 160 292 292 292 F F F 20 400-419 159 159 159 291 291 291 F F F 21 420-439 156 156 156 284 284 284 F F F 22 440-459 156 156 156 284 284 284 F F F 23 460-479 150 150 150 271 271 271 F F F 24 480-499 159 159 159 291 291 291 F F F 25 500-519 159 159 159 291 291 291 F F F 26 520-539 160 160 160 292 292 292 F F F 27 540-559 160 160 160 292 292 292 F F F 28 560-579 150 150 150 271 271 271 F F F 29 580-599 159 159 159 291 291 291 F F F Max gen: 160, Max flush: 292 Begin loop 81 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:09.104702592Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1830,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:09.14846515Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1830,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:09.149134081Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1830} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"abd5430f1508af0bc008143ea2566277ef714ec2\",\n git_commit_timestamp: \"2025-01-17T20:55:03.000000000Z\",\n git_branch: \"alan/tokio-to-140\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:09.149173083Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1830} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:09.149187232Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1830} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:09.14920208Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1830} {"msg":"Crucible 6477b4a9-36ac-44fe-8b12-19bdec2dea3b has session id: b2c495f8-0a66-48a4-a23c-7449904b278d","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:09.149626558Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1830,"session_id":"b2c495f8-0a66-48a4-a23c-7449904b278d"} {"msg":"Upstairs opts: Upstairs UUID: 6477b4a9-36ac-44fe-8b12-19bdec2dea3b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:09.149665251Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1830,"session_id":"b2c495f8-0a66-48a4-a23c-7449904b278d"} {"msg":"Crucible stats registered with UUID: 6477b4a9-36ac-44fe-8b12-19bdec2dea3b","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:09.149688144Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1830,"session_id":"b2c495f8-0a66-48a4-a23c-7449904b278d"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:18:09.149710097Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1830,"session_id":"b2c495f8-0a66-48a4-a23c-7449904b278d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:09.150158089Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1830,"task":"crutest"} {"msg":"The guest has requested activation with gen:162","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:09.150182183Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1830} {"msg":"6477b4a9-36ac-44fe-8b12-19bdec2dea3b active request set","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:09.15024533Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1830,"session_id":"b2c495f8-0a66-48a4-a23c-7449904b278d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:09.15026236Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1830,"client":"0","":"downstairs","session_id":"b2c495f8-0a66-48a4-a23c-7449904b278d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:09.150292087Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1830,"client":"0","":"downstairs","session_id":"b2c495f8-0a66-48a4-a23c-7449904b278d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:09.150328088Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1830,"client":"1","":"downstairs","session_id":"b2c495f8-0a66-48a4-a23c-7449904b278d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:09.150359587Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1830,"client":"1","":"downstairs","session_id":"b2c495f8-0a66-48a4-a23c-7449904b278d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:09.150386422Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1830,"client":"2","":"downstairs","session_id":"b2c495f8-0a66-48a4-a23c-7449904b278d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:09.150411928Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1830,"client":"2","":"downstairs","session_id":"b2c495f8-0a66-48a4-a23c-7449904b278d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:09.150437943Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1830,"":"io task","client":"2","":"downstairs","session_id":"b2c495f8-0a66-48a4-a23c-7449904b278d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:09.150464968Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1830,"":"io task","client":"0","":"downstairs","session_id":"b2c495f8-0a66-48a4-a23c-7449904b278d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:09.150492154Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1830,"":"io task","client":"1","":"downstairs","session_id":"b2c495f8-0a66-48a4-a23c-7449904b278d"} {"msg":"ds_connection connected from Ok(127.0.0.1:62531)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:09.1505191Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1830,"":"io task","client":"0","":"downstairs","session_id":"b2c495f8-0a66-48a4-a23c-7449904b278d"} {"msg":"ds_connection connected from Ok(127.0.0.1:65035)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:09.150547707Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1830,"":"io task","client":"2","":"downstairs","session_id":"b2c495f8-0a66-48a4-a23c-7449904b278d"} {"msg":"ds_connection connected from Ok(127.0.0.1:44717)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:09.150615396Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1830,"":"io task","client":"1","":"downstairs","session_id":"b2c495f8-0a66-48a4-a23c-7449904b278d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:09.151131968Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1830,"client":"0","":"downstairs","session_id":"b2c495f8-0a66-48a4-a23c-7449904b278d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:09.151170841Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1830,"client":"2","":"downstairs","session_id":"b2c495f8-0a66-48a4-a23c-7449904b278d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:09.151808284Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1830,"client":"2","":"downstairs","session_id":"b2c495f8-0a66-48a4-a23c-7449904b278d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:09.151940151Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1830,"client":"0","":"downstairs","session_id":"b2c495f8-0a66-48a4-a23c-7449904b278d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:09.152151244Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1830,"client":"1","":"downstairs","session_id":"b2c495f8-0a66-48a4-a23c-7449904b278d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:09.152409045Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1830,"session_id":"b2c495f8-0a66-48a4-a23c-7449904b278d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:09.152574721Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1830,"session_id":"b2c495f8-0a66-48a4-a23c-7449904b278d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:09.152749013Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1830,"client":"1","":"downstairs","session_id":"b2c495f8-0a66-48a4-a23c-7449904b278d"} {"msg":"[0]R flush_numbers[0..12]: [291, 292, 292, 292, 291, 290, 292, 292, 278, 292, 292, 292]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:09.153229784Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1830,"":"downstairs","":"downstairs","session_id":"b2c495f8-0a66-48a4-a23c-7449904b278d"} {"msg":"[0]R generation[0..12]: [159, 160, 160, 160, 159, 158, 160, 160, 152, 160, 160, 160]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:09.153254199Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1830,"":"downstairs","":"downstairs","session_id":"b2c495f8-0a66-48a4-a23c-7449904b278d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:09.153269037Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1830,"":"downstairs","":"downstairs","session_id":"b2c495f8-0a66-48a4-a23c-7449904b278d"} {"msg":"[1]R flush_numbers[0..12]: [291, 292, 292, 292, 291, 290, 292, 292, 278, 292, 292, 292]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:09.153284126Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1830,"":"downstairs","":"downstairs","session_id":"b2c495f8-0a66-48a4-a23c-7449904b278d"} {"msg":"[1]R generation[0..12]: [159, 160, 160, 160, 159, 158, 160, 160, 152, 160, 160, 160]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:09.153298444Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1830,"":"downstairs","":"downstairs","session_id":"b2c495f8-0a66-48a4-a23c-7449904b278d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:09.153313123Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1830,"":"downstairs","":"downstairs","session_id":"b2c495f8-0a66-48a4-a23c-7449904b278d"} {"msg":"[2]R flush_numbers[0..12]: [291, 292, 292, 292, 291, 290, 292, 292, 278, 292, 292, 292]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:09.153326651Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1830,"":"downstairs","":"downstairs","session_id":"b2c495f8-0a66-48a4-a23c-7449904b278d"} {"msg":"[2]R generation[0..12]: [159, 160, 160, 160, 159, 158, 160, 160, 152, 160, 160, 160]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:09.153340059Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1830,"":"downstairs","":"downstairs","session_id":"b2c495f8-0a66-48a4-a23c-7449904b278d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:09.153353497Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1830,"":"downstairs","":"downstairs","session_id":"b2c495f8-0a66-48a4-a23c-7449904b278d"} {"msg":"Max found gen is 161","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:09.153366755Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1830,"":"downstairs","":"downstairs","session_id":"b2c495f8-0a66-48a4-a23c-7449904b278d"} {"msg":"Generation requested: 162 >= found:161","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:09.153379552Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1830,"":"downstairs","":"downstairs","session_id":"b2c495f8-0a66-48a4-a23c-7449904b278d"} Wait for a query_work_queue command to finish before sending IO {"msg":"Next flush: 293","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:09.153402085Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1830,"":"downstairs","":"downstairs","session_id":"b2c495f8-0a66-48a4-a23c-7449904b278d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:09.153416103Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1830,"":"downstairs","":"downstairs","session_id":"b2c495f8-0a66-48a4-a23c-7449904b278d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:09.153429611Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1830,"session_id":"b2c495f8-0a66-48a4-a23c-7449904b278d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:09.153441808Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1830,"client":"0","":"downstairs","session_id":"b2c495f8-0a66-48a4-a23c-7449904b278d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:09.153455406Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1830,"client":"1","":"downstairs","session_id":"b2c495f8-0a66-48a4-a23c-7449904b278d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:09.153468784Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1830,"client":"2","":"downstairs","session_id":"b2c495f8-0a66-48a4-a23c-7449904b278d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:09.153481462Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1830,"session_id":"b2c495f8-0a66-48a4-a23c-7449904b278d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:09.153493389Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1830,"session_id":"b2c495f8-0a66-48a4-a23c-7449904b278d"} {"msg":"6477b4a9-36ac-44fe-8b12-19bdec2dea3b is now active with session: b2c495f8-0a66-48a4-a23c-7449904b278d","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:09.153506466Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1830,"session_id":"b2c495f8-0a66-48a4-a23c-7449904b278d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:09.153526288Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1830,"session_id":"b2c495f8-0a66-48a4-a23c-7449904b278d"} {"msg":"The guest has finished waiting for activation with:162","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:09.153539776Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1830} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 561 len 4096 data: 16 02/30 Read block 135 len 12288 03/30 Write block 468 len 16384 data: 14 15 14 15 04/30 Read block 394 len 28672 05/30 Read block 218 len 28672 06/30 Write block 72 len 12288 data: 13 12 11 07/30 Write block 415 len 24576 data: 13 12 11 9 11 11 08/30 Write block 249 len 32768 data: 10 9 8 9 8 10 12 11 09/30 Write block 1 len 40960 data: 7 10 9 11 12 11 12 12 11 13 10/30 Read block 82 len 16384 11/30 Read block 528 len 24576 12/30 Read block 467 len 40960 13/30 Write block 312 len 32768 data: 10 12 12 10 12 12 13 13 14/30 Flush 15/30 Write block 551 len 16384 data: 10 9 7 8 16/30 Read block 570 len 36864 17/30 Read block 202 len 40960 18/30 Write block 512 len 20480 data: 18 19 21 24 23 19/30 Flush 20/30 Write block 143 len 4096 data: 11 21/30 Read block 344 len 28672 22/30 Read block 397 len 8192 23/30 Flush 24/30 Write block 558 len 16384 data: 14 12 16 17 25/30 Read block 401 len 40960 26/30 Write block 269 len 8192 data: 11 12 27/30 Write block 238 len 12288 data: 10 7 10 28/30 Read block 396 len 40960 29/30 Write block 234 len 36864 data: 15 17 14 15 11 8 11 13 13 30/30 Read block 302 len 4096 ---------------------------------------------------------------- Crucible gen:162 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 4 Done Done Done false 1024 Acked Read 10 Done Done Done false 1025 Acked Write 2 Done Done Done false 1026 Acked Write 3 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Write 9 Done Done Done false 1029 Acked Read 1 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 7 7 7 21 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:18:09.173157291Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1830} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 162 162 162 293 293 293 F F F 1 020-039 160 160 160 292 292 292 F F F 2 040-059 160 160 160 292 292 292 F F F 3 060-079 162 162 162 293 293 293 F F F 4 080-099 159 159 159 291 291 291 F F F 5 100-119 158 158 158 290 290 290 F F F 6 120-139 160 160 160 292 292 292 F F F 7 140-159 162 162 162 295 295 295 F F F 8 160-179 152 152 152 278 278 278 F F F 9 180-199 160 160 160 292 292 292 F F F 10 200-219 160 160 160 292 292 292 F F F 11 220-239 160 160 160 292 292 292 T T T 12 240-259 162 162 162 293 293 293 T T T 13 260-279 160 160 160 292 292 292 T T T 14 280-299 154 154 154 281 281 281 F F F 15 300-319 162 162 162 293 293 293 F F F 16 320-339 158 158 158 288 288 288 F F F 17 340-359 160 160 160 292 292 292 F F F 18 360-379 156 156 156 286 286 286 F F F 19 380-399 160 160 160 292 292 292 F F F 20 400-419 162 162 162 293 293 293 F F F 21 420-439 162 162 162 293 293 293 F F F 22 440-459 156 156 156 284 284 284 F F F 23 460-479 162 162 162 293 293 293 F F F 24 480-499 159 159 159 291 291 291 F F F 25 500-519 162 162 162 294 294 294 F F F 26 520-539 160 160 160 292 292 292 F F F 27 540-559 162 162 162 294 294 294 T T T 28 560-579 162 162 162 293 293 293 T T T 29 580-599 159 159 159 291 291 291 F F F Max gen: 162, Max flush: 295 A Difference in extent metadata is expected here On loop 81 Verifying data now No change after verify Loop: 81 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 162 162 162 293 293 293 F F F 1 020-039 160 160 160 292 292 292 F F F 2 040-059 160 160 160 292 292 292 F F F 3 060-079 162 162 162 293 293 293 F F F 4 080-099 159 159 159 291 291 291 F F F 5 100-119 158 158 158 290 290 290 F F F 6 120-139 160 160 160 292 292 292 F F F 7 140-159 162 162 162 295 295 295 F F F 8 160-179 152 152 152 278 278 278 F F F 9 180-199 160 160 160 292 292 292 F F F 10 200-219 160 160 160 292 292 292 F F F 11 220-239 163 163 163 296 296 296 F F F 12 240-259 163 163 163 296 296 296 F F F 13 260-279 163 163 163 296 296 296 F F F 14 280-299 154 154 154 281 281 281 F F F 15 300-319 162 162 162 293 293 293 F F F 16 320-339 158 158 158 288 288 288 F F F 17 340-359 160 160 160 292 292 292 F F F 18 360-379 156 156 156 286 286 286 F F F 19 380-399 160 160 160 292 292 292 F F F 20 400-419 162 162 162 293 293 293 F F F 21 420-439 162 162 162 293 293 293 F F F 22 440-459 156 156 156 284 284 284 F F F 23 460-479 162 162 162 293 293 293 F F F 24 480-499 159 159 159 291 291 291 F F F 25 500-519 162 162 162 294 294 294 F F F 26 520-539 160 160 160 292 292 292 F F F 27 540-559 163 163 163 296 296 296 F F F 28 560-579 163 163 163 296 296 296 F F F 29 580-599 159 159 159 291 291 291 F F F Max gen: 163, Max flush: 296 Begin loop 82 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:13.410037245Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1838,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:13.455373013Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1838,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:13.456025324Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1838} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"abd5430f1508af0bc008143ea2566277ef714ec2\",\n git_commit_timestamp: \"2025-01-17T20:55:03.000000000Z\",\n git_branch: \"alan/tokio-to-140\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:13.456065758Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1838} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:13.456080326Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1838} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:13.456095135Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1838} {"msg":"Crucible b3f9fa94-5126-4852-bb07-aeab09eae79d has session id: 501281ed-41ce-438e-91b4-f1f41dfb6d37","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:13.45665134Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1838,"session_id":"501281ed-41ce-438e-91b4-f1f41dfb6d37"} {"msg":"Upstairs opts: Upstairs UUID: b3f9fa94-5126-4852-bb07-aeab09eae79d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:13.456700089Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1838,"session_id":"501281ed-41ce-438e-91b4-f1f41dfb6d37"} {"msg":"Crucible stats registered with UUID: b3f9fa94-5126-4852-bb07-aeab09eae79d","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:13.456725203Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1838,"session_id":"501281ed-41ce-438e-91b4-f1f41dfb6d37"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:18:13.456763496Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1838,"session_id":"501281ed-41ce-438e-91b4-f1f41dfb6d37"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:13.457270852Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1838,"task":"crutest"} {"msg":"The guest has requested activation with gen:164","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:13.457295096Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1838} {"msg":"b3f9fa94-5126-4852-bb07-aeab09eae79d active request set","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:13.457360715Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1838,"session_id":"501281ed-41ce-438e-91b4-f1f41dfb6d37"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:13.457377665Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1838,"client":"0","":"downstairs","session_id":"501281ed-41ce-438e-91b4-f1f41dfb6d37"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:13.457401929Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1838,"client":"0","":"downstairs","session_id":"501281ed-41ce-438e-91b4-f1f41dfb6d37"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:13.457415777Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1838,"client":"1","":"downstairs","session_id":"501281ed-41ce-438e-91b4-f1f41dfb6d37"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:13.457429635Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1838,"client":"1","":"downstairs","session_id":"501281ed-41ce-438e-91b4-f1f41dfb6d37"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:13.457442492Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1838,"client":"2","":"downstairs","session_id":"501281ed-41ce-438e-91b4-f1f41dfb6d37"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:13.457479214Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1838,"client":"2","":"downstairs","session_id":"501281ed-41ce-438e-91b4-f1f41dfb6d37"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:13.457492822Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1838,"":"io task","client":"2","":"downstairs","session_id":"501281ed-41ce-438e-91b4-f1f41dfb6d37"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:13.45750706Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1838,"":"io task","client":"0","":"downstairs","session_id":"501281ed-41ce-438e-91b4-f1f41dfb6d37"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:13.457520408Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1838,"":"io task","client":"1","":"downstairs","session_id":"501281ed-41ce-438e-91b4-f1f41dfb6d37"} {"msg":"ds_connection connected from Ok(127.0.0.1:41156)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:13.457595462Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1838,"":"io task","client":"0","":"downstairs","session_id":"501281ed-41ce-438e-91b4-f1f41dfb6d37"} {"msg":"ds_connection connected from Ok(127.0.0.1:62294)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:13.457635185Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1838,"":"io task","client":"1","":"downstairs","session_id":"501281ed-41ce-438e-91b4-f1f41dfb6d37"} {"msg":"ds_connection connected from Ok(127.0.0.1:38075)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:13.457793858Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1838,"":"io task","client":"2","":"downstairs","session_id":"501281ed-41ce-438e-91b4-f1f41dfb6d37"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:13.458428739Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1838,"client":"0","":"downstairs","session_id":"501281ed-41ce-438e-91b4-f1f41dfb6d37"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:13.458469693Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1838,"client":"2","":"downstairs","session_id":"501281ed-41ce-438e-91b4-f1f41dfb6d37"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:13.458498069Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1838,"client":"1","":"downstairs","session_id":"501281ed-41ce-438e-91b4-f1f41dfb6d37"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:13.459048741Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1838,"client":"0","":"downstairs","session_id":"501281ed-41ce-438e-91b4-f1f41dfb6d37"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:13.459182239Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1838,"client":"1","":"downstairs","session_id":"501281ed-41ce-438e-91b4-f1f41dfb6d37"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:13.459201971Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1838,"client":"2","":"downstairs","session_id":"501281ed-41ce-438e-91b4-f1f41dfb6d37"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:13.459569266Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1838,"session_id":"501281ed-41ce-438e-91b4-f1f41dfb6d37"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:13.459770583Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1838,"session_id":"501281ed-41ce-438e-91b4-f1f41dfb6d37"} {"msg":"[0]R flush_numbers[0..12]: [293, 292, 292, 293, 291, 290, 292, 295, 278, 292, 292, 296]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:13.459794697Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1838,"":"downstairs","":"downstairs","session_id":"501281ed-41ce-438e-91b4-f1f41dfb6d37"} {"msg":"[0]R generation[0..12]: [162, 160, 160, 162, 159, 158, 160, 162, 152, 160, 160, 163]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:13.459809936Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1838,"":"downstairs","":"downstairs","session_id":"501281ed-41ce-438e-91b4-f1f41dfb6d37"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:13.459824485Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1838,"":"downstairs","":"downstairs","session_id":"501281ed-41ce-438e-91b4-f1f41dfb6d37"} {"msg":"[1]R flush_numbers[0..12]: [293, 292, 292, 293, 291, 290, 292, 295, 278, 292, 292, 296]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:13.459838213Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1838,"":"downstairs","":"downstairs","session_id":"501281ed-41ce-438e-91b4-f1f41dfb6d37"} {"msg":"[1]R generation[0..12]: [162, 160, 160, 162, 159, 158, 160, 162, 152, 160, 160, 163]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:13.459851691Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1838,"":"downstairs","":"downstairs","session_id":"501281ed-41ce-438e-91b4-f1f41dfb6d37"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:13.459865279Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1838,"":"downstairs","":"downstairs","session_id":"501281ed-41ce-438e-91b4-f1f41dfb6d37"} {"msg":"[2]R flush_numbers[0..12]: [293, 292, 292, 293, 291, 290, 292, 295, 278, 292, 292, 296]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:13.459881868Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1838,"":"downstairs","":"downstairs","session_id":"501281ed-41ce-438e-91b4-f1f41dfb6d37"} {"msg":"[2]R generation[0..12]: [162, 160, 160, 162, 159, 158, 160, 162, 152, 160, 160, 163]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:13.459895626Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1838,"":"downstairs","":"downstairs","session_id":"501281ed-41ce-438e-91b4-f1f41dfb6d37"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:13.459909154Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1838,"":"downstairs","":"downstairs","session_id":"501281ed-41ce-438e-91b4-f1f41dfb6d37"} {"msg":"Max found gen is 164","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:13.459921511Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1838,"":"downstairs","":"downstairs","session_id":"501281ed-41ce-438e-91b4-f1f41dfb6d37"} {"msg":"Generation requested: 164 >= found:164","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:13.459934319Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1838,"":"downstairs","":"downstairs","session_id":"501281ed-41ce-438e-91b4-f1f41dfb6d37"} {"msg":"Next flush: 297","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:13.459946766Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1838,"":"downstairs","":"downstairs","session_id":"501281ed-41ce-438e-91b4-f1f41dfb6d37"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:13.459959153Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1838,"":"downstairs","":"downstairs","session_id":"501281ed-41ce-438e-91b4-f1f41dfb6d37"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:13.459972131Z","hostname":"Wait for a query_work_queue command to finish before sending IO w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1838,"session_id":"501281ed-41ce-438e-91b4-f1f41dfb6d37"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:13.459991782Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1838,"client":"0","":"downstairs","session_id":"501281ed-41ce-438e-91b4-f1f41dfb6d37"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:13.460015727Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1838,"client":"1","":"downstairs","session_id":"501281ed-41ce-438e-91b4-f1f41dfb6d37"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:13.460034828Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1838,"client":"2","":"downstairs","session_id":"501281ed-41ce-438e-91b4-f1f41dfb6d37"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:13.460048215Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1838,"session_id":"501281ed-41ce-438e-91b4-f1f41dfb6d37"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:13.460060253Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1838,"session_id":"501281ed-41ce-438e-91b4-f1f41dfb6d37"} {"msg":"b3f9fa94-5126-4852-bb07-aeab09eae79d is now active with session: 501281ed-41ce-438e-91b4-f1f41dfb6d37","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:13.46007257Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1838,"session_id":"501281ed-41ce-438e-91b4-f1f41dfb6d37"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:13.460084236Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1838,"session_id":"501281ed-41ce-438e-91b4-f1f41dfb6d37"} {"msg":"The guest has finished waiting for activation with:164","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:13.460096114Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1838} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 26 len 16384 data: 13 14 12 11 02/30 Read block 308 len 24576 03/30 Read block 418 len 12288 04/30 Write block 249 len 4096 data: 11 05/30 Read block 210 len 28672 06/30 Flush 07/30 Write block 207 len 20480 data: 8 10 12 12 11 08/30 Read block 350 len 16384 09/30 Write block 376 len 12288 data: 19 19 19 10/30 Flush 11/30 Write block 316 len 4096 data: 13 12/30 Flush 13/30 Write block 281 len 24576 data: 14 13 10 11 10 12 14/30 Flush 15/30 Write block 326 len 12288 data: 11 8 7 16/30 Write block 453 len 32768 data: 6 10 10 10 11 12 12 12 17/30 Read block 513 len 32768 18/30 Write block 64 len 28672 data: 10 10 11 10 13 12 12 19/30 Write block 373 len 36864 data: 17 18 20 20 20 20 20 17 14 20/30 Read block 159 len 4096 21/30 Write block 451 len 32768 data: 8 8 7 11 11 11 12 13 22/30 Read block 31 len 16384 23/30 Read block 113 len 12288 24/30 Write block 356 len 16384 data: 15 16 15 16 25/30 Read block 242 len 28672 26/30 Flush 27/30 Write block 444 len 12288 data: 8 6 7 28/30 Write block 144 len 4096 data: 10 29/30 Read block 294 len 16384 30/30 Read block 270 len 28672 ---------------------------------------------------------------- Crucible gen:164 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 3 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Read 4 Done Done Done false 1029 Acked Read 7 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 1 2 Done 3 4 3 10 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:18:13.478970225Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1838} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 162 162 162 293 293 293 F F F 1 020-039 164 164 164 297 297 297 F F F 2 040-059 160 160 160 292 292 292 F F F 3 060-079 164 164 164 301 301 301 F F F 4 080-099 159 159 159 291 291 291 F F F 5 100-119 158 158 158 290 290 290 F F F 6 120-139 160 160 160 292 292 292 F F F 7 140-159 162 162 162 295 295 295 T T T 8 160-179 152 152 152 278 278 278 F F F 9 180-199 160 160 160 292 292 292 F F F 10 200-219 164 164 164 298 298 298 F F F 11 220-239 163 163 163 296 296 296 F F F 12 240-259 164 164 164 297 297 297 F F F 13 260-279 163 163 163 296 296 296 F F F 14 280-299 164 164 164 300 300 300 F F F 15 300-319 164 164 164 299 299 299 F F F 16 320-339 164 164 164 301 301 301 F F F 17 340-359 164 164 164 301 301 301 F F F 18 360-379 164 164 164 301 301 301 F F F 19 380-399 164 164 164 301 301 301 F F F 20 400-419 162 162 162 293 293 293 F F F 21 420-439 162 162 162 293 293 293 F F F 22 440-459 164 164 164 301 301 301 T T T 23 460-479 164 164 164 301 301 301 F F F 24 480-499 159 159 159 291 291 291 F F F 25 500-519 162 162 162 294 294 294 F F F 26 520-539 160 160 160 292 292 292 F F F 27 540-559 163 163 163 296 296 296 F F F 28 560-579 163 163 163 296 296 296 F F F 29 580-599 159 159 159 291 291 291 F F F Max gen: 164, Max flush: 301 A Difference in extent metadata is expected here On loop 82 Verifying data now No change after verify Loop: 82 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 162 162 162 293 293 293 F F F 1 020-039 164 164 164 297 297 297 F F F 2 040-059 160 160 160 292 292 292 F F F 3 060-079 164 164 164 301 301 301 F F F 4 080-099 159 159 159 291 291 291 F F F 5 100-119 158 158 158 290 290 290 F F F 6 120-139 160 160 160 292 292 292 F F F 7 140-159 165 165 165 302 302 302 F F F 8 160-179 152 152 152 278 278 278 F F F 9 180-199 160 160 160 292 292 292 F F F 10 200-219 164 164 164 298 298 298 F F F 11 220-239 163 163 163 296 296 296 F F F 12 240-259 164 164 164 297 297 297 F F F 13 260-279 163 163 163 296 296 296 F F F 14 280-299 164 164 164 300 300 300 F F F 15 300-319 164 164 164 299 299 299 F F F 16 320-339 164 164 164 301 301 301 F F F 17 340-359 164 164 164 301 301 301 F F F 18 360-379 164 164 164 301 301 301 F F F 19 380-399 164 164 164 301 301 301 F F F 20 400-419 162 162 162 293 293 293 F F F 21 420-439 162 162 162 293 293 293 F F F 22 440-459 165 165 165 302 302 302 F F F 23 460-479 164 164 164 301 301 301 F F F 24 480-499 159 159 159 291 291 291 F F F 25 500-519 162 162 162 294 294 294 F F F 26 520-539 160 160 160 292 292 292 F F F 27 540-559 163 163 163 296 296 296 F F F 28 560-579 163 163 163 296 296 296 F F F 29 580-599 159 159 159 291 291 291 F F F Max gen: 165, Max flush: 302 Begin loop 83 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:17.703728224Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1846,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:17.747930899Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1846,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:17.748609846Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1846} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"abd5430f1508af0bc008143ea2566277ef714ec2\",\n git_commit_timestamp: \"2025-01-17T20:55:03.000000000Z\",\n git_branch: \"alan/tokio-to-140\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:17.74864999Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1846} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:17.748664558Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1846} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:17.748679337Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1846} {"msg":"Crucible e77a8567-a3dc-44c2-ad9b-ad2e905a5691 has session id: c431952b-6e03-4f4a-bc37-c9f2387e3884","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:17.749221044Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1846,"session_id":"c431952b-6e03-4f4a-bc37-c9f2387e3884"} {"msg":"Upstairs opts: Upstairs UUID: e77a8567-a3dc-44c2-ad9b-ad2e905a5691, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:17.749264329Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1846,"session_id":"c431952b-6e03-4f4a-bc37-c9f2387e3884"} {"msg":"Crucible stats registered with UUID: e77a8567-a3dc-44c2-ad9b-ad2e905a5691","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:17.749288543Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1846,"session_id":"c431952b-6e03-4f4a-bc37-c9f2387e3884"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:18:17.749308985Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1846,"session_id":"c431952b-6e03-4f4a-bc37-c9f2387e3884"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:17.749702405Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1846,"task":"crutest"} {"msg":"The guest has requested activation with gen:166","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:17.749726279Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1846} {"msg":"e77a8567-a3dc-44c2-ad9b-ad2e905a5691 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:17.74981324Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1846,"session_id":"c431952b-6e03-4f4a-bc37-c9f2387e3884"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:17.74983036Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1846,"client":"0","":"downstairs","session_id":"c431952b-6e03-4f4a-bc37-c9f2387e3884"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:17.749853263Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1846,"client":"0","":"downstairs","session_id":"c431952b-6e03-4f4a-bc37-c9f2387e3884"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:17.749866951Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1846,"client":"1","":"downstairs","session_id":"c431952b-6e03-4f4a-bc37-c9f2387e3884"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:17.749880839Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1846,"client":"1","":"downstairs","session_id":"c431952b-6e03-4f4a-bc37-c9f2387e3884"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:17.749893637Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1846,"client":"2","":"downstairs","session_id":"c431952b-6e03-4f4a-bc37-c9f2387e3884"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:17.749907235Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1846,"client":"2","":"downstairs","session_id":"c431952b-6e03-4f4a-bc37-c9f2387e3884"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:17.749962537Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1846,"":"io task","client":"2","":"downstairs","session_id":"c431952b-6e03-4f4a-bc37-c9f2387e3884"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:17.750047927Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1846,"":"io task","client":"0","":"downstairs","session_id":"c431952b-6e03-4f4a-bc37-c9f2387e3884"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:17.750083348Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1846,"":"io task","client":"1","":"downstairs","session_id":"c431952b-6e03-4f4a-bc37-c9f2387e3884"} {"msg":"ds_connection connected from Ok(127.0.0.1:32993)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:17.750114566Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1846,"":"io task","client":"0","":"downstairs","session_id":"c431952b-6e03-4f4a-bc37-c9f2387e3884"} {"msg":"ds_connection connected from Ok(127.0.0.1:46298)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:17.750358468Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1846,"":"io task","client":"1","":"downstairs","session_id":"c431952b-6e03-4f4a-bc37-c9f2387e3884"} {"msg":"ds_connection connected from Ok(127.0.0.1:58493)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:17.750397391Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1846,"":"io task","client":"2","":"downstairs","session_id":"c431952b-6e03-4f4a-bc37-c9f2387e3884"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:17.750726353Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1846,"client":"0","":"downstairs","session_id":"c431952b-6e03-4f4a-bc37-c9f2387e3884"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:17.750909921Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1846,"client":"1","":"downstairs","session_id":"c431952b-6e03-4f4a-bc37-c9f2387e3884"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:17.75138783Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1846,"client":"0","":"downstairs","session_id":"c431952b-6e03-4f4a-bc37-c9f2387e3884"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:17.751552156Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1846,"client":"1","":"downstairs","session_id":"c431952b-6e03-4f4a-bc37-c9f2387e3884"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:17.751739265Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1846,"client":"2","":"downstairs","session_id":"c431952b-6e03-4f4a-bc37-c9f2387e3884"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:17.751925464Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1846,"session_id":"c431952b-6e03-4f4a-bc37-c9f2387e3884"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:17.752076743Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1846,"session_id":"c431952b-6e03-4f4a-bc37-c9f2387e3884"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:17.752341998Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1846,"client":"2","":"downstairs","session_id":"c431952b-6e03-4f4a-bc37-c9f2387e3884"} {"msg":"[0]R flush_numbers[0..12]: [293, 297, 292, 301, 291, 290, 292, 302, 278, 292, 298, 296]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:17.752834496Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1846,"":"downstairs","":"downstairs","session_id":"c431952b-6e03-4f4a-bc37-c9f2387e3884"} {"msg":"[0]R generation[0..12]: [162, 164, 160, 164, 159, 158, 160, 165, 152, 160, 164, 163]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:17.752854097Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1846,"":"downstairs","":"downstairs","session_id":"c431952b-6e03-4f4a-bc37-c9f2387e3884"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:17.752868205Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1846,"":"downstairs","":"downstairs","session_id":"c431952b-6e03-4f4a-bc37-c9f2387e3884"} {"msg":"[1]R flush_numbers[0..12]: [293, 297, 292, 301, 291, 290, 292, 302, 278, 292, 298, 296]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:17.752885435Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1846,"":"downstairs","":"downstairs","session_id":"c431952b-6e03-4f4a-bc37-c9f2387e3884"} {"msg":"[1]R generation[0..12]: [162, 164, 160, 164, 159, 158, 160, 165, 152, 160, 164, 163]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:17.752899334Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1846,"":"downstairs","":"downstairs","session_id":"c431952b-6e03-4f4a-bc37-c9f2387e3884"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:17.752913352Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1846,"":"downstairs","":"downstairs","session_id":"c431952b-6e03-4f4a-bc37-c9f2387e3884"} {"msg":"[2]R flush_numbers[0..12]: [293, 297, 292, 301, 291, 290, 292, 302, 278, 292, 298, 296]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:17.75292704Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1846,"":"downstairs","":"downstairs","session_id":"c431952b-6e03-4f4a-bc37-c9f2387e3884"} {"msg":"[2]R generation[0..12]: [162, 164, 160, 164, 159, 158, 160, 165, 152, 160, 164, 163]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:17.752941498Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1846,"":"downstairs","":"downstairs","session_id":"c431952b-6e03-4f4a-bc37-c9f2387e3884"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:17.752955226Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1846,"":"downstairs","":"downstairs","session_id":"c431952b-6e03-4f4a-bc37-c9f2387e3884"} {"msg":"Max found gen is 166","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:17.752967863Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1846,"":"downstairs","":"downstairs","session_id":"c431952b-6e03-4f4a-bc37-c9f2387e3884"} {"msg":"Generation requested: 166 >= found:166","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:17.752980601Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1846,"":"downstairs","":"downstairs","session_id":"c431952b-6e03-4f4a-bc37-c9f2387e3884"} {"msg":"Next flush: 303","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:17.752993168Z","hostname":"Wait for a query_work_queue command to finish before sending IO w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1846,"":"downstairs","":"downstairs","session_id":"c431952b-6e03-4f4a-bc37-c9f2387e3884"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:17.753012129Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1846,"":"downstairs","":"downstairs","session_id":"c431952b-6e03-4f4a-bc37-c9f2387e3884"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:17.753024977Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1846,"session_id":"c431952b-6e03-4f4a-bc37-c9f2387e3884"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:17.753036644Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1846,"client":"0","":"downstairs","session_id":"c431952b-6e03-4f4a-bc37-c9f2387e3884"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:17.753049241Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1846,"client":"1","":"downstairs","session_id":"c431952b-6e03-4f4a-bc37-c9f2387e3884"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:17.753061778Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1846,"client":"2","":"downstairs","session_id":"c431952b-6e03-4f4a-bc37-c9f2387e3884"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:17.753076967Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1846,"session_id":"c431952b-6e03-4f4a-bc37-c9f2387e3884"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:17.753088774Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1846,"session_id":"c431952b-6e03-4f4a-bc37-c9f2387e3884"} {"msg":"e77a8567-a3dc-44c2-ad9b-ad2e905a5691 is now active with session: c431952b-6e03-4f4a-bc37-c9f2387e3884","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:17.753101161Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1846,"session_id":"c431952b-6e03-4f4a-bc37-c9f2387e3884"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:17.753112528Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1846,"session_id":"c431952b-6e03-4f4a-bc37-c9f2387e3884"} {"msg":"The guest has finished waiting for activation with:166","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:17.753124255Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1846} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 4 len 36864 data: 12 13 12 13 13 12 14 15 13 02/30 Read block 406 len 12288 03/30 Read block 1 len 4096 04/30 Write block 559 len 40960 data: 13 17 18 18 19 12 11 15 10 12 05/30 Read block 491 len 12288 06/30 Read block 9 len 32768 07/30 Write block 569 len 40960 data: 13 14 13 11 12 13 11 14 14 12 08/30 Read block 41 len 40960 09/30 Write block 269 len 8192 data: 12 13 10/30 Read block 129 len 36864 11/30 Write block 375 len 24576 data: 21 21 21 21 21 18 12/30 Write block 68 len 20480 data: 14 13 13 15 14 13/30 Read block 308 len 32768 14/30 Read block 328 len 36864 15/30 Write block 526 len 40960 data: 15 13 11 9 9 12 13 13 12 11 16/30 Read block 519 len 36864 17/30 Write block 404 len 8192 data: 9 9 18/30 Write block 20 len 12288 data: 8 10 10 19/30 Read block 50 len 4096 20/30 Read block 389 len 32768 21/30 Read block 419 len 12288 22/30 Flush 23/30 Write block 178 len 4096 data: 12 24/30 Write block 337 len 20480 data: 16 17 18 17 17 25/30 Write block 5 len 36864 data: 14 13 14 14 13 15 16 14 15 26/30 Read block 332 len 20480 27/30 Write block 246 len 24576 data: 10 12 11 12 10 9 28/30 Write block 141 len 16384 data: 10 13 12 11 29/30 Write block 250 len 36864 data: 11 10 10 9 11 13 12 12 11 30/30 Write block 223 len 24576 data: 11 9 10 13 14 15 ---------------------------------------------------------------- Crucible gen:166 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 1 Done Done Done false 1023 Acked Write 5 Done Done Done false 1024 Acked Write 9 Done Done Done false 1025 Acked Read 5 Done Done Done false 1026 Acked Write 6 Done Done Done false 1027 Acked Write 4 Done Done Done false 1028 Acked Write 9 Sent Sent Sent false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 6 6 6 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1021 1021 1021 Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:18:17.773453236Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1846} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 166 166 166 303 303 303 T T T 1 020-039 166 166 166 303 303 303 F F F 2 040-059 160 160 160 292 292 292 F F F 3 060-079 166 166 166 303 303 303 F F F 4 080-099 159 159 159 291 291 291 F F F 5 100-119 158 158 158 290 290 290 F F F 6 120-139 160 160 160 292 292 292 F F F 7 140-159 165 165 165 302 302 302 T T T 8 160-179 152 152 152 278 278 278 T T T 9 180-199 160 160 160 292 292 292 F F F 10 200-219 164 164 164 298 298 298 F F F 11 220-239 163 163 163 296 296 296 T T T 12 240-259 164 164 164 297 297 297 T T T 13 260-279 166 166 166 303 303 303 F F F 14 280-299 164 164 164 300 300 300 F F F 15 300-319 164 164 164 299 299 299 F F F 16 320-339 164 164 164 301 301 301 T T T 17 340-359 164 164 164 301 301 301 T T T 18 360-379 166 166 166 303 303 303 F F F 19 380-399 166 166 166 303 303 303 F F F 20 400-419 166 166 166 303 303 303 F F F 21 420-439 162 162 162 293 293 293 F F F 22 440-459 165 165 165 302 302 302 F F F 23 460-479 164 164 164 301 301 301 F F F 24 480-499 159 159 159 291 291 291 F F F 25 500-519 162 162 162 294 294 294 F F F 26 520-539 166 166 166 303 303 303 F F F 27 540-559 166 166 166 303 303 303 F F F 28 560-579 166 166 166 303 303 303 F F F 29 580-599 159 159 159 291 291 291 F F F Max gen: 166, Max flush: 303 A Difference in extent metadata is expected here On loop 83 Verifying data now No change after verify Loop: 83 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 167 167 167 304 304 304 F F F 1 020-039 166 166 166 303 303 303 F F F 2 040-059 160 160 160 292 292 292 F F F 3 060-079 166 166 166 303 303 303 F F F 4 080-099 159 159 159 291 291 291 F F F 5 100-119 158 158 158 290 290 290 F F F 6 120-139 160 160 160 292 292 292 F F F 7 140-159 167 167 167 304 304 304 F F F 8 160-179 167 167 167 304 304 304 F F F 9 180-199 160 160 160 292 292 292 F F F 10 200-219 164 164 164 298 298 298 F F F 11 220-239 167 167 167 304 304 304 F F F 12 240-259 167 167 167 304 304 304 F F F 13 260-279 166 166 166 303 303 303 F F F 14 280-299 164 164 164 300 300 300 F F F 15 300-319 164 164 164 299 299 299 F F F 16 320-339 167 167 167 304 304 304 F F F 17 340-359 167 167 167 304 304 304 F F F 18 360-379 166 166 166 303 303 303 F F F 19 380-399 166 166 166 303 303 303 F F F 20 400-419 166 166 166 303 303 303 F F F 21 420-439 162 162 162 293 293 293 F F F 22 440-459 165 165 165 302 302 302 F F F 23 460-479 164 164 164 301 301 301 F F F 24 480-499 159 159 159 291 291 291 F F F 25 500-519 162 162 162 294 294 294 F F F 26 520-539 166 166 166 303 303 303 F F F 27 540-559 166 166 166 303 303 303 F F F 28 560-579 166 166 166 303 303 303 F F F 29 580-599 159 159 159 291 291 291 F F F Max gen: 167, Max flush: 304 Begin loop 84 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:22.02644686Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1854,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:22.071452104Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1854,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:22.072076729Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1854} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"abd5430f1508af0bc008143ea2566277ef714ec2\",\n git_commit_timestamp: \"2025-01-17T20:55:03.000000000Z\",\n git_branch: \"alan/tokio-to-140\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:22.072117193Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1854} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:22.072131131Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1854} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:22.072145339Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1854} {"msg":"Crucible e2635ab3-a164-4684-91cb-d56d68158e96 has session id: 4483e527-52ab-4cee-9c77-694c157771f4","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:22.072735934Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1854,"session_id":"4483e527-52ab-4cee-9c77-694c157771f4"} {"msg":"Upstairs opts: Upstairs UUID: e2635ab3-a164-4684-91cb-d56d68158e96, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:22.072772345Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1854,"session_id":"4483e527-52ab-4cee-9c77-694c157771f4"} {"msg":"Crucible stats registered with UUID: e2635ab3-a164-4684-91cb-d56d68158e96","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:22.072787975Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1854,"session_id":"4483e527-52ab-4cee-9c77-694c157771f4"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:18:22.072807326Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1854,"session_id":"4483e527-52ab-4cee-9c77-694c157771f4"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:22.073329511Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1854,"task":"crutest"} {"msg":"The guest has requested activation with gen:168","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:22.073353185Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1854} {"msg":"e2635ab3-a164-4684-91cb-d56d68158e96 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:22.073415351Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1854,"session_id":"4483e527-52ab-4cee-9c77-694c157771f4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:22.073432031Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1854,"client":"0","":"downstairs","session_id":"4483e527-52ab-4cee-9c77-694c157771f4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:22.073455975Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1854,"client":"0","":"downstairs","session_id":"4483e527-52ab-4cee-9c77-694c157771f4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:22.073469553Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1854,"client":"1","":"downstairs","session_id":"4483e527-52ab-4cee-9c77-694c157771f4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:22.073487213Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1854,"client":"1","":"downstairs","session_id":"4483e527-52ab-4cee-9c77-694c157771f4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:22.073500791Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1854,"client":"2","":"downstairs","session_id":"4483e527-52ab-4cee-9c77-694c157771f4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:22.073514719Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1854,"client":"2","":"downstairs","session_id":"4483e527-52ab-4cee-9c77-694c157771f4"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:22.073527407Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1854,"":"io task","client":"2","":"downstairs","session_id":"4483e527-52ab-4cee-9c77-694c157771f4"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:22.073540645Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1854,"":"io task","client":"0","":"downstairs","session_id":"4483e527-52ab-4cee-9c77-694c157771f4"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:22.073553922Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1854,"":"io task","client":"1","":"downstairs","session_id":"4483e527-52ab-4cee-9c77-694c157771f4"} {"msg":"ds_connection connected from Ok(127.0.0.1:44722)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:22.073716708Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1854,"":"io task","client":"2","":"downstairs","session_id":"4483e527-52ab-4cee-9c77-694c157771f4"} {"msg":"ds_connection connected from Ok(127.0.0.1:36660)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:22.07375539Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1854,"":"io task","client":"1","":"downstairs","session_id":"4483e527-52ab-4cee-9c77-694c157771f4"} {"msg":"ds_connection connected from Ok(127.0.0.1:58081)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:22.073837868Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1854,"":"io task","client":"0","":"downstairs","session_id":"4483e527-52ab-4cee-9c77-694c157771f4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:22.074417177Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1854,"client":"0","":"downstairs","session_id":"4483e527-52ab-4cee-9c77-694c157771f4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:22.074447004Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1854,"client":"2","":"downstairs","session_id":"4483e527-52ab-4cee-9c77-694c157771f4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:22.074532745Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1854,"client":"1","":"downstairs","session_id":"4483e527-52ab-4cee-9c77-694c157771f4"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:22.0750732Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1854,"client":"0","":"downstairs","session_id":"4483e527-52ab-4cee-9c77-694c157771f4"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:22.075268064Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1854,"client":"1","":"downstairs","session_id":"4483e527-52ab-4cee-9c77-694c157771f4"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:22.075298082Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1854,"client":"2","":"downstairs","session_id":"4483e527-52ab-4cee-9c77-694c157771f4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:22.07560302Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1854,"session_id":"4483e527-52ab-4cee-9c77-694c157771f4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:22.075823819Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1854,"session_id":"4483e527-52ab-4cee-9c77-694c157771f4"} {"msg":"[0]R flush_numbers[0..12]: [304, 303, 292, 303, 291, 290, 292, 304, 304, 292, 298, 304]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:22.07594602Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1854,"":"downstairs","":"downstairs","session_id":"4483e527-52ab-4cee-9c77-694c157771f4"} {"msg":"[0]R generation[0..12]: [167, 166, 160, 166, 159, 158, 160, 167, 167, 160, 164, 167]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:22.075966152Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1854,"":"downstairs","":"downstairs","session_id":"4483e527-52ab-4cee-9c77-694c157771f4"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:22.075980661Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1854,"":"downstairs","":"downstairs","session_id":"4483e527-52ab-4cee-9c77-694c157771f4"} {"msg":"[1]R flush_numbers[0..12]: [304, 303, 292, 303, 291, 290, 292, 304, 304, 292, 298, 304]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:22.075994269Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1854,"":"downstairs","":"downstairs","session_id":"4483e527-52ab-4cee-9c77-694c157771f4"} {"msg":"[1]R generation[0..12]: [167, 166, 160, 166, 159, 158, 160, 167, 167, 160, 164, 167]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:22.076007747Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1854,"":"downstairs","":"downstairs","session_id":"4483e527-52ab-4cee-9c77-694c157771f4"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:22.076021344Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1854,"":"downstairs","":"downstairs","session_id":"4483e527-52ab-4cee-9c77-694c157771f4"} {"msg":"[2]R flush_numbers[0..12]: [304, 303, 292, 303, 291, 290, 292, 304, 304, 292, 298, 304]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:22.076034772Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1854,"":"downstairs","":"downstairs","session_id":"4483e527-52ab-4cee-9c77-694c157771f4"} {"msg":"[2]R generation[0..12]: [167, 166, 160, 166, 159, 158, 160, 167, 167, 160, 164, 167]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:22.07604797Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1854,"":"downstairs","":"downstairs","session_id":"4483e527-52ab-4cee-9c77-694c157771f4"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:22.076061578Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1854,"":"downstairs","":"downstairs","session_id":"4483e527-52ab-4cee-9c77-694c157771f4"} {"msg":"Max found gen is 168","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:22.076074205Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1854,"":"downstairs","":"downstairs","session_id":"4483e527-52ab-4cee-9c77-694c157771f4"} {"msg":"Generation requested: 168 >= found:168","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:22.076087063Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1854,"":"downstairs","":"downstairs","session_id":"4483e527-52ab-4cee-9c77-694c157771f4"} {"msg":"Next flush: 305","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:22.076102152Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1854,"":"downstairs","":"downstairs","session_id":"4483e527-52ab-4cee-9c77-694c157771f4"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:22.076114919Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1854,"":"downstairs","":"downstairs","session_id":"4483e527-52ab-4cee-9c77-694c157771f4"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-17T21:18:22.076127586Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1854,"session_id":"4483e527-52ab-4cee-9c77-694c157771f4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:22.076142365Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1854,"client":"0","":"downstairs","session_id":"4483e527-52ab-4cee-9c77-694c157771f4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:22.076155293Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1854,"client":"1","":"downstairs","session_id":"4483e527-52ab-4cee-9c77-694c157771f4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:22.07616799Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1854,"client":"2","":"downstairs","session_id":"4483e527-52ab-4cee-9c77-694c157771f4"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:22.076180758Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1854,"session_id":"4483e527-52ab-4cee-9c77-694c157771f4"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:22.076192875Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1854,"session_id":"4483e527-52ab-4cee-9c77-694c157771f4"} {"msg":"e2635ab3-a164-4684-91cb-d56d68158e96 is now active with session: 4483e527-52ab-4cee-9c77-694c157771f4","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:22.076205502Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1854,"session_id":"4483e527-52ab-4cee-9c77-694c157771f4"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:22.076217049Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1854,"session_id":"4483e527-52ab-4cee-9c77-694c157771f4"} {"msg":"The guest has finished waiting for activation with:168","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:22.076228916Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1854} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 455 len 12288 data: 12 12 13 02/30 Read block 197 len 12288 03/30 Read block 8 len 28672 04/30 Read block 395 len 4096 05/30 Read block 444 len 36864 06/30 Read block 14 len 12288 07/30 Read block 184 len 24576 08/30 Read block 511 len 8192 09/30 Write block 508 len 4096 data: 17 10/30 Read block 103 len 28672 11/30 Write block 233 len 24576 data: 14 16 18 15 16 12 12/30 Write block 302 len 20480 data: 9 13 12 14 12 13/30 Flush 14/30 Write block 72 len 8192 data: 15 13 15/30 Write block 569 len 4096 data: 14 16/30 Write block 329 len 40960 data: 10 9 10 10 10 13 14 13 17 18 17/30 Write block 442 len 4096 data: 6 18/30 Write block 77 len 20480 data: 12 12 12 12 12 19/30 Write block 504 len 20480 data: 15 17 15 17 18 20/30 Read block 494 len 32768 21/30 Write block 350 len 36864 data: 13 10 12 11 12 13 16 17 16 22/30 Write block 257 len 4096 data: 13 23/30 Write block 364 len 28672 data: 10 8 7 6 6 6 11 24/30 Read block 359 len 20480 25/30 Read block 400 len 40960 26/30 Write block 147 len 28672 data: 13 12 13 12 12 11 11 27/30 Write block 304 len 12288 data: 13 15 13 28/30 Read block 517 len 12288 29/30 Read block 336 len 4096 30/30 Read block 57 len 8192 ---------------------------------------------------------------- Crucible gen:168 GIO:true work queues: Upstairs:0 downstairs:17 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1013 Acked Write 2 Done Done Done false 1014 Acked Write 1 Done Done Done false 1015 Acked Write 10 Done Done Done false 1016 Acked Write 1 Done Done Done false 1017 Acked Write 5 Done Done Done false 1018 Acked Write 5 Done Done Done false 1019 Acked Read 8 Done Done Done false 1020 Acked Write 9 Done Done Done false 1021 Acked Write 1 Done Done Done false 1022 Acked Write 7 Done Done Done false 1023 Acked Read 5 Done Done Done false 1024 Acked Read 10 Done Done Done false 1025 Acked Write 7 Done Done Done false 1026 Acked Write 3 Done Done Done false 1027 Acked Read 3 Done Done Done false 1028 Acked Read 1 Done Done Done false 1029 Acked Read 2 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 17 17 17 51 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1012 1012 1012 Downstairs last five completed: 1012 1011 1010 1009 1008 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:18:22.094273542Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1854} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 167 167 167 304 304 304 F F F 1 020-039 166 166 166 303 303 303 F F F 2 040-059 160 160 160 292 292 292 F F F 3 060-079 166 166 166 303 303 303 T T T 4 080-099 159 159 159 291 291 291 T T T 5 100-119 158 158 158 290 290 290 F F F 6 120-139 160 160 160 292 292 292 F F F 7 140-159 167 167 167 304 304 304 T T T 8 160-179 167 167 167 304 304 304 F F F 9 180-199 160 160 160 292 292 292 F F F 10 200-219 164 164 164 298 298 298 F F F 11 220-239 168 168 168 305 305 305 F F F 12 240-259 167 167 167 304 304 304 T T T 13 260-279 166 166 166 303 303 303 F F F 14 280-299 164 164 164 300 300 300 F F F 15 300-319 168 168 168 305 305 305 T T T 16 320-339 167 167 167 304 304 304 T T T 17 340-359 167 167 167 304 304 304 T T T 18 360-379 166 166 166 303 303 303 T T T 19 380-399 166 166 166 303 303 303 F F F 20 400-419 166 166 166 303 303 303 F F F 21 420-439 162 162 162 293 293 293 F F F 22 440-459 168 168 168 305 305 305 T T T 23 460-479 164 164 164 301 301 301 F F F 24 480-499 159 159 159 291 291 291 F F F 25 500-519 168 168 168 305 305 305 T T T 26 520-539 166 166 166 303 303 303 F F F 27 540-559 166 166 166 303 303 303 F F F 28 560-579 166 166 166 303 303 303 T T T 29 580-599 159 159 159 291 291 291 F F F Max gen: 168, Max flush: 305 A Difference in extent metadata is expected here On loop 84 Verifying data now No change after verify Loop: 84 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 167 167 167 304 304 304 F F F 1 020-039 166 166 166 303 303 303 F F F 2 040-059 160 160 160 292 292 292 F F F 3 060-079 169 169 169 306 306 306 F F F 4 080-099 169 169 169 306 306 306 F F F 5 100-119 158 158 158 290 290 290 F F F 6 120-139 160 160 160 292 292 292 F F F 7 140-159 169 169 169 306 306 306 F F F 8 160-179 167 167 167 304 304 304 F F F 9 180-199 160 160 160 292 292 292 F F F 10 200-219 164 164 164 298 298 298 F F F 11 220-239 168 168 168 305 305 305 F F F 12 240-259 169 169 169 306 306 306 F F F 13 260-279 166 166 166 303 303 303 F F F 14 280-299 164 164 164 300 300 300 F F F 15 300-319 169 169 169 306 306 306 F F F 16 320-339 169 169 169 306 306 306 F F F 17 340-359 169 169 169 306 306 306 F F F 18 360-379 169 169 169 306 306 306 F F F 19 380-399 166 166 166 303 303 303 F F F 20 400-419 166 166 166 303 303 303 F F F 21 420-439 162 162 162 293 293 293 F F F 22 440-459 169 169 169 306 306 306 F F F 23 460-479 164 164 164 301 301 301 F F F 24 480-499 159 159 159 291 291 291 F F F 25 500-519 169 169 169 306 306 306 F F F 26 520-539 166 166 166 303 303 303 F F F 27 540-559 166 166 166 303 303 303 F F F 28 560-579 169 169 169 306 306 306 F F F 29 580-599 159 159 159 291 291 291 F F F Max gen: 169, Max flush: 306 Begin loop 85 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:26.372109604Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1862,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:26.416612165Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1862,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:26.417342322Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1862} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"abd5430f1508af0bc008143ea2566277ef714ec2\",\n git_commit_timestamp: \"2025-01-17T20:55:03.000000000Z\",\n git_branch: \"alan/tokio-to-140\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:26.417381875Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1862} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:26.417396263Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1862} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:26.417410241Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1862} {"msg":"Crucible 25b1b99b-fa42-4ab8-ac40-4e4dbdda7582 has session id: 3ad02af7-01ca-4832-8cdf-50887f766252","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:26.41800733Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1862,"session_id":"3ad02af7-01ca-4832-8cdf-50887f766252"} {"msg":"Upstairs opts: Upstairs UUID: 25b1b99b-fa42-4ab8-ac40-4e4dbdda7582, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:26.418046343Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1862,"session_id":"3ad02af7-01ca-4832-8cdf-50887f766252"} {"msg":"Crucible stats registered with UUID: 25b1b99b-fa42-4ab8-ac40-4e4dbdda7582","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:26.418072058Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1862,"session_id":"3ad02af7-01ca-4832-8cdf-50887f766252"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:18:26.418091399Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1862,"session_id":"3ad02af7-01ca-4832-8cdf-50887f766252"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:26.418500859Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1862,"task":"crutest"} {"msg":"The guest has requested activation with gen:170","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:26.418528315Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1862} {"msg":"25b1b99b-fa42-4ab8-ac40-4e4dbdda7582 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:26.418591042Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1862,"session_id":"3ad02af7-01ca-4832-8cdf-50887f766252"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:26.418608492Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1862,"client":"0","":"downstairs","session_id":"3ad02af7-01ca-4832-8cdf-50887f766252"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:26.418633336Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1862,"client":"0","":"downstairs","session_id":"3ad02af7-01ca-4832-8cdf-50887f766252"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:26.418647905Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1862,"client":"1","":"downstairs","session_id":"3ad02af7-01ca-4832-8cdf-50887f766252"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:26.418661723Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1862,"client":"1","":"downstairs","session_id":"3ad02af7-01ca-4832-8cdf-50887f766252"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:26.4186745Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1862,"client":"2","":"downstairs","session_id":"3ad02af7-01ca-4832-8cdf-50887f766252"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:26.418688529Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1862,"client":"2","":"downstairs","session_id":"3ad02af7-01ca-4832-8cdf-50887f766252"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:26.418701476Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1862,"":"io task","client":"2","":"downstairs","session_id":"3ad02af7-01ca-4832-8cdf-50887f766252"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:26.418722008Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1862,"":"io task","client":"0","":"downstairs","session_id":"3ad02af7-01ca-4832-8cdf-50887f766252"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:26.418747523Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1862,"":"io task","client":"1","":"downstairs","session_id":"3ad02af7-01ca-4832-8cdf-50887f766252"} {"msg":"ds_connection connected from Ok(127.0.0.1:33988)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:26.418820005Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1862,"":"io task","client":"2","":"downstairs","session_id":"3ad02af7-01ca-4832-8cdf-50887f766252"} {"msg":"ds_connection connected from Ok(127.0.0.1:46484)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:26.418858028Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1862,"":"io task","client":"0","":"downstairs","session_id":"3ad02af7-01ca-4832-8cdf-50887f766252"} {"msg":"ds_connection connected from Ok(127.0.0.1:57701)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:26.418890637Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1862,"":"io task","client":"1","":"downstairs","session_id":"3ad02af7-01ca-4832-8cdf-50887f766252"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:26.41947806Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1862,"client":"0","":"downstairs","session_id":"3ad02af7-01ca-4832-8cdf-50887f766252"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:26.419608977Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1862,"client":"1","":"downstairs","session_id":"3ad02af7-01ca-4832-8cdf-50887f766252"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:26.419983416Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1862,"client":"0","":"downstairs","session_id":"3ad02af7-01ca-4832-8cdf-50887f766252"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:26.420103095Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1862,"client":"1","":"downstairs","session_id":"3ad02af7-01ca-4832-8cdf-50887f766252"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:26.420458903Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1862,"session_id":"3ad02af7-01ca-4832-8cdf-50887f766252"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:26.420690008Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1862,"session_id":"3ad02af7-01ca-4832-8cdf-50887f766252"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:26.421423847Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1862,"client":"2","":"downstairs","session_id":"3ad02af7-01ca-4832-8cdf-50887f766252"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:26.421912773Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1862,"client":"2","":"downstairs","session_id":"3ad02af7-01ca-4832-8cdf-50887f766252"} {"msg":"[0]R flush_numbers[0..12]: [304, 303, 292, 306, 306, 290, 292, 306, 304, 292, 298, 305]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:26.422385379Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1862,"":"downstairs","":"downstairs","session_id":"3ad02af7-01ca-4832-8cdf-50887f766252"} {"msg":"[0]R generation[0..12]: [167, 166, 160, 169, 169, 158, 160, 169, 167, 160, 164, 168]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:26.422404951Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1862,"":"downstairs","":"downstairs","session_id":"3ad02af7-01ca-4832-8cdf-50887f766252"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:26.422419019Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1862,"":"downstairs","":"downstairs","session_id":"3ad02af7-01ca-4832-8cdf-50887f766252"} {"msg":"[1]R flush_numbers[0..12]: [304, 303, 292, 306, 306, 290, 292, 306, 304, 292, 298, 305]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:26.422432387Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1862,"":"downstairs","":"downstairs","session_id":"3ad02af7-01ca-4832-8cdf-50887f766252"} {"msg":"[1]R generation[0..12]: [167, 166, 160, 169, 169, 158, 160, 169, 167, 160, 164, 168]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:26.422448816Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1862,"":"downstairs","":"downstairs","session_id":"3ad02af7-01ca-4832-8cdf-50887f766252"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:26.422462554Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1862,"":"downstairs","":"downstairs","session_id":"3ad02af7-01ca-4832-8cdf-50887f766252"} {"msg":"[2]R flush_numbers[0..12]: [304, 303, 292, 306, 306, 290, 292, 306, 304, 292, 298, 305]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:26.422476002Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1862,"":"downstairs","":"downstairs","session_id":"3ad02af7-01ca-4832-8cdf-50887f766252"} {"msg":"[2]R generation[0..12]: [167, 166, 160, 169, 169, 158, 160, 169, 167, 160, 164, 168]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:26.42248955Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1862,"":"downstairs","":"downstairs","session_id":"3ad02af7-01ca-4832-8cdf-50887f766252"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:26.422504259Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1862,"":"downstairs","":"downstairs","session_id":"3ad02af7-01ca-4832-8cdf-50887f766252"} {"msg":"Max found gen is 170","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:26.422516946Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1862,"":"downstairs","":"downstairs","session_id":"3ad02af7-01ca-4832-8cdf-50887f766252"} {"msg":"Generation requested: 170 >= found:170","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:26.422529844Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1862,"":"downstairs","":"downstairs","session_id":"3ad02af7-01ca-4832-8cdf-50887f766252"} {"msg":"Next flush: 307","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:26.422542321Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1862,"":"downstairs","":"downstairs","session_id":"3ad02af7-01ca-4832-8cdf-50887f766252"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:26.422554868Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1862,"":"downstairs","":"downstairs","session_id":"3ad02af7-01ca-4832-8cdf-50887f766252"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:26.422567596Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1862,"session_id":"Wait for a query_work_queue command to finish before sending IO 3ad02af7-01ca-4832-8cdf-50887f766252"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:26.422587207Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1862,"client":"0","":"downstairs","session_id":"3ad02af7-01ca-4832-8cdf-50887f766252"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:26.422600095Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1862,"client":"1","":"downstairs","session_id":"3ad02af7-01ca-4832-8cdf-50887f766252"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:26.422612642Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1862,"client":"2","":"downstairs","session_id":"3ad02af7-01ca-4832-8cdf-50887f766252"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:26.422625329Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1862,"session_id":"3ad02af7-01ca-4832-8cdf-50887f766252"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:26.422637107Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1862,"session_id":"3ad02af7-01ca-4832-8cdf-50887f766252"} {"msg":"25b1b99b-fa42-4ab8-ac40-4e4dbdda7582 is now active with session: 3ad02af7-01ca-4832-8cdf-50887f766252","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:26.422651525Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1862,"session_id":"3ad02af7-01ca-4832-8cdf-50887f766252"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:26.422663212Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1862,"session_id":"3ad02af7-01ca-4832-8cdf-50887f766252"} {"msg":"The guest has finished waiting for activation with:170","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:26.422675069Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1862} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 566 len 36864 data: 16 11 13 15 15 14 12 13 14 03/30 Read block 63 len 28672 04/30 Write block 593 len 20480 data: 9 9 6 7 6 05/30 Write block 180 len 40960 data: 11 11 11 8 6 5 4 6 10 12 06/30 Read block 221 len 12288 07/30 Read block 249 len 8192 08/30 Read block 385 len 12288 09/30 Read block 39 len 24576 10/30 Read block 345 len 12288 11/30 Write block 135 len 8192 data: 6 9 12/30 Write block 150 len 12288 data: 13 13 12 13/30 Read block 198 len 16384 14/30 Read block 17 len 8192 15/30 Read block 426 len 20480 16/30 Write block 487 len 8192 data: 10 11 17/30 Read block 18 len 4096 18/30 Read block 318 len 36864 19/30 Read block 141 len 12288 20/30 Write block 159 len 8192 data: 13 13 21/30 Write block 259 len 12288 data: 14 12 11 22/30 Flush 23/30 Write block 137 len 16384 data: 9 9 12 11 24/30 Read block 335 len 8192 25/30 Read block 462 len 32768 26/30 Read block 123 len 12288 27/30 Write block 128 len 24576 data: 10 12 13 11 10 7 28/30 Write block 31 len 28672 data: 12 16 18 18 21 15 15 29/30 Write block 225 len 28672 data: 11 14 15 16 16 17 14 30/30 Write block 42 len 12288 data: 11 9 10 ---------------------------------------------------------------- Crucible gen:170 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 4 Done Done Done false 1023 Acked Read 2 Done Done Done false 1024 Acked Read 8 Done Done Done false 1025 Acked Read 3 Done Done Done false 1026 Acked Write 6 Done Done Done false 1027 Acked Write 7 Done Done Done false 1028 Acked Write 7 Sent Sent Sent false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 6 6 6 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1021 1021 1021 Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:18:26.441653241Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1862} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 167 167 167 304 304 304 F F F 1 020-039 166 166 166 303 303 303 T T T 2 040-059 160 160 160 292 292 292 T T T 3 060-079 169 169 169 306 306 306 F F F 4 080-099 169 169 169 306 306 306 F F F 5 100-119 158 158 158 290 290 290 F F F 6 120-139 170 170 170 308 308 308 T T T 7 140-159 170 170 170 308 308 308 T T T 8 160-179 170 170 170 308 308 308 F F F 9 180-199 170 170 170 308 308 308 F F F 10 200-219 164 164 164 298 298 298 F F F 11 220-239 168 168 168 305 305 305 T T T 12 240-259 170 170 170 308 308 308 F F F 13 260-279 170 170 170 308 308 308 F F F 14 280-299 164 164 164 300 300 300 F F F 15 300-319 169 169 169 306 306 306 F F F 16 320-339 169 169 169 306 306 306 F F F 17 340-359 169 169 169 306 306 306 F F F 18 360-379 169 169 169 306 306 306 F F F 19 380-399 166 166 166 303 303 303 F F F 20 400-419 166 166 166 303 303 303 F F F 21 420-439 162 162 162 293 293 293 F F F 22 440-459 169 169 169 306 306 306 F F F 23 460-479 164 164 164 301 301 301 F F F 24 480-499 170 170 170 308 308 308 F F F 25 500-519 169 169 169 306 306 306 F F F 26 520-539 166 166 166 303 303 303 F F F 27 540-559 166 166 166 303 303 303 F F F 28 560-579 170 170 170 308 308 308 F F F 29 580-599 170 170 170 308 308 308 F F F Max gen: 170, Max flush: 308 A Difference in extent metadata is expected here On loop 85 Verifying data now No change after verify Loop: 85 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 167 167 167 304 304 304 F F F 1 020-039 171 171 171 309 309 309 F F F 2 040-059 171 171 171 309 309 309 F F F 3 060-079 169 169 169 306 306 306 F F F 4 080-099 169 169 169 306 306 306 F F F 5 100-119 158 158 158 290 290 290 F F F 6 120-139 171 171 171 309 309 309 F F F 7 140-159 171 171 171 309 309 309 F F F 8 160-179 170 170 170 308 308 308 F F F 9 180-199 170 170 170 308 308 308 F F F 10 200-219 164 164 164 298 298 298 F F F 11 220-239 171 171 171 309 309 309 F F F 12 240-259 170 170 170 308 308 308 F F F 13 260-279 170 170 170 308 308 308 F F F 14 280-299 164 164 164 300 300 300 F F F 15 300-319 169 169 169 306 306 306 F F F 16 320-339 169 169 169 306 306 306 F F F 17 340-359 169 169 169 306 306 306 F F F 18 360-379 169 169 169 306 306 306 F F F 19 380-399 166 166 166 303 303 303 F F F 20 400-419 166 166 166 303 303 303 F F F 21 420-439 162 162 162 293 293 293 F F F 22 440-459 169 169 169 306 306 306 F F F 23 460-479 164 164 164 301 301 301 F F F 24 480-499 170 170 170 308 308 308 F F F 25 500-519 169 169 169 306 306 306 F F F 26 520-539 166 166 166 303 303 303 F F F 27 540-559 166 166 166 303 303 303 F F F 28 560-579 170 170 170 308 308 308 F F F 29 580-599 170 170 170 308 308 308 F F F Max gen: 171, Max flush: 309 Begin loop 86 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:30.679074101Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1870,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:18:30.721838876Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1870,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:30.722249296Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1870,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:30.754067372Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1870,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:30.754685373Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1870} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"abd5430f1508af0bc008143ea2566277ef714ec2\",\n git_commit_timestamp: \"2025-01-17T20:55:03.000000000Z\",\n git_branch: \"alan/tokio-to-140\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:30.754722966Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1870} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:30.754737494Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1870} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:30.754751852Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1870} {"msg":"Crucible 871a6ea1-a6d8-45ef-935e-a8a24f6588fe has session id: c29d2bf2-db1c-4504-b1a1-2224d120596b","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:30.755330951Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1870,"session_id":"c29d2bf2-db1c-4504-b1a1-2224d120596b"} {"msg":"Upstairs opts: Upstairs UUID: 871a6ea1-a6d8-45ef-935e-a8a24f6588fe, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:30.755358687Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1870,"session_id":"c29d2bf2-db1c-4504-b1a1-2224d120596b"} {"msg":"Crucible stats registered with UUID: 871a6ea1-a6d8-45ef-935e-a8a24f6588fe","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:30.755372605Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1870,"session_id":"c29d2bf2-db1c-4504-b1a1-2224d120596b"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:18:30.755385073Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1870,"session_id":"c29d2bf2-db1c-4504-b1a1-2224d120596b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:30.755875679Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1870,"task":"crutest"} {"msg":"The guest has requested activation with gen:172","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:30.755907818Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1870} {"msg":"871a6ea1-a6d8-45ef-935e-a8a24f6588fe active request set","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:30.755966292Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1870,"session_id":"c29d2bf2-db1c-4504-b1a1-2224d120596b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:30.755992368Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1870,"client":"0","":"downstairs","session_id":"c29d2bf2-db1c-4504-b1a1-2224d120596b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:30.756019443Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1870,"client":"0","":"downstairs","session_id":"c29d2bf2-db1c-4504-b1a1-2224d120596b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:30.756033862Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1870,"client":"1","":"downstairs","session_id":"c29d2bf2-db1c-4504-b1a1-2224d120596b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:30.75604808Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1870,"client":"1","":"downstairs","session_id":"c29d2bf2-db1c-4504-b1a1-2224d120596b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:30.7560656Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1870,"client":"2","":"downstairs","session_id":"c29d2bf2-db1c-4504-b1a1-2224d120596b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:30.756080249Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1870,"client":"2","":"downstairs","session_id":"c29d2bf2-db1c-4504-b1a1-2224d120596b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:30.756093086Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1870,"":"io task","client":"2","":"downstairs","session_id":"c29d2bf2-db1c-4504-b1a1-2224d120596b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:30.756106234Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1870,"":"io task","client":"0","":"downstairs","session_id":"c29d2bf2-db1c-4504-b1a1-2224d120596b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:30.756119422Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1870,"":"io task","client":"1","":"downstairs","session_id":"c29d2bf2-db1c-4504-b1a1-2224d120596b"} {"msg":"ds_connection connected from Ok(127.0.0.1:60362)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:30.756148349Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1870,"":"io task","client":"2","":"downstairs","session_id":"c29d2bf2-db1c-4504-b1a1-2224d120596b"} {"msg":"ds_connection connected from Ok(127.0.0.1:42911)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:30.756163277Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1870,"":"io task","client":"0","":"downstairs","session_id":"c29d2bf2-db1c-4504-b1a1-2224d120596b"} {"msg":"ds_connection connected from Ok(127.0.0.1:54411)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:30.75618526Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1870,"":"io task","client":"1","":"downstairs","session_id":"c29d2bf2-db1c-4504-b1a1-2224d120596b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:30.756921891Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1870,"client":"2","":"downstairs","session_id":"c29d2bf2-db1c-4504-b1a1-2224d120596b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:30.757088768Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1870,"client":"1","":"downstairs","session_id":"c29d2bf2-db1c-4504-b1a1-2224d120596b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:30.757122038Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1870,"client":"0","":"downstairs","session_id":"c29d2bf2-db1c-4504-b1a1-2224d120596b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:30.757543384Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1870,"client":"2","":"downstairs","session_id":"c29d2bf2-db1c-4504-b1a1-2224d120596b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:30.757593293Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1870,"client":"1","":"downstairs","session_id":"c29d2bf2-db1c-4504-b1a1-2224d120596b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:30.757830942Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1870,"client":"0","":"downstairs","session_id":"c29d2bf2-db1c-4504-b1a1-2224d120596b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:30.758073864Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1870,"session_id":"c29d2bf2-db1c-4504-b1a1-2224d120596b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:30.758155312Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1870,"session_id":"c29d2bf2-db1c-4504-b1a1-2224d120596b"} {"msg":"[0]R flush_numbers[0..12]: [304, 309, 309, 306, 306, 290, 309, 309, 308, 308, 298, 309]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:30.758356219Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1870,"":"downstairs","":"downstairs","session_id":"c29d2bf2-db1c-4504-b1a1-2224d120596b"} {"msg":"[0]R generation[0..12]: [167, 171, 171, 169, 169, 158, 171, 171, 170, 170, 164, 171]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:30.75837515Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1870,"":"downstairs","":"downstairs","session_id":"c29d2bf2-db1c-4504-b1a1-2224d120596b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:30.758389769Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1870,"":"downstairs","":"downstairs","session_id":"c29d2bf2-db1c-4504-b1a1-2224d120596b"} {"msg":"[1]R flush_numbers[0..12]: [304, 309, 309, 306, 306, 290, 309, 309, 308, 308, 298, 309]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:30.758403697Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1870,"":"downstairs","":"downstairs","session_id":"c29d2bf2-db1c-4504-b1a1-2224d120596b"} {"msg":"[1]R generation[0..12]: [167, 171, 171, 169, 169, 158, 171, 171, 170, 170, 164, 171]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:30.758417135Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1870,"":"downstairs","":"downstairs","session_id":"c29d2bf2-db1c-4504-b1a1-2224d120596b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:30.758430602Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1870,"":"downstairs","":"downstairs","session_id":"c29d2bf2-db1c-4504-b1a1-2224d120596b"} {"msg":"[2]R flush_numbers[0..12]: [304, 309, 309, 306, 306, 290, 309, 309, 308, 308, 298, 309]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:30.75844381Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1870,"":"downstairs","":"downstairs","session_id":"c29d2bf2-db1c-4504-b1a1-2224d120596b"} {"msg":"[2]R generation[0..12]: [167, 171, 171, 169, 169, 158, 171, 171, 170, 170, 164, 171]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:30.758457178Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1870,"":"downstairs","":"downstairs","session_id":"c29d2bf2-db1c-4504-b1a1-2224d120596b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:30.758470756Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1870,"":"downstairs","":"downstairs","session_id":"c29d2bf2-db1c-4504-b1a1-2224d120596b"} {"msg":"Max found gen is 172","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:30.758483233Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1870,"":"downstairs","":"downstairs","session_id":"c29d2bf2-db1c-4504-b1a1-2224d120596b"} {"msg":"Generation requested: 172 >= found:172","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:30.758496061Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1870,"":"downstairs","":"downstairs","session_id":"c29d2bf2-db1c-4504-b1a1-2224d120596b"} {"msg":"Next flush: 310","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-17T21:18:30.758518994Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1870,"":"downstairs","":"downstairs","session_id":"c29d2bf2-db1c-4504-b1a1-2224d120596b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:30.758540607Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1870,"":"downstairs","":"downstairs","session_id":"c29d2bf2-db1c-4504-b1a1-2224d120596b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:30.758553665Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1870,"session_id":"c29d2bf2-db1c-4504-b1a1-2224d120596b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:30.758565431Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1870,"client":"0","":"downstairs","session_id":"c29d2bf2-db1c-4504-b1a1-2224d120596b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:30.758578239Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1870,"client":"1","":"downstairs","session_id":"c29d2bf2-db1c-4504-b1a1-2224d120596b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:30.758590936Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1870,"client":"2","":"downstairs","session_id":"c29d2bf2-db1c-4504-b1a1-2224d120596b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:30.758603584Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1870,"session_id":"c29d2bf2-db1c-4504-b1a1-2224d120596b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:30.758615591Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1870,"session_id":"c29d2bf2-db1c-4504-b1a1-2224d120596b"} {"msg":"871a6ea1-a6d8-45ef-935e-a8a24f6588fe is now active with session: c29d2bf2-db1c-4504-b1a1-2224d120596b","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:30.758628028Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1870,"session_id":"c29d2bf2-db1c-4504-b1a1-2224d120596b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:30.758639655Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1870,"session_id":"c29d2bf2-db1c-4504-b1a1-2224d120596b"} {"msg":"The guest has finished waiting for activation with:172","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:30.758651602Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1870} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 228 len 16384 data: 17 17 18 15 02/30 Read block 186 len 16384 03/30 Read block 393 len 40960 04/30 Read block 74 len 40960 05/30 Read block 463 len 20480 06/30 Write block 464 len 20480 data: 14 14 13 16 15 07/30 Write block 269 len 12288 data: 13 14 11 08/30 Read block 100 len 32768 09/30 Read block 497 len 36864 10/30 Write block 141 len 24576 data: 11 14 13 12 11 10 11/30 Read block 7 len 20480 12/30 Write block 197 len 40960 data: 15 15 20 17 15 15 11 10 10 9 13/30 Write block 560 len 28672 data: 18 19 19 20 13 12 17 14/30 Write block 253 len 24576 data: 10 12 14 13 14 12 15/30 Write block 267 len 12288 data: 9 9 14 16/30 Read block 199 len 24576 17/30 Read block 166 len 12288 18/30 Read block 259 len 12288 19/30 Read block 172 len 40960 20/30 Write block 557 len 36864 data: 11 15 14 19 20 20 21 14 13 21/30 Flush 22/30 Write block 252 len 28672 data: 11 11 13 15 14 15 13 23/30 Read block 228 len 20480 24/30 Write block 280 len 36864 data: 11 15 14 11 12 11 13 12 14 25/30 Write block 15 len 4096 data: 11 26/30 Read block 75 len 16384 27/30 Write block 36 len 40960 data: 16 16 15 15 16 15 12 10 11 9 28/30 Write block 12 len 32768 data: 15 16 13 12 13 12 11 10 29/30 Write block 514 len 24576 data: 22 25 24 19 17 16 30/30 Write block 516 len 32768 data: 25 20 18 17 14 13 10 18 ---------------------------------------------------------------- Crucible gen:172 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 7 Done Done Done false 1022 Acked Read 5 Done Done Done false 1023 Acked Write 9 Done Done Done false 1024 Acked Write 1 Done Done Done false 1025 Acked Read 4 Done Done Done false 1026 Acked Write 10 Done Done Done false 1027 Acked Write 8 Done Done Done false 1028 Acked Write 6 Sent Sent Sent false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 7 7 7 21 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1020 1020 1020 Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:18:30.778835748Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1870} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 167 167 167 304 304 304 T T T 1 020-039 171 171 171 309 309 309 T T T 2 040-059 171 171 171 309 309 309 T T T 3 060-079 169 169 169 306 306 306 F F F 4 080-099 169 169 169 306 306 306 F F F 5 100-119 158 158 158 290 290 290 F F F 6 120-139 171 171 171 309 309 309 F F F 7 140-159 172 172 172 310 310 310 F F F 8 160-179 170 170 170 308 308 308 F F F 9 180-199 172 172 172 310 310 310 F F F 10 200-219 172 172 172 310 310 310 F F F 11 220-239 172 172 172 310 310 310 F F F 12 240-259 172 172 172 310 310 310 T T T 13 260-279 172 172 172 310 310 310 F F F 14 280-299 164 164 164 300 300 300 T T T 15 300-319 169 169 169 306 306 306 F F F 16 320-339 169 169 169 306 306 306 F F F 17 340-359 169 169 169 306 306 306 F F F 18 360-379 169 169 169 306 306 306 F F F 19 380-399 166 166 166 303 303 303 F F F 20 400-419 166 166 166 303 303 303 F F F 21 420-439 162 162 162 293 293 293 F F F 22 440-459 169 169 169 306 306 306 F F F 23 460-479 172 172 172 310 310 310 F F F 24 480-499 170 170 170 308 308 308 F F F 25 500-519 169 169 169 306 306 306 T T T 26 520-539 166 166 166 303 303 303 T T T 27 540-559 172 172 172 310 310 310 F F F 28 560-579 172 172 172 310 310 310 F F F 29 580-599 170 170 170 308 308 308 F F F Max gen: 172, Max flush: 310 A Difference in extent metadata is expected here On loop 86 Verifying data now No change after verify Loop: 86 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 173 173 173 311 311 311 F F F 1 020-039 173 173 173 311 311 311 F F F 2 040-059 173 173 173 311 311 311 F F F 3 060-079 169 169 169 306 306 306 F F F 4 080-099 169 169 169 306 306 306 F F F 5 100-119 158 158 158 290 290 290 F F F 6 120-139 171 171 171 309 309 309 F F F 7 140-159 172 172 172 310 310 310 F F F 8 160-179 170 170 170 308 308 308 F F F 9 180-199 172 172 172 310 310 310 F F F 10 200-219 172 172 172 310 310 310 F F F 11 220-239 172 172 172 310 310 310 F F F 12 240-259 173 173 173 311 311 311 F F F 13 260-279 172 172 172 310 310 310 F F F 14 280-299 173 173 173 311 311 311 F F F 15 300-319 169 169 169 306 306 306 F F F 16 320-339 169 169 169 306 306 306 F F F 17 340-359 169 169 169 306 306 306 F F F 18 360-379 169 169 169 306 306 306 F F F 19 380-399 166 166 166 303 303 303 F F F 20 400-419 166 166 166 303 303 303 F F F 21 420-439 162 162 162 293 293 293 F F F 22 440-459 169 169 169 306 306 306 F F F 23 460-479 172 172 172 310 310 310 F F F 24 480-499 170 170 170 308 308 308 F F F 25 500-519 173 173 173 311 311 311 F F F 26 520-539 173 173 173 311 311 311 F F F 27 540-559 172 172 172 310 310 310 F F F 28 560-579 172 172 172 310 310 310 F F F 29 580-599 170 170 170 308 308 308 F F F Max gen: 173, Max flush: 311 Begin loop 87 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:35.066076516Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1878,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:18:35.108620582Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1878,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:35.109050853Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1878,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:35.140517905Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1878,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:35.141126531Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1878} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"abd5430f1508af0bc008143ea2566277ef714ec2\",\n git_commit_timestamp: \"2025-01-17T20:55:03.000000000Z\",\n git_branch: \"alan/tokio-to-140\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:35.141161001Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1878} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:35.141174899Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1878} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:35.141188957Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1878} {"msg":"Crucible ca3b64cd-f7a5-4c07-a9a0-15d66a99e772 has session id: 89a43acf-b9b6-4b2b-adfc-bffb15908fac","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:35.141795131Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1878,"session_id":"89a43acf-b9b6-4b2b-adfc-bffb15908fac"} {"msg":"Upstairs opts: Upstairs UUID: ca3b64cd-f7a5-4c07-a9a0-15d66a99e772, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:35.141833434Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1878,"session_id":"89a43acf-b9b6-4b2b-adfc-bffb15908fac"} {"msg":"Crucible stats registered with UUID: ca3b64cd-f7a5-4c07-a9a0-15d66a99e772","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:35.141862601Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1878,"session_id":"89a43acf-b9b6-4b2b-adfc-bffb15908fac"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:18:35.141884254Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1878,"session_id":"89a43acf-b9b6-4b2b-adfc-bffb15908fac"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:35.142367186Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1878,"task":"crutest"} {"msg":"The guest has requested activation with gen:174","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:35.14239177Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1878} {"msg":"ca3b64cd-f7a5-4c07-a9a0-15d66a99e772 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:35.142454357Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1878,"session_id":"89a43acf-b9b6-4b2b-adfc-bffb15908fac"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:35.142471427Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1878,"client":"0","":"downstairs","session_id":"89a43acf-b9b6-4b2b-adfc-bffb15908fac"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:35.142495991Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1878,"client":"0","":"downstairs","session_id":"89a43acf-b9b6-4b2b-adfc-bffb15908fac"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:35.142510099Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1878,"client":"1","":"downstairs","session_id":"89a43acf-b9b6-4b2b-adfc-bffb15908fac"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:35.142524758Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1878,"client":"1","":"downstairs","session_id":"89a43acf-b9b6-4b2b-adfc-bffb15908fac"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:35.142537756Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1878,"client":"2","":"downstairs","session_id":"89a43acf-b9b6-4b2b-adfc-bffb15908fac"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:35.142551073Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1878,"client":"2","":"downstairs","session_id":"89a43acf-b9b6-4b2b-adfc-bffb15908fac"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:35.14261426Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1878,"":"io task","client":"2","":"downstairs","session_id":"89a43acf-b9b6-4b2b-adfc-bffb15908fac"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:35.142649591Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1878,"":"io task","client":"0","":"downstairs","session_id":"89a43acf-b9b6-4b2b-adfc-bffb15908fac"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:35.142667051Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1878,"":"io task","client":"1","":"downstairs","session_id":"89a43acf-b9b6-4b2b-adfc-bffb15908fac"} {"msg":"ds_connection connected from Ok(127.0.0.1:47332)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:35.142690515Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1878,"":"io task","client":"1","":"downstairs","session_id":"89a43acf-b9b6-4b2b-adfc-bffb15908fac"} {"msg":"ds_connection connected from Ok(127.0.0.1:33093)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:35.143047143Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1878,"":"io task","client":"0","":"downstairs","session_id":"89a43acf-b9b6-4b2b-adfc-bffb15908fac"} {"msg":"ds_connection connected from Ok(127.0.0.1:61954)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:35.143121226Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1878,"":"io task","client":"2","":"downstairs","session_id":"89a43acf-b9b6-4b2b-adfc-bffb15908fac"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:35.143512445Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1878,"client":"1","":"downstairs","session_id":"89a43acf-b9b6-4b2b-adfc-bffb15908fac"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:35.143551988Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1878,"client":"2","":"downstairs","session_id":"89a43acf-b9b6-4b2b-adfc-bffb15908fac"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:35.143582706Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1878,"client":"0","":"downstairs","session_id":"89a43acf-b9b6-4b2b-adfc-bffb15908fac"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:35.144164516Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1878,"client":"1","":"downstairs","session_id":"89a43acf-b9b6-4b2b-adfc-bffb15908fac"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:35.144203629Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1878,"client":"2","":"downstairs","session_id":"89a43acf-b9b6-4b2b-adfc-bffb15908fac"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:35.144232216Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1878,"client":"0","":"downstairs","session_id":"89a43acf-b9b6-4b2b-adfc-bffb15908fac"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:35.144699379Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1878,"session_id":"89a43acf-b9b6-4b2b-adfc-bffb15908fac"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:35.14480342Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1878,"session_id":"89a43acf-b9b6-4b2b-adfc-bffb15908fac"} {"msg":"[0]R flush_numbers[0..12]: [311, 311, 311, 306, 306, 290, 309, 310, 308, 310, 310, 310]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:35.144871169Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1878,"":"downstairs","":"downstairs","session_id":"89a43acf-b9b6-4b2b-adfc-bffb15908fac"} {"msg":"[0]R generation[0..12]: [173, 173, 173, 169, 169, 158, 171, 172, 170, 172, 172, 172]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:35.144891061Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1878,"":"downstairs","":"downstairs","session_id":"89a43acf-b9b6-4b2b-adfc-bffb15908fac"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:35.14490564Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1878,"":"downstairs","":"downstairs","session_id":"89a43acf-b9b6-4b2b-adfc-bffb15908fac"} {"msg":"[1]R flush_numbers[0..12]: [311, 311, 311, 306, 306, 290, 309, 310, 308, 310, 310, 310]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:35.144919377Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1878,"":"downstairs","":"downstairs","session_id":"89a43acf-b9b6-4b2b-adfc-bffb15908fac"} {"msg":"[1]R generation[0..12]: [173, 173, 173, 169, 169, 158, 171, 172, 170, 172, 172, 172]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:35.144932955Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1878,"":"downstairs","":"downstairs","session_id":"89a43acf-b9b6-4b2b-adfc-bffb15908fac"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:35.144950386Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1878,"":"downstairs","":"downstairs","session_id":"89a43acf-b9b6-4b2b-adfc-bffb15908fac"} {"msg":"[2]R flush_numbers[0..12]: [311, 311, 311, 306, 306, 290, 309, 310, 308, 310, 310, 310]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:35.144964104Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1878,"":"downstairs","":"downstairs","session_id":"89a43acf-b9b6-4b2b-adfc-bffb15908fac"} {"msg":"[2]R generation[0..12]: [173, 173, 173, 169, 169, 158, 171, 172, 170, 172, 172, 172]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:35.144977502Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1878,"":"downstairs","":"downstairs","session_id":"89a43acf-b9b6-4b2b-adfc-bffb15908fac"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:35.145014533Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1878,"":"downstairs","":"downstairs","session_id":"89a43acf-b9b6-4b2b-adfc-bffb15908fac"} {"msg":"Max found gen is 174","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:35.145029032Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1878,"":"downstairs","":"downstairs","session_id":"89a43acf-b9b6-4b2b-adfc-bffb15908fac"} {"msg":"Generation requested: 174 >= found:174","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:35.145042089Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1878,"":"downstairs","":"downstairs","session_id":"89a43acf-b9b6-4b2b-adfc-bffb15908fac"} {"msg":"Next flush: 312","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:35.145054647Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1878,"":"downstairs","":"downstairs","session_id":"89a43acf-b9b6-4b2b-adfc-bffb15908fac"} {"msg":"All extents match","v":Wait for a query_work_queue command to finish before sending IO 0,"name":"crucible","level":30,"time":"2025-01-17T21:18:35.145075449Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1878,"":"downstairs","":"downstairs","session_id":"89a43acf-b9b6-4b2b-adfc-bffb15908fac"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:35.145088206Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1878,"session_id":"89a43acf-b9b6-4b2b-adfc-bffb15908fac"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:35.145099903Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1878,"client":"0","":"downstairs","session_id":"89a43acf-b9b6-4b2b-adfc-bffb15908fac"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:35.145112621Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1878,"client":"1","":"downstairs","session_id":"89a43acf-b9b6-4b2b-adfc-bffb15908fac"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:35.145125198Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1878,"client":"2","":"downstairs","session_id":"89a43acf-b9b6-4b2b-adfc-bffb15908fac"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:35.145137925Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1878,"session_id":"89a43acf-b9b6-4b2b-adfc-bffb15908fac"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:35.145149782Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1878,"session_id":"89a43acf-b9b6-4b2b-adfc-bffb15908fac"} {"msg":"ca3b64cd-f7a5-4c07-a9a0-15d66a99e772 is now active with session: 89a43acf-b9b6-4b2b-adfc-bffb15908fac","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:35.14516223Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1878,"session_id":"89a43acf-b9b6-4b2b-adfc-bffb15908fac"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:35.145176027Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1878,"session_id":"89a43acf-b9b6-4b2b-adfc-bffb15908fac"} {"msg":"The guest has finished waiting for activation with:174","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:35.145188315Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1878} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 90 len 20480 data: 20 19 18 16 17 03/30 Read block 439 len 4096 04/30 Write block 452 len 36864 data: 9 8 12 13 13 14 14 13 13 05/30 Read block 466 len 32768 06/30 Flush 07/30 Write block 308 len 4096 data: 10 08/30 Write block 252 len 12288 data: 12 12 14 09/30 Read block 158 len 4096 10/30 Read block 238 len 4096 11/30 Read block 426 len 40960 12/30 Read block 301 len 36864 13/30 Read block 339 len 4096 14/30 Write block 173 len 16384 data: 12 12 14 13 15/30 Write block 232 len 4096 data: 13 16/30 Read block 53 len 32768 17/30 Flush 18/30 Write block 275 len 16384 data: 14 15 14 11 19/30 Read block 445 len 16384 20/30 Read block 198 len 4096 21/30 Read block 463 len 16384 22/30 Write block 494 len 8192 data: 11 13 23/30 Read block 360 len 4096 24/30 Write block 278 len 20480 data: 12 9 12 16 15 25/30 Write block 344 len 24576 data: 11 10 8 10 13 11 26/30 Read block 120 len 20480 27/30 Write block 367 len 24576 data: 7 7 7 12 12 16 28/30 Write block 432 len 20480 data: 13 10 13 10 10 29/30 Write block 377 len 4096 data: 22 30/30 Write block 162 len 28672 data: 13 10 13 14 14 14 15 ---------------------------------------------------------------- Crucible gen:174 GIO:true work queues: Upstairs:0 downstairs:13 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1017 Acked Write 4 Done Done Done false 1018 Acked Read 4 Done Done Done false 1019 Acked Read 1 Done Done Done false 1020 Acked Read 4 Done Done Done false 1021 Acked Write 2 Done Done Done false 1022 Acked Read 1 Done Done Done false 1023 Acked Write 5 Done Done Done false 1024 Acked Write 6 Done Done Done false 1025 Acked Read 5 Done Done Done false 1026 Acked Write 6 Done Done Done false 1027 Acked Write 5 Done Done Sent false 1028 Acked Write 1 Done Sent Sent false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 2 3 6 Done 12 11 10 33 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1016 1016 1016 Downstairs last five completed: 1016 1015 1014 1013 1012 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:18:35.16278608Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1878} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 173 173 173 311 311 311 F F F 1 020-039 173 173 173 311 311 311 F F F 2 040-059 173 173 173 311 311 311 F F F 3 060-079 169 169 169 306 306 306 F F F 4 080-099 174 174 174 313 313 313 F F F 5 100-119 158 158 158 290 290 290 F F F 6 120-139 171 171 171 309 309 309 F F F 7 140-159 172 172 172 310 310 310 F F F 8 160-179 174 174 174 314 314 314 T T T 9 180-199 172 172 172 310 310 310 F F F 10 200-219 172 172 172 310 310 310 F F F 11 220-239 174 174 174 314 314 314 F F F 12 240-259 174 174 174 314 314 314 F F F 13 260-279 172 172 172 310 310 310 T T T 14 280-299 173 173 173 311 311 311 T T T 15 300-319 174 174 174 314 314 314 F F F 16 320-339 169 169 169 306 306 306 F F F 17 340-359 169 169 169 306 306 306 T T T 18 360-379 169 169 169 306 306 306 T T T 19 380-399 166 166 166 303 303 303 F F F 20 400-419 166 166 166 303 303 303 F F F 21 420-439 162 162 162 293 293 293 T T T 22 440-459 174 174 174 313 313 313 F F F 23 460-479 174 174 174 313 313 313 F F F 24 480-499 170 170 170 308 308 308 T T T 25 500-519 173 173 173 311 311 311 F F F 26 520-539 173 173 173 311 311 311 F F F 27 540-559 172 172 172 310 310 310 F F F 28 560-579 172 172 172 310 310 310 F F F 29 580-599 170 170 170 308 308 308 F F F Max gen: 174, Max flush: 314 A Difference in extent metadata is expected here On loop 87 Verifying data now No change after verify Loop: 87 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 173 173 173 311 311 311 F F F 1 020-039 173 173 173 311 311 311 F F F 2 040-059 173 173 173 311 311 311 F F F 3 060-079 169 169 169 306 306 306 F F F 4 080-099 174 174 174 313 313 313 F F F 5 100-119 158 158 158 290 290 290 F F F 6 120-139 171 171 171 309 309 309 F F F 7 140-159 172 172 172 310 310 310 F F F 8 160-179 175 175 175 315 315 315 F F F 9 180-199 172 172 172 310 310 310 F F F 10 200-219 172 172 172 310 310 310 F F F 11 220-239 174 174 174 314 314 314 F F F 12 240-259 174 174 174 314 314 314 F F F 13 260-279 175 175 175 315 315 315 F F F 14 280-299 175 175 175 315 315 315 F F F 15 300-319 174 174 174 314 314 314 F F F 16 320-339 169 169 169 306 306 306 F F F 17 340-359 175 175 175 315 315 315 F F F 18 360-379 175 175 175 315 315 315 F F F 19 380-399 166 166 166 303 303 303 F F F 20 400-419 166 166 166 303 303 303 F F F 21 420-439 175 175 175 315 315 315 F F F 22 440-459 174 174 174 313 313 313 F F F 23 460-479 174 174 174 313 313 313 F F F 24 480-499 175 175 175 315 315 315 F F F 25 500-519 173 173 173 311 311 311 F F F 26 520-539 173 173 173 311 311 311 F F F 27 540-559 172 172 172 310 310 310 F F F 28 560-579 172 172 172 310 310 310 F F F 29 580-599 170 170 170 308 308 308 F F F Max gen: 175, Max flush: 315 Begin loop 88 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:39.447092893Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1886,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:18:39.490420567Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1886,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:39.49087107Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1886,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:39.522837154Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1886,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:39.523430681Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1886} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"abd5430f1508af0bc008143ea2566277ef714ec2\",\n git_commit_timestamp: \"2025-01-17T20:55:03.000000000Z\",\n git_branch: \"alan/tokio-to-140\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:39.52348183Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1886} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:39.523500431Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1886} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:39.523518282Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1886} {"msg":"Crucible 34cc49bb-f7de-43f5-86a7-8a3b2d07d139 has session id: cab2a6ea-5865-4346-bf94-f27c4ff3496e","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:39.524083232Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1886,"session_id":"cab2a6ea-5865-4346-bf94-f27c4ff3496e"} {"msg":"Upstairs opts: Upstairs UUID: 34cc49bb-f7de-43f5-86a7-8a3b2d07d139, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:39.524123365Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1886,"session_id":"cab2a6ea-5865-4346-bf94-f27c4ff3496e"} {"msg":"Crucible stats registered with UUID: 34cc49bb-f7de-43f5-86a7-8a3b2d07d139","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:39.52414857Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1886,"session_id":"cab2a6ea-5865-4346-bf94-f27c4ff3496e"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:18:39.524169112Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1886,"session_id":"cab2a6ea-5865-4346-bf94-f27c4ff3496e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:39.524553197Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1886,"task":"crutest"} {"msg":"The guest has requested activation with gen:176","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:39.524577261Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1886} {"msg":"34cc49bb-f7de-43f5-86a7-8a3b2d07d139 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:39.524645351Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1886,"session_id":"cab2a6ea-5865-4346-bf94-f27c4ff3496e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:39.524662821Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1886,"client":"0","":"downstairs","session_id":"cab2a6ea-5865-4346-bf94-f27c4ff3496e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:39.524686194Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1886,"client":"0","":"downstairs","session_id":"cab2a6ea-5865-4346-bf94-f27c4ff3496e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:39.524700083Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1886,"client":"1","":"downstairs","session_id":"cab2a6ea-5865-4346-bf94-f27c4ff3496e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:39.524717132Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1886,"client":"1","":"downstairs","session_id":"cab2a6ea-5865-4346-bf94-f27c4ff3496e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:39.524732892Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1886,"client":"2","":"downstairs","session_id":"cab2a6ea-5865-4346-bf94-f27c4ff3496e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:39.524756375Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1886,"client":"2","":"downstairs","session_id":"cab2a6ea-5865-4346-bf94-f27c4ff3496e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:39.524778989Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1886,"":"io task","client":"2","":"downstairs","session_id":"cab2a6ea-5865-4346-bf94-f27c4ff3496e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:39.524802282Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1886,"":"io task","client":"0","":"downstairs","session_id":"cab2a6ea-5865-4346-bf94-f27c4ff3496e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:39.524826937Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1886,"":"io task","client":"1","":"downstairs","session_id":"cab2a6ea-5865-4346-bf94-f27c4ff3496e"} {"msg":"ds_connection connected from Ok(127.0.0.1:52388)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:39.524862938Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1886,"":"io task","client":"0","":"downstairs","session_id":"cab2a6ea-5865-4346-bf94-f27c4ff3496e"} {"msg":"ds_connection connected from Ok(127.0.0.1:42228)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:39.525048626Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1886,"":"io task","client":"1","":"downstairs","session_id":"cab2a6ea-5865-4346-bf94-f27c4ff3496e"} {"msg":"ds_connection connected from Ok(127.0.0.1:63740)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:39.525086028Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1886,"":"io task","client":"2","":"downstairs","session_id":"cab2a6ea-5865-4346-bf94-f27c4ff3496e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:39.525668108Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1886,"client":"0","":"downstairs","session_id":"cab2a6ea-5865-4346-bf94-f27c4ff3496e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:39.525696715Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1886,"client":"1","":"downstairs","session_id":"cab2a6ea-5865-4346-bf94-f27c4ff3496e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:39.525711674Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1886,"client":"2","":"downstairs","session_id":"cab2a6ea-5865-4346-bf94-f27c4ff3496e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:39.526328895Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1886,"client":"2","":"downstairs","session_id":"cab2a6ea-5865-4346-bf94-f27c4ff3496e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:39.526357211Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1886,"client":"1","":"downstairs","session_id":"cab2a6ea-5865-4346-bf94-f27c4ff3496e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:39.526465944Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1886,"client":"0","":"downstairs","session_id":"cab2a6ea-5865-4346-bf94-f27c4ff3496e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:39.526864677Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1886,"session_id":"cab2a6ea-5865-4346-bf94-f27c4ff3496e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:39.526892524Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1886,"session_id":"cab2a6ea-5865-4346-bf94-f27c4ff3496e"} {"msg":"[0]R flush_numbers[0..12]: [311, 311, 311, 306, 313, 290, 309, 310, 315, 310, 310, 314]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:39.526979655Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1886,"":"downstairs","":"downstairs","session_id":"cab2a6ea-5865-4346-bf94-f27c4ff3496e"} {"msg":"[0]R generation[0..12]: [173, 173, 173, 169, 174, 158, 171, 172, 175, 172, 172, 174]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:39.526998306Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1886,"":"downstairs","":"downstairs","session_id":"cab2a6ea-5865-4346-bf94-f27c4ff3496e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:39.527012614Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1886,"":"downstairs","":"downstairs","session_id":"cab2a6ea-5865-4346-bf94-f27c4ff3496e"} {"msg":"[1]R flush_numbers[0..12]: [311, 311, 311, 306, 313, 290, 309, 310, 315, 310, 310, 314]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:39.527026292Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1886,"":"downstairs","":"downstairs","session_id":"cab2a6ea-5865-4346-bf94-f27c4ff3496e"} {"msg":"[1]R generation[0..12]: [173, 173, 173, 169, 174, 158, 171, 172, 175, 172, 172, 174]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:39.52703967Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1886,"":"downstairs","":"downstairs","session_id":"cab2a6ea-5865-4346-bf94-f27c4ff3496e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:39.527053148Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1886,"":"downstairs","":"downstairs","session_id":"cab2a6ea-5865-4346-bf94-f27c4ff3496e"} {"msg":"[2]R flush_numbers[0..12]: [311, 311, 311, 306, 313, 290, 309, 310, 315, 310, 310, 314]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:39.527066575Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1886,"":"downstairs","":"downstairs","session_id":"cab2a6ea-5865-4346-bf94-f27c4ff3496e"} {"msg":"[2]R generation[0..12]: [173, 173, 173, 169, 174, 158, 171, 172, 175, 172, 172, 174]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:39.527079823Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1886,"":"downstairs","":"downstairs","session_id":"cab2a6ea-5865-4346-bf94-f27c4ff3496e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:39.527093211Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1886,"":"downstairs","":"downstairs","session_id":"cab2a6ea-5865-4346-bf94-f27c4ff3496e"} {"msg":"Max found gen is 176","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:39.527105638Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1886,"":"downstairs","":"downstairs","session_id":"cab2a6ea-5865-4346-bf94-f27c4ff3496e"} {"msg":"Generation requested: 176 >= found:176","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:39.527118376Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1886,"":"downstairs","":"downstairs","session_id":"cab2a6ea-5865-4346-bf94-f27c4ff3496e"} {"msg":"Next flush: 316","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:39.527134525Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1886,"":"downstairs","":"downstairs","session_id":"cab2a6ea-5865-4346-bf94-f27c4ff3496e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:39.527147223Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1886,"":"downstairs","":"downstairs","session_id":"cab2a6ea-5865-4346-bf94-f27c4ff3496e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:39.52715997Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1886,"session_id":"cab2a6ea-5865-4346-bf94-f27c4ff3496e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:39.527171637Z","hostnameWait for a query_work_queue command to finish before sending IO ":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1886,"client":"0","":"downstairs","session_id":"cab2a6ea-5865-4346-bf94-f27c4ff3496e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:39.527192759Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1886,"client":"1","":"downstairs","session_id":"cab2a6ea-5865-4346-bf94-f27c4ff3496e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:39.527205707Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1886,"client":"2","":"downstairs","session_id":"cab2a6ea-5865-4346-bf94-f27c4ff3496e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:39.527218524Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1886,"session_id":"cab2a6ea-5865-4346-bf94-f27c4ff3496e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:39.527230421Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1886,"session_id":"cab2a6ea-5865-4346-bf94-f27c4ff3496e"} {"msg":"34cc49bb-f7de-43f5-86a7-8a3b2d07d139 is now active with session: cab2a6ea-5865-4346-bf94-f27c4ff3496e","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:39.527242629Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1886,"session_id":"cab2a6ea-5865-4346-bf94-f27c4ff3496e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:39.527253985Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1886,"session_id":"cab2a6ea-5865-4346-bf94-f27c4ff3496e"} {"msg":"The guest has finished waiting for activation with:176","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:39.527265782Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1886} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 274 len 4096 data: 12 02/30 Flush 03/30 Write block 396 len 20480 data: 13 14 14 12 11 04/30 Read block 420 len 8192 05/30 Flush 06/30 Write block 423 len 32768 data: 13 12 11 12 14 16 16 14 07/30 Read block 149 len 32768 08/30 Write block 40 len 36864 data: 17 16 13 11 12 10 11 12 14 09/30 Read block 210 len 24576 10/30 Read block 154 len 24576 11/30 Read block 302 len 4096 12/30 Write block 256 len 4096 data: 15 13/30 Write block 243 len 24576 data: 12 13 11 11 13 12 14/30 Read block 583 len 8192 15/30 Write block 223 len 28672 data: 12 10 12 15 16 18 18 16/30 Write block 202 len 36864 data: 16 12 11 11 10 9 11 13 13 17/30 Read block 572 len 16384 18/30 Read block 494 len 20480 19/30 Write block 24 len 32768 data: 8 10 14 15 13 12 15 13 20/30 Write block 249 len 4096 data: 13 21/30 Write block 193 len 20480 data: 16 16 15 16 16 22/30 Read block 215 len 12288 23/30 Flush 24/30 Write block 490 len 24576 data: 14 13 10 10 12 14 25/30 Flush 26/30 Write block 260 len 8192 data: 13 12 27/30 Read block 137 len 20480 28/30 Write block 378 len 16384 data: 22 22 19 15 29/30 Write block 377 len 16384 data: 23 23 23 20 30/30 Read block 17 len 20480 ---------------------------------------------------------------- Crucible gen:176 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 2 Done Done Done false 1026 Acked Read 5 Done Done Done false 1027 Acked Write 4 Done Done Done false 1028 Acked Write 4 Done Done Sent false 1029 Acked Read 5 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 2 3 Done 4 5 3 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:18:39.546041036Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1886} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 173 173 173 311 311 311 F F F 1 020-039 176 176 176 318 318 318 F F F 2 040-059 176 176 176 318 318 318 F F F 3 060-079 169 169 169 306 306 306 F F F 4 080-099 174 174 174 313 313 313 F F F 5 100-119 158 158 158 290 290 290 F F F 6 120-139 171 171 171 309 309 309 F F F 7 140-159 172 172 172 310 310 310 F F F 8 160-179 175 175 175 315 315 315 F F F 9 180-199 176 176 176 318 318 318 F F F 10 200-219 176 176 176 318 318 318 F F F 11 220-239 176 176 176 318 318 318 F F F 12 240-259 176 176 176 318 318 318 F F F 13 260-279 176 176 176 316 316 316 T T T 14 280-299 175 175 175 315 315 315 F F F 15 300-319 174 174 174 314 314 314 F F F 16 320-339 169 169 169 306 306 306 F F F 17 340-359 175 175 175 315 315 315 F F F 18 360-379 175 175 175 315 315 315 T T T 19 380-399 176 176 176 317 317 317 T T T 20 400-419 176 176 176 317 317 317 F F F 21 420-439 176 176 176 318 318 318 F F F 22 440-459 174 174 174 313 313 313 F F F 23 460-479 174 174 174 313 313 313 F F F 24 480-499 176 176 176 319 319 319 F F F 25 500-519 173 173 173 311 311 311 F F F 26 520-539 173 173 173 311 311 311 F F F 27 540-559 172 172 172 310 310 310 F F F 28 560-579 172 172 172 310 310 310 F F F 29 580-599 170 170 170 308 308 308 F F F Max gen: 176, Max flush: 319 A Difference in extent metadata is expected here On loop 88 Verifying data now No change after verify Loop: 88 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 173 173 173 311 311 311 F F F 1 020-039 176 176 176 318 318 318 F F F 2 040-059 176 176 176 318 318 318 F F F 3 060-079 169 169 169 306 306 306 F F F 4 080-099 174 174 174 313 313 313 F F F 5 100-119 158 158 158 290 290 290 F F F 6 120-139 171 171 171 309 309 309 F F F 7 140-159 172 172 172 310 310 310 F F F 8 160-179 175 175 175 315 315 315 F F F 9 180-199 176 176 176 318 318 318 F F F 10 200-219 176 176 176 318 318 318 F F F 11 220-239 176 176 176 318 318 318 F F F 12 240-259 176 176 176 318 318 318 F F F 13 260-279 177 177 177 320 320 320 F F F 14 280-299 175 175 175 315 315 315 F F F 15 300-319 174 174 174 314 314 314 F F F 16 320-339 169 169 169 306 306 306 F F F 17 340-359 175 175 175 315 315 315 F F F 18 360-379 177 177 177 320 320 320 F F F 19 380-399 177 177 177 320 320 320 F F F 20 400-419 176 176 176 317 317 317 F F F 21 420-439 176 176 176 318 318 318 F F F 22 440-459 174 174 174 313 313 313 F F F 23 460-479 174 174 174 313 313 313 F F F 24 480-499 176 176 176 319 319 319 F F F 25 500-519 173 173 173 311 311 311 F F F 26 520-539 173 173 173 311 311 311 F F F 27 540-559 172 172 172 310 310 310 F F F 28 560-579 172 172 172 310 310 310 F F F 29 580-599 170 170 170 308 308 308 F F F Max gen: 177, Max flush: 320 Begin loop 89 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:43.807033112Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1894,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:18:43.849685391Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1894,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:43.850124077Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1894,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:43.881587256Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1894,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:43.882195682Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1894} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"abd5430f1508af0bc008143ea2566277ef714ec2\",\n git_commit_timestamp: \"2025-01-17T20:55:03.000000000Z\",\n git_branch: \"alan/tokio-to-140\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:43.882230402Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1894} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:43.88224427Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1894} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:43.882259779Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1894} {"msg":"Crucible 77eac5aa-f28c-4813-a29b-5841fcd46bbe has session id: 838557e1-4551-4143-ba02-98927cf0945a","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:43.882807479Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1894,"session_id":"838557e1-4551-4143-ba02-98927cf0945a"} {"msg":"Upstairs opts: Upstairs UUID: 77eac5aa-f28c-4813-a29b-5841fcd46bbe, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:43.882846032Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1894,"session_id":"838557e1-4551-4143-ba02-98927cf0945a"} {"msg":"Crucible stats registered with UUID: 77eac5aa-f28c-4813-a29b-5841fcd46bbe","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:43.882870636Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1894,"session_id":"838557e1-4551-4143-ba02-98927cf0945a"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:18:43.882896171Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1894,"session_id":"838557e1-4551-4143-ba02-98927cf0945a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:43.883260114Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1894,"task":"crutest"} {"msg":"The guest has requested activation with gen:178","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:43.883283388Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1894} {"msg":"77eac5aa-f28c-4813-a29b-5841fcd46bbe active request set","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:43.883347115Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1894,"session_id":"838557e1-4551-4143-ba02-98927cf0945a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:43.883364225Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1894,"client":"0","":"downstairs","session_id":"838557e1-4551-4143-ba02-98927cf0945a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:43.883387298Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1894,"client":"0","":"downstairs","session_id":"838557e1-4551-4143-ba02-98927cf0945a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:43.883400846Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1894,"client":"1","":"downstairs","session_id":"838557e1-4551-4143-ba02-98927cf0945a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:43.883414845Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1894,"client":"1","":"downstairs","session_id":"838557e1-4551-4143-ba02-98927cf0945a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:43.883427672Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1894,"client":"2","":"downstairs","session_id":"838557e1-4551-4143-ba02-98927cf0945a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:43.883470347Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1894,"client":"2","":"downstairs","session_id":"838557e1-4551-4143-ba02-98927cf0945a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:43.883546722Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1894,"":"io task","client":"2","":"downstairs","session_id":"838557e1-4551-4143-ba02-98927cf0945a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:43.883593699Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1894,"":"io task","client":"0","":"downstairs","session_id":"838557e1-4551-4143-ba02-98927cf0945a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:43.88362851Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1894,"":"io task","client":"1","":"downstairs","session_id":"838557e1-4551-4143-ba02-98927cf0945a"} {"msg":"ds_connection connected from Ok(127.0.0.1:36638)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:43.883731079Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1894,"":"io task","client":"2","":"downstairs","session_id":"838557e1-4551-4143-ba02-98927cf0945a"} {"msg":"ds_connection connected from Ok(127.0.0.1:37139)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:43.883770773Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1894,"":"io task","client":"1","":"downstairs","session_id":"838557e1-4551-4143-ba02-98927cf0945a"} {"msg":"ds_connection connected from Ok(127.0.0.1:38031)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:43.883797598Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1894,"":"io task","client":"0","":"downstairs","session_id":"838557e1-4551-4143-ba02-98927cf0945a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:43.884338144Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1894,"client":"1","":"downstairs","session_id":"838557e1-4551-4143-ba02-98927cf0945a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:43.88436482Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1894,"client":"2","":"downstairs","session_id":"838557e1-4551-4143-ba02-98927cf0945a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:43.884379939Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1894,"client":"0","":"downstairs","session_id":"838557e1-4551-4143-ba02-98927cf0945a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:43.884966111Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1894,"client":"2","":"downstairs","session_id":"838557e1-4551-4143-ba02-98927cf0945a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:43.88511783Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1894,"client":"1","":"downstairs","session_id":"838557e1-4551-4143-ba02-98927cf0945a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:43.885149308Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1894,"client":"0","":"downstairs","session_id":"838557e1-4551-4143-ba02-98927cf0945a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:43.885458049Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1894,"session_id":"838557e1-4551-4143-ba02-98927cf0945a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:43.885658336Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1894,"session_id":"838557e1-4551-4143-ba02-98927cf0945a"} {"msg":"[0]R flush_numbers[0..12]: [311, 318, 318, 306, 313, 290, 309, 310, 315, 318, 318, 318]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:43.885788652Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1894,"":"downstairs","":"downstairs","session_id":"838557e1-4551-4143-ba02-98927cf0945a"} {"msg":"[0]R generation[0..12]: [173, 176, 176, 169, 174, 158, 171, 172, 175, 176, 176, 176]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:43.885808364Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1894,"":"downstairs","":"downstairs","session_id":"838557e1-4551-4143-ba02-98927cf0945a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:43.885822882Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1894,"":"downstairs","":"downstairs","session_id":"838557e1-4551-4143-ba02-98927cf0945a"} {"msg":"[1]R flush_numbers[0..12]: [311, 318, 318, 306, 313, 290, 309, 310, 315, 318, 318, 318]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:43.88583636Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1894,"":"downstairs","":"downstairs","session_id":"838557e1-4551-4143-ba02-98927cf0945a"} {"msg":"[1]R generation[0..12]: [173, 176, 176, 169, 174, 158, 171, 172, 175, 176, 176, 176]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:43.885849948Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1894,"":"downstairs","":"downstairs","session_id":"838557e1-4551-4143-ba02-98927cf0945a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:43.885868489Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1894,"":"downstairs","":"downstairs","session_id":"838557e1-4551-4143-ba02-98927cf0945a"} {"msg":"[2]R flush_numbers[0..12]: [311, 318, 318, 306, 313, 290, 309, 310, 315, 318, 318, 318]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:43.885882807Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1894,"":"downstairs","":"downstairs","session_id":"838557e1-4551-4143-ba02-98927cf0945a"} {"msg":"[2]R generation[0..12]: [173, 176, 176, 169, 174, 158, 171, 172, 175, 176, 176, 176]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:43.885896295Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1894,"":"downstairs","":"downstairs","session_id":"838557e1-4551-4143-ba02-98927cf0945a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:43.885909793Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1894,"":"downstairs","":"downstairs","session_id":"838557e1-4551-4143-ba02-98927cf0945a"} {"msg":"Max found gen is 178","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:43.885937059Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1894,"":"downstairs","":"downstairs","session_id":"838557e1-4551-4143-ba02-98927cf0945a"} {"msg":"Generation requested: 178 >= found:178","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:43.885950717Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1894,"":"downstairs","":"downstairs","session_id":"838557e1-4551-4143-ba02-98927cf0945a"} {"msg":"Next flush: 321","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:43.885963064Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1894,"":"downstairs","":"downstairs","session_id":"838557e1-4551-4143-ba02-98927cf0945a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:43.885975501Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1894,"":"downstairs","":"downstairs","session_id":"838557e1-4551-4143-ba02-98927cf0945a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level"Wait for a query_work_queue command to finish before sending IO :30,"time":"2025-01-17T21:18:43.886035726Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1894,"session_id":"838557e1-4551-4143-ba02-98927cf0945a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:43.886053257Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1894,"client":"0","":"downstairs","session_id":"838557e1-4551-4143-ba02-98927cf0945a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:43.886066855Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1894,"client":"1","":"downstairs","session_id":"838557e1-4551-4143-ba02-98927cf0945a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:43.886079802Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1894,"client":"2","":"downstairs","session_id":"838557e1-4551-4143-ba02-98927cf0945a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:43.88609264Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1894,"session_id":"838557e1-4551-4143-ba02-98927cf0945a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:43.886104426Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1894,"session_id":"838557e1-4551-4143-ba02-98927cf0945a"} {"msg":"77eac5aa-f28c-4813-a29b-5841fcd46bbe is now active with session: 838557e1-4551-4143-ba02-98927cf0945a","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:43.886116874Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1894,"session_id":"838557e1-4551-4143-ba02-98927cf0945a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:43.886130862Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1894,"session_id":"838557e1-4551-4143-ba02-98927cf0945a"} {"msg":"The guest has finished waiting for activation with:178","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:43.886142919Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1894} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 13 len 32768 data: 17 14 13 14 13 12 11 9 02/30 Read block 368 len 16384 03/30 Write block 282 len 40960 data: 16 12 13 12 14 13 15 12 13 15 04/30 Read block 176 len 16384 05/30 Write block 24 len 16384 data: 9 11 15 16 06/30 Write block 176 len 28672 data: 14 14 13 9 12 12 12 07/30 Read block 205 len 4096 08/30 Read block 80 len 4096 09/30 Write block 569 len 24576 data: 16 16 15 13 14 15 10/30 Flush 11/30 Write block 369 len 24576 data: 8 13 13 17 18 19 12/30 Read block 440 len 12288 13/30 Read block 224 len 32768 14/30 Write block 436 len 28672 data: 11 11 10 8 7 9 7 15/30 Write block 362 len 28672 data: 12 13 11 9 8 8 8 16/30 Read block 222 len 24576 17/30 Write block 17 len 40960 data: 14 13 12 10 11 11 9 10 12 16 18/30 Write block 87 len 4096 data: 12 19/30 Read block 6 len 4096 20/30 Read block 234 len 16384 21/30 Read block 273 len 20480 22/30 Read block 448 len 32768 23/30 Read block 318 len 24576 24/30 Write block 151 len 24576 data: 14 13 12 13 14 12 25/30 Write block 57 len 20480 data: 16 16 16 13 13 26/30 Write block 320 len 4096 data: 14 27/30 Write block 94 len 32768 data: 18 17 17 12 15 14 14 15 28/30 Read block 194 len 28672 29/30 Write block 515 len 28672 data: 26 26 21 19 18 15 14 30/30 Write block 53 len 20480 data: 18 18 16 17 17 ---------------------------------------------------------------- Crucible gen:178 GIO:true work queues: Upstairs:0 downstairs:20 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1010 Acked Write 6 Done Done Done false 1011 Acked Read 3 Done Done Done false 1012 Acked Read 8 Done Done Done false 1013 Acked Write 7 Done Done Done false 1014 Acked Write 7 Done Done Done false 1015 Acked Read 6 Done Done Done false 1016 Acked Write 10 Done Done Done false 1017 Acked Write 1 Done Done Done false 1018 Acked Read 1 Done Done Done false 1019 Acked Read 4 Done Done Done false 1020 Acked Read 5 Done Done Done false 1021 Acked Read 8 Done Done Done false 1022 Acked Read 6 Done Done Done false 1023 Acked Write 6 Done Done Done false 1024 Acked Write 5 Done Done Done false 1025 Acked Write 1 Done Done Done false 1026 Acked Write 8 Done Done Done false 1027 Acked Read 7 Done Done Done false 1028 Acked Write 7 Sent Sent Sent false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 18 18 18 54 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1009 1009 1009 Downstairs last five completed: 1009 1008 1007 1006 1005 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:18:43.906342795Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1894} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 178 178 178 321 321 321 T T T 1 020-039 178 178 178 321 321 321 T T T 2 040-059 176 176 176 318 318 318 T T T 3 060-079 169 169 169 306 306 306 T T T 4 080-099 174 174 174 313 313 313 T T T 5 100-119 158 158 158 290 290 290 T T T 6 120-139 171 171 171 309 309 309 F F F 7 140-159 172 172 172 310 310 310 T T T 8 160-179 178 178 178 321 321 321 F F F 9 180-199 178 178 178 321 321 321 F F F 10 200-219 176 176 176 318 318 318 F F F 11 220-239 176 176 176 318 318 318 F F F 12 240-259 176 176 176 318 318 318 F F F 13 260-279 177 177 177 320 320 320 F F F 14 280-299 178 178 178 321 321 321 F F F 15 300-319 174 174 174 314 314 314 F F F 16 320-339 169 169 169 306 306 306 T T T 17 340-359 175 175 175 315 315 315 F F F 18 360-379 177 177 177 320 320 320 T T T 19 380-399 177 177 177 320 320 320 F F F 20 400-419 176 176 176 317 317 317 F F F 21 420-439 176 176 176 318 318 318 T T T 22 440-459 174 174 174 313 313 313 T T T 23 460-479 174 174 174 313 313 313 F F F 24 480-499 176 176 176 319 319 319 F F F 25 500-519 173 173 173 311 311 311 T T T 26 520-539 173 173 173 311 311 311 T T T 27 540-559 172 172 172 310 310 310 F F F 28 560-579 178 178 178 321 321 321 F F F 29 580-599 170 170 170 308 308 308 F F F Max gen: 178, Max flush: 321 A Difference in extent metadata is expected here On loop 89 Verifying data now No change after verify Loop: 89 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 179 179 179 322 322 322 F F F 1 020-039 179 179 179 322 322 322 F F F 2 040-059 179 179 179 322 322 322 F F F 3 060-079 179 179 179 322 322 322 F F F 4 080-099 179 179 179 322 322 322 F F F 5 100-119 179 179 179 322 322 322 F F F 6 120-139 171 171 171 309 309 309 F F F 7 140-159 179 179 179 322 322 322 F F F 8 160-179 178 178 178 321 321 321 F F F 9 180-199 178 178 178 321 321 321 F F F 10 200-219 176 176 176 318 318 318 F F F 11 220-239 176 176 176 318 318 318 F F F 12 240-259 176 176 176 318 318 318 F F F 13 260-279 177 177 177 320 320 320 F F F 14 280-299 178 178 178 321 321 321 F F F 15 300-319 174 174 174 314 314 314 F F F 16 320-339 179 179 179 322 322 322 F F F 17 340-359 175 175 175 315 315 315 F F F 18 360-379 179 179 179 322 322 322 F F F 19 380-399 177 177 177 320 320 320 F F F 20 400-419 176 176 176 317 317 317 F F F 21 420-439 179 179 179 322 322 322 F F F 22 440-459 179 179 179 322 322 322 F F F 23 460-479 174 174 174 313 313 313 F F F 24 480-499 176 176 176 319 319 319 F F F 25 500-519 179 179 179 322 322 322 F F F 26 520-539 179 179 179 322 322 322 F F F 27 540-559 172 172 172 310 310 310 F F F 28 560-579 178 178 178 321 321 321 F F F 29 580-599 170 170 170 308 308 308 F F F Max gen: 179, Max flush: 322 Begin loop 90 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:48.223086989Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1902,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:48.267429176Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1902,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:48.268232185Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1902} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"abd5430f1508af0bc008143ea2566277ef714ec2\",\n git_commit_timestamp: \"2025-01-17T20:55:03.000000000Z\",\n git_branch: \"alan/tokio-to-140\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:48.268274399Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1902} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:48.268288528Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1902} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:48.268303647Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1902} {"msg":"Crucible 0220c586-1925-4388-b977-a024815fd0dc has session id: 5681e198-ab0c-4a4a-b3e9-de28a5aa8698","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:48.268934945Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1902,"session_id":"5681e198-ab0c-4a4a-b3e9-de28a5aa8698"} {"msg":"Upstairs opts: Upstairs UUID: 0220c586-1925-4388-b977-a024815fd0dc, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:48.268974529Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1902,"session_id":"5681e198-ab0c-4a4a-b3e9-de28a5aa8698"} {"msg":"Crucible stats registered with UUID: 0220c586-1925-4388-b977-a024815fd0dc","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:48.269024068Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1902,"session_id":"5681e198-ab0c-4a4a-b3e9-de28a5aa8698"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:18:48.269048602Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1902,"session_id":"5681e198-ab0c-4a4a-b3e9-de28a5aa8698"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:48.269508151Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1902,"task":"crutest"} {"msg":"The guest has requested activation with gen:180","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:48.269533155Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1902} {"msg":"0220c586-1925-4388-b977-a024815fd0dc active request set","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:48.269595511Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1902,"session_id":"5681e198-ab0c-4a4a-b3e9-de28a5aa8698"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:48.269612481Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1902,"client":"0","":"downstairs","session_id":"5681e198-ab0c-4a4a-b3e9-de28a5aa8698"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:48.26964445Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1902,"client":"0","":"downstairs","session_id":"5681e198-ab0c-4a4a-b3e9-de28a5aa8698"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:48.269670005Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1902,"client":"1","":"downstairs","session_id":"5681e198-ab0c-4a4a-b3e9-de28a5aa8698"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:48.269774136Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1902,"client":"1","":"downstairs","session_id":"5681e198-ab0c-4a4a-b3e9-de28a5aa8698"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:48.269805685Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1902,"client":"2","":"downstairs","session_id":"5681e198-ab0c-4a4a-b3e9-de28a5aa8698"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:48.269834301Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1902,"client":"2","":"downstairs","session_id":"5681e198-ab0c-4a4a-b3e9-de28a5aa8698"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:48.269859486Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1902,"":"io task","client":"2","":"downstairs","session_id":"5681e198-ab0c-4a4a-b3e9-de28a5aa8698"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:48.269882939Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1902,"":"io task","client":"0","":"downstairs","session_id":"5681e198-ab0c-4a4a-b3e9-de28a5aa8698"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:48.269933639Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1902,"":"io task","client":"1","":"downstairs","session_id":"5681e198-ab0c-4a4a-b3e9-de28a5aa8698"} {"msg":"ds_connection connected from Ok(127.0.0.1:64850)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:48.270049547Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1902,"":"io task","client":"2","":"downstairs","session_id":"5681e198-ab0c-4a4a-b3e9-de28a5aa8698"} {"msg":"ds_connection connected from Ok(127.0.0.1:59696)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:48.270076543Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1902,"":"io task","client":"0","":"downstairs","session_id":"5681e198-ab0c-4a4a-b3e9-de28a5aa8698"} {"msg":"ds_connection connected from Ok(127.0.0.1:43775)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:48.270100517Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1902,"":"io task","client":"1","":"downstairs","session_id":"5681e198-ab0c-4a4a-b3e9-de28a5aa8698"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:48.270526386Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1902,"client":"0","":"downstairs","session_id":"5681e198-ab0c-4a4a-b3e9-de28a5aa8698"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:48.270565258Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1902,"client":"1","":"downstairs","session_id":"5681e198-ab0c-4a4a-b3e9-de28a5aa8698"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:48.270768457Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1902,"client":"2","":"downstairs","session_id":"5681e198-ab0c-4a4a-b3e9-de28a5aa8698"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:48.271250389Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1902,"client":"1","":"downstairs","session_id":"5681e198-ab0c-4a4a-b3e9-de28a5aa8698"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:48.271278075Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1902,"client":"0","":"downstairs","session_id":"5681e198-ab0c-4a4a-b3e9-de28a5aa8698"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:48.271293074Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1902,"client":"2","":"downstairs","session_id":"5681e198-ab0c-4a4a-b3e9-de28a5aa8698"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:48.271741366Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1902,"session_id":"5681e198-ab0c-4a4a-b3e9-de28a5aa8698"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:48.271828917Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1902,"session_id":"5681e198-ab0c-4a4a-b3e9-de28a5aa8698"} {"msg":"[0]R flush_numbers[0..12]: [322, 322, 322, 322, 322, 322, 309, 322, 321, 321, 318, 318]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:48.271901229Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1902,"":"downstairs","":"downstairs","session_id":"5681e198-ab0c-4a4a-b3e9-de28a5aa8698"} {"msg":"[0]R generation[0..12]: [179, 179, 179, 179, 179, 179, 171, 179, 178, 178, 176, 176]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:48.271921391Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1902,"":"downstairs","":"downstairs","session_id":"5681e198-ab0c-4a4a-b3e9-de28a5aa8698"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:48.27193614Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1902,"":"downstairs","":"downstairs","session_id":"5681e198-ab0c-4a4a-b3e9-de28a5aa8698"} {"msg":"[1]R flush_numbers[0..12]: [322, 322, 322, 322, 322, 322, 309, 322, 321, 321, 318, 318]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:48.271950318Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1902,"":"downstairs","":"downstairs","session_id":"5681e198-ab0c-4a4a-b3e9-de28a5aa8698"} {"msg":"[1]R generation[0..12]: [179, 179, 179, 179, 179, 179, 171, 179, 178, 178, 176, 176]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:48.271963736Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1902,"":"downstairs","":"downstairs","session_id":"5681e198-ab0c-4a4a-b3e9-de28a5aa8698"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:48.271977334Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1902,"":"downstairs","":"downstairs","session_id":"5681e198-ab0c-4a4a-b3e9-de28a5aa8698"} {"msg":"[2]R flush_numbers[0..12]: [322, 322, 322, 322, 322, 322, 309, 322, 321, 321, 318, 318]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:48.271990731Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1902,"":"downstairs","":"downstairs","session_id":"5681e198-ab0c-4a4a-b3e9-de28a5aa8698"} {"msg":"[2]R generation[0..12]: [179, 179, 179, 179, 179, 179, 171, 179, 178, 178, 176, 176]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:48.27200441Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1902,"":"downstairs","":"downstairs","session_id":"5681e198-ab0c-4a4a-b3e9-de28a5aa8698"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:48.272018378Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1902,"":"downstairs","":"downstairs","session_id":"5681e198-ab0c-4a4a-b3e9-de28a5aa8698"} {"msg":"Max found gen is 180","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:48.272030865Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1902,"":"downstairs","":"downstairs","session_id":"5681e198-ab0c-4a4a-b3e9-de28a5aa8698"} {"msg":"Generation requested: 180 >= found:180","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:48.272043822Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1902,"":"downstairs","":"downstairs","session_id":"5681e198-ab0c-4a4a-b3e9-de28a5aa8698"} {"msg":"Next flush: 323","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:48.27205636Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1902,"":"downstairs","":"downstairs","session_id":"5681e198-ab0c-4a4a-b3e9-de28a5aa8698"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:48.272072169Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1902,"":"downstairs","":"downstairs","session_id":"5681e198-ab0c-4a4a-b3e9-de28a5aa8698"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO 2025-01-17T21:18:48.272085587Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1902,"session_id":"5681e198-ab0c-4a4a-b3e9-de28a5aa8698"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:48.272103878Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1902,"client":"0","":"downstairs","session_id":"5681e198-ab0c-4a4a-b3e9-de28a5aa8698"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:48.272116965Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1902,"client":"1","":"downstairs","session_id":"5681e198-ab0c-4a4a-b3e9-de28a5aa8698"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:48.272129783Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1902,"client":"2","":"downstairs","session_id":"5681e198-ab0c-4a4a-b3e9-de28a5aa8698"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:48.27214254Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1902,"session_id":"5681e198-ab0c-4a4a-b3e9-de28a5aa8698"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:48.272154577Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1902,"session_id":"5681e198-ab0c-4a4a-b3e9-de28a5aa8698"} {"msg":"0220c586-1925-4388-b977-a024815fd0dc is now active with session: 5681e198-ab0c-4a4a-b3e9-de28a5aa8698","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:48.272166894Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1902,"session_id":"5681e198-ab0c-4a4a-b3e9-de28a5aa8698"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:48.272178391Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1902,"session_id":"5681e198-ab0c-4a4a-b3e9-de28a5aa8698"} {"msg":"The guest has finished waiting for activation with:180","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:18:48.272190508Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1902} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 171 len 32768 data: 13 11 13 13 15 15 15 14 02/30 Write block 6 len 24576 data: 14 15 15 14 16 17 03/30 Write block 8 len 8192 data: 16 15 04/30 Flush 05/30 Write block 300 len 24576 data: 10 12 10 14 14 16 06/30 Write block 576 len 36864 data: 15 15 13 13 13 14 12 12 13 07/30 Write block 337 len 20480 data: 18 19 19 18 18 08/30 Write block 53 len 20480 data: 19 19 17 18 18 09/30 Read block 400 len 40960 10/30 Read block 80 len 8192 11/30 Read block 474 len 40960 12/30 Write block 242 len 12288 data: 14 13 14 13/30 Read block 247 len 16384 14/30 Read block 79 len 12288 15/30 Read block 39 len 24576 16/30 Write block 454 len 12288 data: 13 14 14 17/30 Write block 160 len 20480 data: 14 14 14 11 14 18/30 Write block 396 len 36864 data: 14 15 15 13 12 12 11 11 10 19/30 Read block 460 len 28672 20/30 Read block 342 len 40960 21/30 Write block 300 len 28672 data: 11 13 11 15 15 17 14 22/30 Read block 284 len 20480 23/30 Write block 408 len 16384 data: 7 8 8 7 24/30 Read block 535 len 40960 25/30 Read block 577 len 4096 26/30 Write block 3 len 8192 data: 10 13 27/30 Write block 582 len 32768 data: 13 13 14 12 12 11 11 10 28/30 Write block 544 len 28672 data: 10 10 12 12 13 12 13 29/30 Write block 316 len 20480 data: 14 13 14 14 15 30/30 Read block 275 len 24576 ---------------------------------------------------------------- Crucible gen:180 GIO:true work queues: Upstairs:0 downstairs:26 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1004 Acked Write 6 Done Done Done false 1005 Acked Write 9 Done Done Done false 1006 Acked Write 5 Done Done Done false 1007 Acked Write 5 Done Done Done false 1008 Acked Read 10 Done Done Done false 1009 Acked Read 2 Done Done Done false 1010 Acked Read 10 Done Done Done false 1011 Acked Write 3 Done Done Done false 1012 Acked Read 4 Done Done Done false 1013 Acked Read 3 Done Done Done false 1014 Acked Read 6 Done Done Done false 1015 Acked Write 3 Done Done Done false 1016 Acked Write 5 Done Done Done false 1017 Acked Write 9 Done Done Done false 1018 Acked Read 7 Done Done Done false 1019 Acked Read 10 Done Done Done false 1020 Acked Write 7 Done Done Done false 1021 Acked Read 5 Done Done Done false 1022 Acked Write 4 Done Done Done false 1023 Acked Read 10 Done Done Done false 1024 Acked Read 1 Done Done Done false 1025 Acked Write 2 Done Done Done false 1026 Acked Write 8 Done Done Done false 1027 Acked Write 7 Done Done Done false 1028 Acked Write 5 Done Done Sent false 1029 Acked Read 6 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 2 3 Done 26 25 24 75 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1003 1003 1003 Downstairs last five completed: 1003 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:18:48.291336178Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1902} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 180 180 180 323 323 323 T T T 1 020-039 179 179 179 322 322 322 F F F 2 040-059 179 179 179 322 322 322 T T T 3 060-079 179 179 179 322 322 322 F F F 4 080-099 179 179 179 322 322 322 F F F 5 100-119 179 179 179 322 322 322 F F F 6 120-139 171 171 171 309 309 309 F F F 7 140-159 179 179 179 322 322 322 F F F 8 160-179 180 180 180 323 323 323 T T T 9 180-199 178 178 178 321 321 321 F F F 10 200-219 176 176 176 318 318 318 F F F 11 220-239 176 176 176 318 318 318 F F F 12 240-259 176 176 176 318 318 318 T T T 13 260-279 177 177 177 320 320 320 F F F 14 280-299 178 178 178 321 321 321 F F F 15 300-319 174 174 174 314 314 314 T T T 16 320-339 179 179 179 322 322 322 T T T 17 340-359 175 175 175 315 315 315 T T T 18 360-379 179 179 179 322 322 322 F F F 19 380-399 177 177 177 320 320 320 T T T 20 400-419 176 176 176 317 317 317 T T T 21 420-439 179 179 179 322 322 322 F F F 22 440-459 179 179 179 322 322 322 T T T 23 460-479 174 174 174 313 313 313 F F F 24 480-499 176 176 176 319 319 319 F F F 25 500-519 179 179 179 322 322 322 F F F 26 520-539 179 179 179 322 322 322 F F F 27 540-559 172 172 172 310 310 310 T T T 28 560-579 178 178 178 321 321 321 T T T 29 580-599 170 170 170 308 308 308 T T T Max gen: 180, Max flush: 323 A Difference in extent metadata is expected here On loop 90 Verifying data now No change after verify Loop: 90 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 181 181 181 324 324 324 F F F 1 020-039 179 179 179 322 322 322 F F F 2 040-059 181 181 181 324 324 324 F F F 3 060-079 179 179 179 322 322 322 F F F 4 080-099 179 179 179 322 322 322 F F F 5 100-119 179 179 179 322 322 322 F F F 6 120-139 171 171 171 309 309 309 F F F 7 140-159 179 179 179 322 322 322 F F F 8 160-179 181 181 181 324 324 324 F F F 9 180-199 178 178 178 321 321 321 F F F 10 200-219 176 176 176 318 318 318 F F F 11 220-239 176 176 176 318 318 318 F F F 12 240-259 181 181 181 324 324 324 F F F 13 260-279 177 177 177 320 320 320 F F F 14 280-299 178 178 178 321 321 321 F F F 15 300-319 181 181 181 324 324 324 F F F 16 320-339 181 181 181 324 324 324 F F F 17 340-359 181 181 181 324 324 324 F F F 18 360-379 179 179 179 322 322 322 F F F 19 380-399 181 181 181 324 324 324 F F F 20 400-419 181 181 181 324 324 324 F F F 21 420-439 179 179 179 322 322 322 F F F 22 440-459 181 181 181 324 324 324 F F F 23 460-479 174 174 174 313 313 313 F F F 24 480-499 176 176 176 319 319 319 F F F 25 500-519 179 179 179 322 322 322 F F F 26 520-539 179 179 179 322 322 322 F F F 27 540-559 181 181 181 324 324 324 F F F 28 560-579 181 181 181 324 324 324 F F F 29 580-599 181 181 181 324 324 324 F F F Max gen: 181, Max flush: 324 Begin loop 91 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:02.57462493Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1910,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:02.618227015Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1910,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:02.618862526Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1910} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"abd5430f1508af0bc008143ea2566277ef714ec2\",\n git_commit_timestamp: \"2025-01-17T20:55:03.000000000Z\",\n git_branch: \"alan/tokio-to-140\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:02.6189025Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1910} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:02.618917508Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1910} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:02.618932657Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1910} {"msg":"Crucible 190a40f3-91bf-4f0e-b1d4-bd772faf92bd has session id: eef316b1-639b-4d67-b583-42190858c5e0","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:02.61950145Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1910,"session_id":"eef316b1-639b-4d67-b583-42190858c5e0"} {"msg":"Upstairs opts: Upstairs UUID: 190a40f3-91bf-4f0e-b1d4-bd772faf92bd, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:02.619545345Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1910,"session_id":"eef316b1-639b-4d67-b583-42190858c5e0"} {"msg":"Crucible stats registered with UUID: 190a40f3-91bf-4f0e-b1d4-bd772faf92bd","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:02.619568709Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1910,"session_id":"eef316b1-639b-4d67-b583-42190858c5e0"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:19:02.619588901Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1910,"session_id":"eef316b1-639b-4d67-b583-42190858c5e0"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:02.620041605Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1910,"task":"crutest"} {"msg":"The guest has requested activation with gen:182","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:02.62006608Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1910} {"msg":"190a40f3-91bf-4f0e-b1d4-bd772faf92bd active request set","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:02.620122582Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1910,"session_id":"eef316b1-639b-4d67-b583-42190858c5e0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:02.620148538Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1910,"client":"0","":"downstairs","session_id":"eef316b1-639b-4d67-b583-42190858c5e0"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:02.620175173Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1910,"client":"0","":"downstairs","session_id":"eef316b1-639b-4d67-b583-42190858c5e0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:02.620189422Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1910,"client":"1","":"downstairs","session_id":"eef316b1-639b-4d67-b583-42190858c5e0"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:02.620212875Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1910,"client":"1","":"downstairs","session_id":"eef316b1-639b-4d67-b583-42190858c5e0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:02.62023763Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1910,"client":"2","":"downstairs","session_id":"eef316b1-639b-4d67-b583-42190858c5e0"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:02.620258542Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1910,"client":"2","":"downstairs","session_id":"eef316b1-639b-4d67-b583-42190858c5e0"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:02.620281866Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1910,"":"io task","client":"0","":"downstairs","session_id":"eef316b1-639b-4d67-b583-42190858c5e0"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:02.620307681Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1910,"":"io task","client":"1","":"downstairs","session_id":"eef316b1-639b-4d67-b583-42190858c5e0"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:02.620332585Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1910,"":"io task","client":"2","":"downstairs","session_id":"eef316b1-639b-4d67-b583-42190858c5e0"} {"msg":"ds_connection connected from Ok(127.0.0.1:55860)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:02.620361152Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1910,"":"io task","client":"0","":"downstairs","session_id":"eef316b1-639b-4d67-b583-42190858c5e0"} {"msg":"ds_connection connected from Ok(127.0.0.1:41197)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:02.620552714Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1910,"":"io task","client":"1","":"downstairs","session_id":"eef316b1-639b-4d67-b583-42190858c5e0"} {"msg":"ds_connection connected from Ok(127.0.0.1:52680)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:02.620657135Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1910,"":"io task","client":"2","":"downstairs","session_id":"eef316b1-639b-4d67-b583-42190858c5e0"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:02.621045152Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1910,"client":"0","":"downstairs","session_id":"eef316b1-639b-4d67-b583-42190858c5e0"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:02.621200443Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1910,"client":"1","":"downstairs","session_id":"eef316b1-639b-4d67-b583-42190858c5e0"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:02.621588539Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1910,"client":"0","":"downstairs","session_id":"eef316b1-639b-4d67-b583-42190858c5e0"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:02.621740668Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1910,"client":"1","":"downstairs","session_id":"eef316b1-639b-4d67-b583-42190858c5e0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:02.62215466Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1910,"session_id":"eef316b1-639b-4d67-b583-42190858c5e0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:02.622304978Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1910,"session_id":"eef316b1-639b-4d67-b583-42190858c5e0"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:02.622396542Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1910,"client":"2","":"downstairs","session_id":"eef316b1-639b-4d67-b583-42190858c5e0"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:02.622919097Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1910,"client":"2","":"downstairs","session_id":"eef316b1-639b-4d67-b583-42190858c5e0"} {"msg":"[0]R flush_numbers[0..12]: [324, 322, 324, 322, 322, 322, 309, 322, 324, 321, 318, 318]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:02.623395595Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1910,"":"downstairs","":"downstairs","session_id":"eef316b1-639b-4d67-b583-42190858c5e0"} {"msg":"[0]R generation[0..12]: [181, 179, 181, 179, 179, 179, 171, 179, 181, 178, 176, 176]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:02.62341972Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1910,"":"downstairs","":"downstairs","session_id":"eef316b1-639b-4d67-b583-42190858c5e0"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:02.623435319Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1910,"":"downstairs","":"downstairs","session_id":"eef316b1-639b-4d67-b583-42190858c5e0"} {"msg":"[1]R flush_numbers[0..12]: [324, 322, 324, 322, 322, 322, 309, 322, 324, 321, 318, 318]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:02.623452629Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1910,"":"downstairs","":"downstairs","session_id":"eef316b1-639b-4d67-b583-42190858c5e0"} {"msg":"[1]R generation[0..12]: [181, 179, 181, 179, 179, 179, 171, 179, 181, 178, 176, 176]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:02.623466647Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1910,"":"downstairs","":"downstairs","session_id":"eef316b1-639b-4d67-b583-42190858c5e0"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:02.623480685Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1910,"":"downstairs","":"downstairs","session_id":"eef316b1-639b-4d67-b583-42190858c5e0"} {"msg":"[2]R flush_numbers[0..12]: [324, 322, 324, 322, 322, 322, 309, 322, 324, 321, 318, 318]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:02.623494203Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1910,"":"downstairs","":"downstairs","session_id":"eef316b1-639b-4d67-b583-42190858c5e0"} {"msg":"[2]R generation[0..12]: [181, 179, 181, 179, 179, 179, 171, 179, 181, 178, 176, 176]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:02.623507491Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1910,"":"downstairs","":"downstairs","session_id":"eef316b1-639b-4d67-b583-42190858c5e0"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:02.623521199Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1910,"":"downstairs","":"downstairs","session_id":"eef316b1-639b-4d67-b583-42190858c5e0"} {"msg":"Max found gen is 182","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:02.623533806Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1910,"":"downstairs","":"downstairs","session_id":"eef316b1-639b-4d67-b583-42190858c5e0"} {"msg":"Generation requested: 182 >= found:182","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:02.623546584Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1910,"":"downstairs","":"downstairs","session_id":"eef316b1-639b-4d67-b583-42190858c5e0"} {"msg":"Next flush: 325","v":0,"name":"crucible","Wait for a query_work_queue command to finish before sending IO level":30,"time":"2025-01-17T21:19:02.623565475Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1910,"":"downstairs","":"downstairs","session_id":"eef316b1-639b-4d67-b583-42190858c5e0"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:02.623578262Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1910,"":"downstairs","":"downstairs","session_id":"eef316b1-639b-4d67-b583-42190858c5e0"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:02.62359099Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1910,"session_id":"eef316b1-639b-4d67-b583-42190858c5e0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:02.623602647Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1910,"client":"0","":"downstairs","session_id":"eef316b1-639b-4d67-b583-42190858c5e0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:02.623615824Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1910,"client":"1","":"downstairs","session_id":"eef316b1-639b-4d67-b583-42190858c5e0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:02.623628812Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1910,"client":"2","":"downstairs","session_id":"eef316b1-639b-4d67-b583-42190858c5e0"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:02.623641979Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1910,"session_id":"eef316b1-639b-4d67-b583-42190858c5e0"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:02.623656048Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1910,"session_id":"eef316b1-639b-4d67-b583-42190858c5e0"} {"msg":"190a40f3-91bf-4f0e-b1d4-bd772faf92bd is now active with session: eef316b1-639b-4d67-b583-42190858c5e0","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:02.623668585Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1910,"session_id":"eef316b1-639b-4d67-b583-42190858c5e0"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:02.623680122Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1910,"session_id":"eef316b1-639b-4d67-b583-42190858c5e0"} {"msg":"The guest has finished waiting for activation with:182","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:02.623691999Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1910} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 0 len 20480 data: 5 8 11 11 14 02/30 Read block 573 len 16384 03/30 Flush 04/30 Write block 184 len 32768 data: 7 6 5 7 11 13 14 14 05/30 Read block 109 len 8192 06/30 Flush 07/30 Write block 21 len 28672 data: 12 12 10 11 13 17 17 08/30 Read block 489 len 32768 09/30 Read block 147 len 4096 10/30 Read block 475 len 20480 11/30 Write block 29 len 40960 data: 13 16 14 17 19 19 22 17 17 16 12/30 Write block 123 len 24576 data: 10 8 11 8 10 11 13/30 Read block 63 len 4096 14/30 Read block 547 len 36864 15/30 Write block 272 len 4096 data: 11 16/30 Read block 164 len 20480 17/30 Write block 203 len 16384 data: 13 12 12 11 18/30 Write block 192 len 16384 data: 16 17 17 16 19/30 Read block 214 len 40960 20/30 Flush 21/30 Write block 163 len 4096 data: 12 22/30 Read block 595 len 4096 23/30 Read block 565 len 8192 24/30 Write block 164 len 36864 data: 15 15 15 15 16 13 12 14 12 25/30 Write block 350 len 40960 data: 14 11 13 12 13 14 17 18 17 17 26/30 Write block 1 len 32768 data: 9 12 12 15 15 15 16 17 27/30 Read block 307 len 36864 28/30 Read block 210 len 4096 29/30 Write block 348 len 12288 data: 14 12 15 30/30 Read block 508 len 4096 ---------------------------------------------------------------- Crucible gen:182 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1020 Acked Write 1 Done Done Done false 1021 Acked Read 1 Done Done Done false 1022 Acked Read 2 Done Done Done false 1023 Acked Write 9 Done Done Done false 1024 Acked Write 10 Done Done Done false 1025 Acked Write 8 Done Done Done false 1026 Acked Read 9 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Write 3 Done Sent Sent false 1029 Acked Read 1 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 2 2 4 Done 10 8 8 26 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1019 1019 1019 Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:19:02.642832216Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1910} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 182 182 182 325 325 325 T T T 1 020-039 182 182 182 327 327 327 F F F 2 040-059 181 181 181 324 324 324 F F F 3 060-079 179 179 179 322 322 322 F F F 4 080-099 179 179 179 322 322 322 F F F 5 100-119 179 179 179 322 322 322 F F F 6 120-139 182 182 182 327 327 327 F F F 7 140-159 179 179 179 322 322 322 F F F 8 160-179 181 181 181 324 324 324 T T T 9 180-199 182 182 182 327 327 327 F F F 10 200-219 182 182 182 327 327 327 F F F 11 220-239 176 176 176 318 318 318 F F F 12 240-259 181 181 181 324 324 324 F F F 13 260-279 182 182 182 327 327 327 F F F 14 280-299 178 178 178 321 321 321 F F F 15 300-319 181 181 181 324 324 324 F F F 16 320-339 181 181 181 324 324 324 F F F 17 340-359 181 181 181 324 324 324 T T T 18 360-379 179 179 179 322 322 322 F F F 19 380-399 181 181 181 324 324 324 F F F 20 400-419 181 181 181 324 324 324 F F F 21 420-439 179 179 179 322 322 322 F F F 22 440-459 181 181 181 324 324 324 F F F 23 460-479 174 174 174 313 313 313 F F F 24 480-499 176 176 176 319 319 319 F F F 25 500-519 179 179 179 322 322 322 F F F 26 520-539 179 179 179 322 322 322 F F F 27 540-559 181 181 181 324 324 324 F F F 28 560-579 181 181 181 324 324 324 F F F 29 580-599 181 181 181 324 324 324 F F F Max gen: 182, Max flush: 327 A Difference in extent metadata is expected here On loop 91 Verifying data now No change after verify Loop: 91 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 183 183 183 328 328 328 F F F 1 020-039 182 182 182 327 327 327 F F F 2 040-059 181 181 181 324 324 324 F F F 3 060-079 179 179 179 322 322 322 F F F 4 080-099 179 179 179 322 322 322 F F F 5 100-119 179 179 179 322 322 322 F F F 6 120-139 182 182 182 327 327 327 F F F 7 140-159 179 179 179 322 322 322 F F F 8 160-179 183 183 183 328 328 328 F F F 9 180-199 182 182 182 327 327 327 F F F 10 200-219 182 182 182 327 327 327 F F F 11 220-239 176 176 176 318 318 318 F F F 12 240-259 181 181 181 324 324 324 F F F 13 260-279 182 182 182 327 327 327 F F F 14 280-299 178 178 178 321 321 321 F F F 15 300-319 181 181 181 324 324 324 F F F 16 320-339 181 181 181 324 324 324 F F F 17 340-359 183 183 183 328 328 328 F F F 18 360-379 179 179 179 322 322 322 F F F 19 380-399 181 181 181 324 324 324 F F F 20 400-419 181 181 181 324 324 324 F F F 21 420-439 179 179 179 322 322 322 F F F 22 440-459 181 181 181 324 324 324 F F F 23 460-479 174 174 174 313 313 313 F F F 24 480-499 176 176 176 319 319 319 F F F 25 500-519 179 179 179 322 322 322 F F F 26 520-539 179 179 179 322 322 322 F F F 27 540-559 181 181 181 324 324 324 F F F 28 560-579 181 181 181 324 324 324 F F F 29 580-599 181 181 181 324 324 324 F F F Max gen: 183, Max flush: 328 Begin loop 92 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:06.872017724Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1918,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:19:06.914693347Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1918,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:06.915119205Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1918,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:06.946850341Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1918,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:06.94746426Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1918} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"abd5430f1508af0bc008143ea2566277ef714ec2\",\n git_commit_timestamp: \"2025-01-17T20:55:03.000000000Z\",\n git_branch: \"alan/tokio-to-140\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:06.947507255Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1918} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:06.947524685Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1918} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:06.947542496Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1918} {"msg":"Crucible 50b3f64c-9a93-40ac-8965-28a78de80d25 has session id: fc80c29a-1d39-498a-a654-b85d547112de","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:06.948105005Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1918,"session_id":"fc80c29a-1d39-498a-a654-b85d547112de"} {"msg":"Upstairs opts: Upstairs UUID: 50b3f64c-9a93-40ac-8965-28a78de80d25, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:06.948139155Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1918,"session_id":"fc80c29a-1d39-498a-a654-b85d547112de"} {"msg":"Crucible stats registered with UUID: 50b3f64c-9a93-40ac-8965-28a78de80d25","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:06.948157405Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1918,"session_id":"fc80c29a-1d39-498a-a654-b85d547112de"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:19:06.948173095Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1918,"session_id":"fc80c29a-1d39-498a-a654-b85d547112de"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:06.948545752Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1918,"task":"crutest"} {"msg":"The guest has requested activation with gen:184","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:06.948569006Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1918} {"msg":"50b3f64c-9a93-40ac-8965-28a78de80d25 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:06.94862791Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1918,"session_id":"fc80c29a-1d39-498a-a654-b85d547112de"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:06.948654016Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1918,"client":"0","":"downstairs","session_id":"fc80c29a-1d39-498a-a654-b85d547112de"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:06.948680981Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1918,"client":"0","":"downstairs","session_id":"fc80c29a-1d39-498a-a654-b85d547112de"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:06.948699162Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1918,"client":"1","":"downstairs","session_id":"fc80c29a-1d39-498a-a654-b85d547112de"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:06.948713831Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1918,"client":"1","":"downstairs","session_id":"fc80c29a-1d39-498a-a654-b85d547112de"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:06.948726548Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1918,"client":"2","":"downstairs","session_id":"fc80c29a-1d39-498a-a654-b85d547112de"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:06.948740186Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1918,"client":"2","":"downstairs","session_id":"fc80c29a-1d39-498a-a654-b85d547112de"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:06.948754184Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1918,"":"io task","client":"2","":"downstairs","session_id":"fc80c29a-1d39-498a-a654-b85d547112de"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:06.948775587Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1918,"":"io task","client":"0","":"downstairs","session_id":"fc80c29a-1d39-498a-a654-b85d547112de"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:06.948789925Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1918,"":"io task","client":"1","":"downstairs","session_id":"fc80c29a-1d39-498a-a654-b85d547112de"} {"msg":"ds_connection connected from Ok(127.0.0.1:37819)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:06.948921292Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1918,"":"io task","client":"2","":"downstairs","session_id":"fc80c29a-1d39-498a-a654-b85d547112de"} {"msg":"ds_connection connected from Ok(127.0.0.1:34267)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:06.948961896Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1918,"":"io task","client":"1","":"downstairs","session_id":"fc80c29a-1d39-498a-a654-b85d547112de"} {"msg":"ds_connection connected from Ok(127.0.0.1:58072)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:06.949153517Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1918,"":"io task","client":"0","":"downstairs","session_id":"fc80c29a-1d39-498a-a654-b85d547112de"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:06.949634539Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1918,"client":"1","":"downstairs","session_id":"fc80c29a-1d39-498a-a654-b85d547112de"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:06.94967104Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1918,"client":"2","":"downstairs","session_id":"fc80c29a-1d39-498a-a654-b85d547112de"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:06.949871057Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1918,"client":"0","":"downstairs","session_id":"fc80c29a-1d39-498a-a654-b85d547112de"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:06.950366767Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1918,"client":"1","":"downstairs","session_id":"fc80c29a-1d39-498a-a654-b85d547112de"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:06.950399566Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1918,"client":"2","":"downstairs","session_id":"fc80c29a-1d39-498a-a654-b85d547112de"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:06.950429443Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1918,"client":"0","":"downstairs","session_id":"fc80c29a-1d39-498a-a654-b85d547112de"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:06.950907213Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1918,"session_id":"fc80c29a-1d39-498a-a654-b85d547112de"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:06.951027843Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1918,"session_id":"fc80c29a-1d39-498a-a654-b85d547112de"} {"msg":"[0]R flush_numbers[0..12]: [328, 327, 324, 322, 322, 322, 327, 322, 328, 327, 327, 318]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:06.951099335Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1918,"":"downstairs","":"downstairs","session_id":"fc80c29a-1d39-498a-a654-b85d547112de"} {"msg":"[0]R generation[0..12]: [183, 182, 181, 179, 179, 179, 182, 179, 183, 182, 182, 176]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:06.951118786Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1918,"":"downstairs","":"downstairs","session_id":"fc80c29a-1d39-498a-a654-b85d547112de"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:06.951133154Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1918,"":"downstairs","":"downstairs","session_id":"fc80c29a-1d39-498a-a654-b85d547112de"} {"msg":"[1]R flush_numbers[0..12]: [328, 327, 324, 322, 322, 322, 327, 322, 328, 327, 327, 318]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:06.951146753Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1918,"":"downstairs","":"downstairs","session_id":"fc80c29a-1d39-498a-a654-b85d547112de"} {"msg":"[1]R generation[0..12]: [183, 182, 181, 179, 179, 179, 182, 179, 183, 182, 182, 176]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:06.95116019Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1918,"":"downstairs","":"downstairs","session_id":"fc80c29a-1d39-498a-a654-b85d547112de"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:06.951173788Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1918,"":"downstairs","":"downstairs","session_id":"fc80c29a-1d39-498a-a654-b85d547112de"} {"msg":"[2]R flush_numbers[0..12]: [328, 327, 324, 322, 322, 322, 327, 322, 328, 327, 327, 318]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:06.951187026Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1918,"":"downstairs","":"downstairs","session_id":"fc80c29a-1d39-498a-a654-b85d547112de"} {"msg":"[2]R generation[0..12]: [183, 182, 181, 179, 179, 179, 182, 179, 183, 182, 182, 176]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:06.951200574Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1918,"":"downstairs","":"downstairs","session_id":"fc80c29a-1d39-498a-a654-b85d547112de"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:06.951214462Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1918,"":"downstairs","":"downstairs","session_id":"fc80c29a-1d39-498a-a654-b85d547112de"} {"msg":"Max found gen is 184","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:06.951226949Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1918,"":"downstairs","":"downstairs","session_id":"fc80c29a-1d39-498a-a654-b85d547112de"} {"msg":"Generation requested: 184 >= found:184","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:06.951242909Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1918,"":"downstairs","":"downstairs","session_id":"fc80c29a-1d39-498a-a654-b85d547112de"} {"msg":"Next flush: 329","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:06.951255566Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1918,"":"downstairs","":"downstairs","session_id":"fc80c29a-1d39-498a-a654-b85d547112de"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-17T21:19:06.951268133Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1918,"":"downstairs","":"downstairs","session_id":"fc80c29a-1d39-498a-a654-b85d547112de"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:06.951285374Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1918,"session_id":"fc80c29a-1d39-498a-a654-b85d547112de"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:06.95129706Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1918,"client":"0","":"downstairs","session_id":"fc80c29a-1d39-498a-a654-b85d547112de"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:06.951310098Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1918,"client":"1","":"downstairs","session_id":"fc80c29a-1d39-498a-a654-b85d547112de"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:06.951323706Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1918,"client":"2","":"downstairs","session_id":"fc80c29a-1d39-498a-a654-b85d547112de"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:06.951336573Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1918,"session_id":"fc80c29a-1d39-498a-a654-b85d547112de"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:06.95134844Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1918,"session_id":"fc80c29a-1d39-498a-a654-b85d547112de"} {"msg":"50b3f64c-9a93-40ac-8965-28a78de80d25 is now active with session: fc80c29a-1d39-498a-a654-b85d547112de","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:06.951360848Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1918,"session_id":"fc80c29a-1d39-498a-a654-b85d547112de"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:06.951372404Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1918,"session_id":"fc80c29a-1d39-498a-a654-b85d547112de"} {"msg":"The guest has finished waiting for activation with:184","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:06.951384171Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1918} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 406 len 4096 data: 8 02/30 Read block 90 len 8192 03/30 Read block 570 len 32768 04/30 Write block 450 len 20480 data: 9 9 10 9 14 05/30 Write block 589 len 32768 data: 11 10 7 10 10 10 7 8 06/30 Write block 24 len 12288 data: 12 14 18 07/30 Read block 389 len 12288 08/30 Read block 531 len 12288 09/30 Write block 150 len 28672 data: 14 15 14 13 14 15 13 10/30 Write block 60 len 40960 data: 14 14 10 9 11 11 12 11 15 14 11/30 Read block 161 len 36864 12/30 Write block 485 len 40960 data: 10 11 11 12 12 15 14 11 11 13 13/30 Flush 14/30 Write block 103 len 24576 data: 17 17 16 14 12 12 15/30 Flush 16/30 Write block 284 len 20480 data: 14 13 15 14 16 17/30 Read block 54 len 40960 18/30 Write block 244 len 40960 data: 15 12 12 14 13 14 12 11 13 13 19/30 Write block 204 len 32768 data: 13 13 12 10 12 14 14 12 20/30 Write block 554 len 36864 data: 9 9 9 12 16 15 20 21 21 21/30 Write block 461 len 20480 data: 14 14 12 15 15 22/30 Write block 279 len 36864 data: 10 13 17 17 13 15 14 16 15 23/30 Read block 125 len 28672 24/30 Read block 239 len 20480 25/30 Read block 420 len 28672 26/30 Write block 564 len 36864 data: 15 14 18 12 14 17 17 16 14 27/30 Write block 578 len 24576 data: 14 14 14 15 14 14 28/30 Read block 573 len 16384 29/30 Read block 198 len 32768 30/30 Write block 68 len 20480 data: 16 15 14 16 16 ---------------------------------------------------------------- Crucible gen:184 GIO:true work queues: Upstairs:0 downstairs:15 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1015 Acked Write 5 Done Done Done false 1016 Acked Read 10 Done Done Done false 1017 Acked Write 10 Done Done Done false 1018 Acked Write 8 Done Done Done false 1019 Acked Write 9 Done Done Done false 1020 Acked Write 5 Done Done Done false 1021 Acked Write 9 Done Done Done false 1022 Acked Read 7 Done Done Done false 1023 Acked Read 5 Done Done Done false 1024 Acked Read 7 Done Done Done false 1025 Acked Write 9 Done Done Done false 1026 Acked Write 6 Done Done Done false 1027 Acked Read 4 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 14 14 14 42 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1014 1014 1014 Downstairs last five completed: 1014 1013 1012 1011 1010 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:19:06.972244564Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1918} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 183 183 183 328 328 328 F F F 1 020-039 184 184 184 329 329 329 F F F 2 040-059 181 181 181 324 324 324 F F F 3 060-079 184 184 184 329 329 329 T T T 4 080-099 179 179 179 322 322 322 F F F 5 100-119 184 184 184 330 330 330 F F F 6 120-139 182 182 182 327 327 327 F F F 7 140-159 184 184 184 329 329 329 F F F 8 160-179 183 183 183 328 328 328 F F F 9 180-199 182 182 182 327 327 327 F F F 10 200-219 182 182 182 327 327 327 T T T 11 220-239 176 176 176 318 318 318 F F F 12 240-259 181 181 181 324 324 324 T T T 13 260-279 182 182 182 327 327 327 T T T 14 280-299 178 178 178 321 321 321 T T T 15 300-319 181 181 181 324 324 324 F F F 16 320-339 181 181 181 324 324 324 F F F 17 340-359 183 183 183 328 328 328 F F F 18 360-379 179 179 179 322 322 322 F F F 19 380-399 181 181 181 324 324 324 F F F 20 400-419 184 184 184 329 329 329 F F F 21 420-439 179 179 179 322 322 322 F F F 22 440-459 184 184 184 329 329 329 F F F 23 460-479 174 174 174 313 313 313 T T T 24 480-499 184 184 184 329 329 329 F F F 25 500-519 179 179 179 322 322 322 F F F 26 520-539 179 179 179 322 322 322 F F F 27 540-559 181 181 181 324 324 324 T T T 28 560-579 181 181 181 324 324 324 T T T 29 580-599 184 184 184 329 329 329 T T T Max gen: 184, Max flush: 330 A Difference in extent metadata is expected here On loop 92 Verifying data now No change after verify Loop: 92 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 183 183 183 328 328 328 F F F 1 020-039 184 184 184 329 329 329 F F F 2 040-059 181 181 181 324 324 324 F F F 3 060-079 185 185 185 331 331 331 F F F 4 080-099 179 179 179 322 322 322 F F F 5 100-119 184 184 184 330 330 330 F F F 6 120-139 182 182 182 327 327 327 F F F 7 140-159 184 184 184 329 329 329 F F F 8 160-179 183 183 183 328 328 328 F F F 9 180-199 182 182 182 327 327 327 F F F 10 200-219 185 185 185 331 331 331 F F F 11 220-239 176 176 176 318 318 318 F F F 12 240-259 185 185 185 331 331 331 F F F 13 260-279 185 185 185 331 331 331 F F F 14 280-299 185 185 185 331 331 331 F F F 15 300-319 181 181 181 324 324 324 F F F 16 320-339 181 181 181 324 324 324 F F F 17 340-359 183 183 183 328 328 328 F F F 18 360-379 179 179 179 322 322 322 F F F 19 380-399 181 181 181 324 324 324 F F F 20 400-419 184 184 184 329 329 329 F F F 21 420-439 179 179 179 322 322 322 F F F 22 440-459 184 184 184 329 329 329 F F F 23 460-479 185 185 185 331 331 331 F F F 24 480-499 184 184 184 329 329 329 F F F 25 500-519 179 179 179 322 322 322 F F F 26 520-539 179 179 179 322 322 322 F F F 27 540-559 185 185 185 331 331 331 F F F 28 560-579 185 185 185 331 331 331 F F F 29 580-599 185 185 185 331 331 331 F F F Max gen: 185, Max flush: 331 Begin loop 93 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:11.262890382Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1926,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:11.307028249Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1926,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:11.307675538Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1926} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"abd5430f1508af0bc008143ea2566277ef714ec2\",\n git_commit_timestamp: \"2025-01-17T20:55:03.000000000Z\",\n git_branch: \"alan/tokio-to-140\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:11.307728829Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1926} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:11.30774788Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1926} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:11.307765891Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1926} {"msg":"Crucible 5088cb64-454f-4e14-8674-654fd0707f3b has session id: b80bc3f3-fec2-4bc9-827f-4b82947464d6","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:11.308198493Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1926,"session_id":"b80bc3f3-fec2-4bc9-827f-4b82947464d6"} {"msg":"Upstairs opts: Upstairs UUID: 5088cb64-454f-4e14-8674-654fd0707f3b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:11.308297691Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1926,"session_id":"b80bc3f3-fec2-4bc9-827f-4b82947464d6"} {"msg":"Crucible stats registered with UUID: 5088cb64-454f-4e14-8674-654fd0707f3b","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:11.30833029Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1926,"session_id":"b80bc3f3-fec2-4bc9-827f-4b82947464d6"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:19:11.308351923Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1926,"session_id":"b80bc3f3-fec2-4bc9-827f-4b82947464d6"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:11.308741611Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1926,"task":"crutest"} {"msg":"The guest has requested activation with gen:186","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:11.30877379Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1926} {"msg":"5088cb64-454f-4e14-8674-654fd0707f3b active request set","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:11.308832694Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1926,"session_id":"b80bc3f3-fec2-4bc9-827f-4b82947464d6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:11.308859099Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1926,"client":"0","":"downstairs","session_id":"b80bc3f3-fec2-4bc9-827f-4b82947464d6"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:11.308885835Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1926,"client":"0","":"downstairs","session_id":"b80bc3f3-fec2-4bc9-827f-4b82947464d6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:11.308900944Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1926,"client":"1","":"downstairs","session_id":"b80bc3f3-fec2-4bc9-827f-4b82947464d6"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:11.308914842Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1926,"client":"1","":"downstairs","session_id":"b80bc3f3-fec2-4bc9-827f-4b82947464d6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:11.30892789Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1926,"client":"2","":"downstairs","session_id":"b80bc3f3-fec2-4bc9-827f-4b82947464d6"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:11.308942618Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1926,"client":"2","":"downstairs","session_id":"b80bc3f3-fec2-4bc9-827f-4b82947464d6"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:11.308955536Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1926,"":"io task","client":"2","":"downstairs","session_id":"b80bc3f3-fec2-4bc9-827f-4b82947464d6"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:11.308978069Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1926,"":"io task","client":"0","":"downstairs","session_id":"b80bc3f3-fec2-4bc9-827f-4b82947464d6"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:11.309004665Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1926,"":"io task","client":"1","":"downstairs","session_id":"b80bc3f3-fec2-4bc9-827f-4b82947464d6"} {"msg":"ds_connection connected from Ok(127.0.0.1:62843)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:11.309041276Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1926,"":"io task","client":"2","":"downstairs","session_id":"b80bc3f3-fec2-4bc9-827f-4b82947464d6"} {"msg":"ds_connection connected from Ok(127.0.0.1:54768)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:11.309068782Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1926,"":"io task","client":"0","":"downstairs","session_id":"b80bc3f3-fec2-4bc9-827f-4b82947464d6"} {"msg":"ds_connection connected from Ok(127.0.0.1:43689)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:11.309095718Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1926,"":"io task","client":"1","":"downstairs","session_id":"b80bc3f3-fec2-4bc9-827f-4b82947464d6"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:11.309701032Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1926,"client":"0","":"downstairs","session_id":"b80bc3f3-fec2-4bc9-827f-4b82947464d6"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:11.309737513Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1926,"client":"2","":"downstairs","session_id":"b80bc3f3-fec2-4bc9-827f-4b82947464d6"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:11.310392276Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1926,"client":"2","":"downstairs","session_id":"b80bc3f3-fec2-4bc9-827f-4b82947464d6"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:11.310563746Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1926,"client":"0","":"downstairs","session_id":"b80bc3f3-fec2-4bc9-827f-4b82947464d6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:11.310966081Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1926,"session_id":"b80bc3f3-fec2-4bc9-827f-4b82947464d6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:11.311110635Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1926,"session_id":"b80bc3f3-fec2-4bc9-827f-4b82947464d6"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:11.31118754Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1926,"client":"1","":"downstairs","session_id":"b80bc3f3-fec2-4bc9-827f-4b82947464d6"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:11.311681399Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1926,"client":"1","":"downstairs","session_id":"b80bc3f3-fec2-4bc9-827f-4b82947464d6"} {"msg":"[0]R flush_numbers[0..12]: [328, 329, 324, 331, 322, 330, 327, 329, 328, 327, 331, 318]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:11.31216181Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1926,"":"downstairs","":"downstairs","session_id":"b80bc3f3-fec2-4bc9-827f-4b82947464d6"} {"msg":"[0]R generation[0..12]: [183, 184, 181, 185, 179, 184, 182, 184, 183, 182, 185, 176]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:11.312180471Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1926,"":"downstairs","":"downstairs","session_id":"b80bc3f3-fec2-4bc9-827f-4b82947464d6"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:11.312194509Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1926,"":"downstairs","":"downstairs","session_id":"b80bc3f3-fec2-4bc9-827f-4b82947464d6"} {"msg":"[1]R flush_numbers[0..12]: [328, 329, 324, 331, 322, 330, 327, 329, 328, 327, 331, 318]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:11.312211109Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1926,"":"downstairs","":"downstairs","session_id":"b80bc3f3-fec2-4bc9-827f-4b82947464d6"} {"msg":"[1]R generation[0..12]: [183, 184, 181, 185, 179, 184, 182, 184, 183, 182, 185, 176]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:11.312224837Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1926,"":"downstairs","":"downstairs","session_id":"b80bc3f3-fec2-4bc9-827f-4b82947464d6"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:11.312238334Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1926,"":"downstairs","":"downstairs","session_id":"b80bc3f3-fec2-4bc9-827f-4b82947464d6"} {"msg":"[2]R flush_numbers[0..12]: [328, 329, 324, 331, 322, 330, 327, 329, 328, 327, 331, 318]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:11.312251702Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1926,"":"downstairs","":"downstairs","session_id":"b80bc3f3-fec2-4bc9-827f-4b82947464d6"} {"msg":"[2]R generation[0..12]: [183, 184, 181, 185, 179, 184, 182, 184, 183, 182, 185, 176]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:11.31226504Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1926,"":"downstairs","":"downstairs","session_id":"b80bc3f3-fec2-4bc9-827f-4b82947464d6"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:11.312278648Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1926,"":"downstairs","":"downstairs","session_id":"b80bc3f3-fec2-4bc9-827f-4b82947464d6"} {"msg":"Max found gen is 186","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:11.312291035Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1926,"":"downstairs","":"downstairs","session_id":"b80bc3f3-fec2-4bc9-827f-4b82947464d6"} {"msg":"Generation requested: 186 >= found:186","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:11.312303813Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1926,"":"downstairs","":"downstairs","session_id":"b80bc3f3-fec2-4bc9-827f-4b82947464d6"} {"msg":"Next flush: 332","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:11.31231641Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1926,"":"Wait for a query_work_queue command to finish before sending IO downstairs","":"downstairs","session_id":"b80bc3f3-fec2-4bc9-827f-4b82947464d6"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:11.312336282Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1926,"":"downstairs","":"downstairs","session_id":"b80bc3f3-fec2-4bc9-827f-4b82947464d6"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:11.312349099Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1926,"session_id":"b80bc3f3-fec2-4bc9-827f-4b82947464d6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:11.312360796Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1926,"client":"0","":"downstairs","session_id":"b80bc3f3-fec2-4bc9-827f-4b82947464d6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:11.312373634Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1926,"client":"1","":"downstairs","session_id":"b80bc3f3-fec2-4bc9-827f-4b82947464d6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:11.312386421Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1926,"client":"2","":"downstairs","session_id":"b80bc3f3-fec2-4bc9-827f-4b82947464d6"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:11.312399059Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1926,"session_id":"b80bc3f3-fec2-4bc9-827f-4b82947464d6"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:11.312414067Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1926,"session_id":"b80bc3f3-fec2-4bc9-827f-4b82947464d6"} {"msg":"5088cb64-454f-4e14-8674-654fd0707f3b is now active with session: b80bc3f3-fec2-4bc9-827f-4b82947464d6","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:11.312426805Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1926,"session_id":"b80bc3f3-fec2-4bc9-827f-4b82947464d6"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:11.312438511Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1926,"session_id":"b80bc3f3-fec2-4bc9-827f-4b82947464d6"} {"msg":"The guest has finished waiting for activation with:186","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:11.312450369Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1926} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 279 len 28672 data: 11 14 18 18 14 16 15 02/30 Read block 314 len 8192 03/30 Read block 231 len 28672 04/30 Read block 72 len 28672 05/30 Read block 129 len 40960 06/30 Read block 293 len 24576 07/30 Write block 441 len 36864 data: 10 8 8 9 7 8 5 8 11 08/30 Write block 584 len 16384 data: 15 13 13 12 09/30 Write block 145 len 20480 data: 12 11 14 13 14 10/30 Read block 142 len 12288 11/30 Read block 94 len 28672 12/30 Read block 32 len 12288 13/30 Read block 287 len 28672 14/30 Read block 290 len 36864 15/30 Read block 382 len 12288 16/30 Flush 17/30 Write block 95 len 4096 data: 18 18/30 Write block 420 len 4096 data: 12 19/30 Read block 391 len 12288 20/30 Flush 21/30 Write block 171 len 4096 data: 15 22/30 Write block 486 len 32768 data: 12 12 13 13 16 15 12 12 23/30 Read block 141 len 16384 24/30 Read block 400 len 12288 25/30 Read block 271 len 4096 26/30 Read block 461 len 16384 27/30 Read block 523 len 20480 28/30 Read block 475 len 24576 29/30 Write block 67 len 20480 data: 12 17 16 15 17 30/30 Read block 540 len 40960 ---------------------------------------------------------------- Crucible gen:186 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1020 Acked Write 1 Done Done Done false 1021 Acked Write 8 Done Done Done false 1022 Acked Read 4 Done Done Done false 1023 Acked Read 3 Done Done Done false 1024 Acked Read 1 Done Done Done false 1025 Acked Read 4 Done Done Done false 1026 Acked Read 5 Done Done Done false 1027 Acked Read 6 Done Done Done false 1028 Acked Write 5 Done Done Done false 1029 Acked Read 10 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 1 2 Done 9 10 9 28 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1019 1019 1019 Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:19:11.332511043Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1926} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 183 183 183 328 328 328 F F F 1 020-039 184 184 184 329 329 329 F F F 2 040-059 181 181 181 324 324 324 F F F 3 060-079 185 185 185 331 331 331 T T T 4 080-099 186 186 186 333 333 333 F F F 5 100-119 184 184 184 330 330 330 F F F 6 120-139 182 182 182 327 327 327 F F F 7 140-159 186 186 186 332 332 332 F F F 8 160-179 183 183 183 328 328 328 T T T 9 180-199 182 182 182 327 327 327 F F F 10 200-219 185 185 185 331 331 331 F F F 11 220-239 176 176 176 318 318 318 F F F 12 240-259 185 185 185 331 331 331 F F F 13 260-279 186 186 186 332 332 332 F F F 14 280-299 186 186 186 332 332 332 F F F 15 300-319 181 181 181 324 324 324 F F F 16 320-339 181 181 181 324 324 324 F F F 17 340-359 183 183 183 328 328 328 F F F 18 360-379 179 179 179 322 322 322 F F F 19 380-399 181 181 181 324 324 324 F F F 20 400-419 184 184 184 329 329 329 F F F 21 420-439 186 186 186 333 333 333 F F F 22 440-459 186 186 186 332 332 332 F F F 23 460-479 185 185 185 331 331 331 F F F 24 480-499 184 184 184 329 329 329 T T T 25 500-519 179 179 179 322 322 322 F F F 26 520-539 179 179 179 322 322 322 F F F 27 540-559 185 185 185 331 331 331 F F F 28 560-579 185 185 185 331 331 331 F F F 29 580-599 186 186 186 332 332 332 F F F Max gen: 186, Max flush: 333 A Difference in extent metadata is expected here On loop 93 Verifying data now No change after verify Loop: 93 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 183 183 183 328 328 328 F F F 1 020-039 184 184 184 329 329 329 F F F 2 040-059 181 181 181 324 324 324 F F F 3 060-079 187 187 187 334 334 334 F F F 4 080-099 186 186 186 333 333 333 F F F 5 100-119 184 184 184 330 330 330 F F F 6 120-139 182 182 182 327 327 327 F F F 7 140-159 186 186 186 332 332 332 F F F 8 160-179 187 187 187 334 334 334 F F F 9 180-199 182 182 182 327 327 327 F F F 10 200-219 185 185 185 331 331 331 F F F 11 220-239 176 176 176 318 318 318 F F F 12 240-259 185 185 185 331 331 331 F F F 13 260-279 186 186 186 332 332 332 F F F 14 280-299 186 186 186 332 332 332 F F F 15 300-319 181 181 181 324 324 324 F F F 16 320-339 181 181 181 324 324 324 F F F 17 340-359 183 183 183 328 328 328 F F F 18 360-379 179 179 179 322 322 322 F F F 19 380-399 181 181 181 324 324 324 F F F 20 400-419 184 184 184 329 329 329 F F F 21 420-439 186 186 186 333 333 333 F F F 22 440-459 186 186 186 332 332 332 F F F 23 460-479 185 185 185 331 331 331 F F F 24 480-499 187 187 187 334 334 334 F F F 25 500-519 179 179 179 322 322 322 F F F 26 520-539 179 179 179 322 322 322 F F F 27 540-559 185 185 185 331 331 331 F F F 28 560-579 185 185 185 331 331 331 F F F 29 580-599 186 186 186 332 332 332 F F F Max gen: 187, Max flush: 334 Begin loop 94 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:15.562246462Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1934,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:15.607331643Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1934,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:15.608009569Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1934} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"abd5430f1508af0bc008143ea2566277ef714ec2\",\n git_commit_timestamp: \"2025-01-17T20:55:03.000000000Z\",\n git_branch: \"alan/tokio-to-140\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:15.608053095Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1934} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:15.608067173Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1934} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:15.608081561Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1934} {"msg":"Crucible b2385ecd-ec30-4f39-b5dc-fe8e92010232 has session id: 50dcc4a0-941a-4a9d-a816-97d891527c60","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:15.608595011Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1934,"session_id":"50dcc4a0-941a-4a9d-a816-97d891527c60"} {"msg":"Upstairs opts: Upstairs UUID: b2385ecd-ec30-4f39-b5dc-fe8e92010232, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:15.608647993Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1934,"session_id":"50dcc4a0-941a-4a9d-a816-97d891527c60"} {"msg":"Crucible stats registered with UUID: b2385ecd-ec30-4f39-b5dc-fe8e92010232","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:15.608671456Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1934,"session_id":"50dcc4a0-941a-4a9d-a816-97d891527c60"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:19:15.608687736Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1934,"session_id":"50dcc4a0-941a-4a9d-a816-97d891527c60"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:15.609221568Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1934,"task":"crutest"} {"msg":"The guest has requested activation with gen:188","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:15.609245311Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1934} {"msg":"b2385ecd-ec30-4f39-b5dc-fe8e92010232 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:15.609295821Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1934,"session_id":"50dcc4a0-941a-4a9d-a816-97d891527c60"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:15.609330131Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1934,"client":"0","":"downstairs","session_id":"50dcc4a0-941a-4a9d-a816-97d891527c60"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:15.609366742Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1934,"client":"0","":"downstairs","session_id":"50dcc4a0-941a-4a9d-a816-97d891527c60"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:15.609393918Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1934,"client":"1","":"downstairs","session_id":"50dcc4a0-941a-4a9d-a816-97d891527c60"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:15.609422855Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1934,"client":"1","":"downstairs","session_id":"50dcc4a0-941a-4a9d-a816-97d891527c60"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:15.609454254Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1934,"client":"2","":"downstairs","session_id":"50dcc4a0-941a-4a9d-a816-97d891527c60"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:15.60948213Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1934,"client":"2","":"downstairs","session_id":"50dcc4a0-941a-4a9d-a816-97d891527c60"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:15.609509996Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1934,"":"io task","client":"2","":"downstairs","session_id":"50dcc4a0-941a-4a9d-a816-97d891527c60"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:15.609537873Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1934,"":"io task","client":"0","":"downstairs","session_id":"50dcc4a0-941a-4a9d-a816-97d891527c60"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:15.609565869Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1934,"":"io task","client":"1","":"downstairs","session_id":"50dcc4a0-941a-4a9d-a816-97d891527c60"} {"msg":"ds_connection connected from Ok(127.0.0.1:50454)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:15.609591294Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1934,"":"io task","client":"1","":"downstairs","session_id":"50dcc4a0-941a-4a9d-a816-97d891527c60"} {"msg":"ds_connection connected from Ok(127.0.0.1:47775)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:15.609613186Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1934,"":"io task","client":"2","":"downstairs","session_id":"50dcc4a0-941a-4a9d-a816-97d891527c60"} {"msg":"ds_connection connected from Ok(127.0.0.1:63754)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:15.609758201Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1934,"":"io task","client":"0","":"downstairs","session_id":"50dcc4a0-941a-4a9d-a816-97d891527c60"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:15.610330866Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1934,"client":"1","":"downstairs","session_id":"50dcc4a0-941a-4a9d-a816-97d891527c60"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:15.610502256Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1934,"client":"0","":"downstairs","session_id":"50dcc4a0-941a-4a9d-a816-97d891527c60"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:15.61054226Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1934,"client":"2","":"downstairs","session_id":"50dcc4a0-941a-4a9d-a816-97d891527c60"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:15.611040871Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1934,"client":"1","":"downstairs","session_id":"50dcc4a0-941a-4a9d-a816-97d891527c60"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:15.611068347Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1934,"client":"0","":"downstairs","session_id":"50dcc4a0-941a-4a9d-a816-97d891527c60"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:15.611315292Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1934,"client":"2","":"downstairs","session_id":"50dcc4a0-941a-4a9d-a816-97d891527c60"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:15.611623141Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1934,"session_id":"50dcc4a0-941a-4a9d-a816-97d891527c60"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:15.611647876Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1934,"session_id":"50dcc4a0-941a-4a9d-a816-97d891527c60"} {"msg":"[0]R flush_numbers[0..12]: [328, 329, 324, 334, 333, 330, 327, 332, 334, 327, 331, 318]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:15.611858679Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1934,"":"downstairs","":"downstairs","session_id":"50dcc4a0-941a-4a9d-a816-97d891527c60"} {"msg":"[0]R generation[0..12]: [183, 184, 181, 187, 186, 184, 182, 186, 187, 182, 185, 176]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:15.611887456Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1934,"":"downstairs","":"downstairs","session_id":"50dcc4a0-941a-4a9d-a816-97d891527c60"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:15.611903675Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1934,"":"downstairs","":"downstairs","session_id":"50dcc4a0-941a-4a9d-a816-97d891527c60"} {"msg":"[1]R flush_numbers[0..12]: [328, 329, 324, 334, 333, 330, 327, 332, 334, 327, 331, 318]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:15.611917684Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1934,"":"downstairs","":"downstairs","session_id":"50dcc4a0-941a-4a9d-a816-97d891527c60"} {"msg":"[1]R generation[0..12]: [183, 184, 181, 187, 186, 184, 182, 186, 187, 182, 185, 176]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:15.611931151Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1934,"":"downstairs","":"downstairs","session_id":"50dcc4a0-941a-4a9d-a816-97d891527c60"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:15.611944629Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1934,"":"downstairs","":"downstairs","session_id":"50dcc4a0-941a-4a9d-a816-97d891527c60"} {"msg":"[2]R flush_numbers[0..12]: [328, 329, 324, 334, 333, 330, 327, 332, 334, 327, 331, 318]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:15.611958067Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1934,"":"downstairs","":"downstairs","session_id":"50dcc4a0-941a-4a9d-a816-97d891527c60"} {"msg":"[2]R generation[0..12]: [183, 184, 181, 187, 186, 184, 182, 186, 187, 182, 185, 176]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:15.611971285Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1934,"":"downstairs","":"downstairs","session_id":"50dcc4a0-941a-4a9d-a816-97d891527c60"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:15.611984753Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1934,"":"downstairs","":"downstairs","session_id":"50dcc4a0-941a-4a9d-a816-97d891527c60"} {"msg":"Max found gen is 188","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:15.61199725Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1934,"":"downstairs","":"downstairs","session_id":"50dcc4a0-941a-4a9d-a816-97d891527c60"} {"msg":"Generation requested: 188 >= found:188","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:15.612010207Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1934,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO 50dcc4a0-941a-4a9d-a816-97d891527c60"} {"msg":"Next flush: 335","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:15.612028628Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1934,"":"downstairs","":"downstairs","session_id":"50dcc4a0-941a-4a9d-a816-97d891527c60"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:15.612046559Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1934,"":"downstairs","":"downstairs","session_id":"50dcc4a0-941a-4a9d-a816-97d891527c60"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:15.612059687Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1934,"session_id":"50dcc4a0-941a-4a9d-a816-97d891527c60"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:15.612071233Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1934,"client":"0","":"downstairs","session_id":"50dcc4a0-941a-4a9d-a816-97d891527c60"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:15.612083981Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1934,"client":"1","":"downstairs","session_id":"50dcc4a0-941a-4a9d-a816-97d891527c60"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:15.612096668Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1934,"client":"2","":"downstairs","session_id":"50dcc4a0-941a-4a9d-a816-97d891527c60"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:15.612109316Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1934,"session_id":"50dcc4a0-941a-4a9d-a816-97d891527c60"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:15.612121323Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1934,"session_id":"50dcc4a0-941a-4a9d-a816-97d891527c60"} {"msg":"b2385ecd-ec30-4f39-b5dc-fe8e92010232 is now active with session: 50dcc4a0-941a-4a9d-a816-97d891527c60","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:15.61213368Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1934,"session_id":"50dcc4a0-941a-4a9d-a816-97d891527c60"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:15.612145246Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1934,"session_id":"50dcc4a0-941a-4a9d-a816-97d891527c60"} {"msg":"The guest has finished waiting for activation with:188","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:15.612157133Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1934} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 96 len 4096 data: 18 03/30 Flush 04/30 Write block 63 len 24576 data: 10 12 12 13 13 18 05/30 Flush 06/30 Write block 478 len 8192 data: 10 9 07/30 Read block 171 len 32768 08/30 Write block 475 len 28672 data: 17 14 12 11 10 11 12 09/30 Write block 467 len 16384 data: 17 16 16 15 10/30 Write block 283 len 20480 data: 15 17 16 17 16 11/30 Write block 55 len 12288 data: 18 19 19 12/30 Read block 223 len 24576 13/30 Read block 504 len 4096 14/30 Write block 476 len 36864 data: 15 13 12 11 12 13 11 9 10 15/30 Read block 143 len 4096 16/30 Read block 272 len 16384 17/30 Write block 345 len 28672 data: 11 9 11 15 13 16 12 18/30 Read block 533 len 4096 19/30 Read block 11 len 36864 20/30 Read block 548 len 20480 21/30 Write block 70 len 4096 data: 16 22/30 Write block 306 len 20480 data: 15 12 11 7 6 23/30 Write block 255 len 28672 data: 16 16 16 14 15 14 13 24/30 Write block 484 len 32768 data: 11 11 13 13 14 14 17 16 25/30 Read block 322 len 40960 26/30 Read block 542 len 4096 27/30 Write block 594 len 24576 data: 11 8 9 7 7 4 28/30 Read block 455 len 16384 29/30 Read block 103 len 8192 30/30 Read block 69 len 20480 ---------------------------------------------------------------- Crucible gen:188 GIO:true work queues: Upstairs:0 downstairs:25 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1005 Acked Write 2 Done Done Done false 1006 Acked Read 8 Done Done Done false 1007 Acked Write 7 Done Done Done false 1008 Acked Write 4 Done Done Done false 1009 Acked Write 5 Done Done Done false 1010 Acked Write 3 Done Done Done false 1011 Acked Read 6 Done Done Done false 1012 Acked Read 1 Done Done Done false 1013 Acked Write 9 Done Done Done false 1014 Acked Read 1 Done Done Done false 1015 Acked Read 4 Done Done Done false 1016 Acked Write 7 Done Done Done false 1017 Acked Read 1 Done Done Done false 1018 Acked Read 9 Done Done Done false 1019 Acked Read 5 Done Done Done false 1020 Acked Write 1 Done Done Done false 1021 Acked Write 5 Done Done Done false 1022 Acked Write 7 Done Done Done false 1023 Acked Write 8 Done Done Done false 1024 Acked Read 10 Done Done Done false 1025 Acked Read 1 Done Done Done false 1026 Acked Write 6 Done Done Done false 1027 Acked Read 4 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Read 5 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 25 25 25 75 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1004 1004 1004 Downstairs last five completed: 1004 1003 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:19:15.632230065Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1934} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 183 183 183 328 328 328 F F F 1 020-039 184 184 184 329 329 329 F F F 2 040-059 181 181 181 324 324 324 T T T 3 060-079 188 188 188 337 337 337 T T T 4 080-099 188 188 188 336 336 336 F F F 5 100-119 184 184 184 330 330 330 F F F 6 120-139 182 182 182 327 327 327 F F F 7 140-159 186 186 186 332 332 332 F F F 8 160-179 187 187 187 334 334 334 F F F 9 180-199 182 182 182 327 327 327 F F F 10 200-219 185 185 185 331 331 331 F F F 11 220-239 176 176 176 318 318 318 F F F 12 240-259 185 185 185 331 331 331 T T T 13 260-279 186 186 186 332 332 332 T T T 14 280-299 186 186 186 332 332 332 T T T 15 300-319 181 181 181 324 324 324 T T T 16 320-339 181 181 181 324 324 324 F F F 17 340-359 183 183 183 328 328 328 T T T 18 360-379 179 179 179 322 322 322 F F F 19 380-399 181 181 181 324 324 324 F F F 20 400-419 184 184 184 329 329 329 F F F 21 420-439 186 186 186 333 333 333 F F F 22 440-459 186 186 186 332 332 332 F F F 23 460-479 185 185 185 331 331 331 T T T 24 480-499 187 187 187 334 334 334 T T T 25 500-519 179 179 179 322 322 322 F F F 26 520-539 179 179 179 322 322 322 F F F 27 540-559 185 185 185 331 331 331 F F F 28 560-579 185 185 185 331 331 331 F F F 29 580-599 186 186 186 332 332 332 T T T Max gen: 188, Max flush: 337 A Difference in extent metadata is expected here On loop 94 Verifying data now No change after verify Loop: 94 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 183 183 183 328 328 328 F F F 1 020-039 184 184 184 329 329 329 F F F 2 040-059 189 189 189 338 338 338 F F F 3 060-079 189 189 189 338 338 338 F F F 4 080-099 188 188 188 336 336 336 F F F 5 100-119 184 184 184 330 330 330 F F F 6 120-139 182 182 182 327 327 327 F F F 7 140-159 186 186 186 332 332 332 F F F 8 160-179 187 187 187 334 334 334 F F F 9 180-199 182 182 182 327 327 327 F F F 10 200-219 185 185 185 331 331 331 F F F 11 220-239 176 176 176 318 318 318 F F F 12 240-259 189 189 189 338 338 338 F F F 13 260-279 189 189 189 338 338 338 F F F 14 280-299 189 189 189 338 338 338 F F F 15 300-319 189 189 189 338 338 338 F F F 16 320-339 181 181 181 324 324 324 F F F 17 340-359 189 189 189 338 338 338 F F F 18 360-379 179 179 179 322 322 322 F F F 19 380-399 181 181 181 324 324 324 F F F 20 400-419 184 184 184 329 329 329 F F F 21 420-439 186 186 186 333 333 333 F F F 22 440-459 186 186 186 332 332 332 F F F 23 460-479 189 189 189 338 338 338 F F F 24 480-499 189 189 189 338 338 338 F F F 25 500-519 179 179 179 322 322 322 F F F 26 520-539 179 179 179 322 322 322 F F F 27 540-559 185 185 185 331 331 331 F F F 28 560-579 185 185 185 331 331 331 F F F 29 580-599 189 189 189 338 338 338 F F F Max gen: 189, Max flush: 338 Begin loop 95 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:19.902066382Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1942,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:19.94706172Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1942,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:19.947723227Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1942} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"abd5430f1508af0bc008143ea2566277ef714ec2\",\n git_commit_timestamp: \"2025-01-17T20:55:03.000000000Z\",\n git_branch: \"alan/tokio-to-140\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:19.947771715Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1942} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:19.947789165Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1942} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:19.947806806Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1942} {"msg":"Crucible 21eb16cc-f26b-481c-9fe4-da1fef40bfd6 has session id: 48ce02ef-ead7-44c9-96db-9b9e60ad8b76","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:19.948397301Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1942,"session_id":"48ce02ef-ead7-44c9-96db-9b9e60ad8b76"} {"msg":"Upstairs opts: Upstairs UUID: 21eb16cc-f26b-481c-9fe4-da1fef40bfd6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:19.948431761Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1942,"session_id":"48ce02ef-ead7-44c9-96db-9b9e60ad8b76"} {"msg":"Crucible stats registered with UUID: 21eb16cc-f26b-481c-9fe4-da1fef40bfd6","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:19.948450542Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1942,"session_id":"48ce02ef-ead7-44c9-96db-9b9e60ad8b76"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:19:19.94846347Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1942,"session_id":"48ce02ef-ead7-44c9-96db-9b9e60ad8b76"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:19.948838609Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1942,"task":"crutest"} {"msg":"The guest has requested activation with gen:190","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:19.948862393Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1942} {"msg":"21eb16cc-f26b-481c-9fe4-da1fef40bfd6 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:19.948935065Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1942,"session_id":"48ce02ef-ead7-44c9-96db-9b9e60ad8b76"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:19.948962511Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1942,"client":"0","":"downstairs","session_id":"48ce02ef-ead7-44c9-96db-9b9e60ad8b76"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:19.949000353Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1942,"client":"0","":"downstairs","session_id":"48ce02ef-ead7-44c9-96db-9b9e60ad8b76"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:19.94902936Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1942,"client":"1","":"downstairs","session_id":"48ce02ef-ead7-44c9-96db-9b9e60ad8b76"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:19.949059118Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1942,"client":"1","":"downstairs","session_id":"48ce02ef-ead7-44c9-96db-9b9e60ad8b76"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:19.949083512Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1942,"client":"2","":"downstairs","session_id":"48ce02ef-ead7-44c9-96db-9b9e60ad8b76"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:19.949108987Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1942,"client":"2","":"downstairs","session_id":"48ce02ef-ead7-44c9-96db-9b9e60ad8b76"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:19.949134542Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1942,"":"io task","client":"2","":"downstairs","session_id":"48ce02ef-ead7-44c9-96db-9b9e60ad8b76"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:19.949159696Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1942,"":"io task","client":"0","":"downstairs","session_id":"48ce02ef-ead7-44c9-96db-9b9e60ad8b76"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:19.949184411Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1942,"":"io task","client":"1","":"downstairs","session_id":"48ce02ef-ead7-44c9-96db-9b9e60ad8b76"} {"msg":"ds_connection connected from Ok(127.0.0.1:42617)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:19.949242815Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1942,"":"io task","client":"0","":"downstairs","session_id":"48ce02ef-ead7-44c9-96db-9b9e60ad8b76"} {"msg":"ds_connection connected from Ok(127.0.0.1:43753)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:19.949316328Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1942,"":"io task","client":"2","":"downstairs","session_id":"48ce02ef-ead7-44c9-96db-9b9e60ad8b76"} {"msg":"ds_connection connected from Ok(127.0.0.1:50835)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:19.949343884Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1942,"":"io task","client":"1","":"downstairs","session_id":"48ce02ef-ead7-44c9-96db-9b9e60ad8b76"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:19.949865879Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1942,"client":"2","":"downstairs","session_id":"48ce02ef-ead7-44c9-96db-9b9e60ad8b76"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:19.949905182Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1942,"client":"0","":"downstairs","session_id":"48ce02ef-ead7-44c9-96db-9b9e60ad8b76"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:19.950070849Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1942,"client":"1","":"downstairs","session_id":"48ce02ef-ead7-44c9-96db-9b9e60ad8b76"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:19.950564057Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1942,"client":"2","":"downstairs","session_id":"48ce02ef-ead7-44c9-96db-9b9e60ad8b76"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:19.950592864Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1942,"client":"0","":"downstairs","session_id":"48ce02ef-ead7-44c9-96db-9b9e60ad8b76"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:19.950623822Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1942,"client":"1","":"downstairs","session_id":"48ce02ef-ead7-44c9-96db-9b9e60ad8b76"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:19.951102982Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1942,"session_id":"48ce02ef-ead7-44c9-96db-9b9e60ad8b76"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:19.951217889Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1942,"session_id":"48ce02ef-ead7-44c9-96db-9b9e60ad8b76"} {"msg":"[0]R flush_numbers[0..12]: [328, 329, 338, 338, 336, 330, 327, 332, 334, 327, 331, 318]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:19.951261375Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1942,"":"downstairs","":"downstairs","session_id":"48ce02ef-ead7-44c9-96db-9b9e60ad8b76"} {"msg":"[0]R generation[0..12]: [183, 184, 189, 189, 188, 184, 182, 186, 187, 182, 185, 176]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:19.951278905Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1942,"":"downstairs","":"downstairs","session_id":"48ce02ef-ead7-44c9-96db-9b9e60ad8b76"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:19.951293223Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1942,"":"downstairs","":"downstairs","session_id":"48ce02ef-ead7-44c9-96db-9b9e60ad8b76"} {"msg":"[1]R flush_numbers[0..12]: [328, 329, 338, 338, 336, 330, 327, 332, 334, 327, 331, 318]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:19.951306671Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1942,"":"downstairs","":"downstairs","session_id":"48ce02ef-ead7-44c9-96db-9b9e60ad8b76"} {"msg":"[1]R generation[0..12]: [183, 184, 189, 189, 188, 184, 182, 186, 187, 182, 185, 176]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:19.951322971Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1942,"":"downstairs","":"downstairs","session_id":"48ce02ef-ead7-44c9-96db-9b9e60ad8b76"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:19.951337239Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1942,"":"downstairs","":"downstairs","session_id":"48ce02ef-ead7-44c9-96db-9b9e60ad8b76"} {"msg":"[2]R flush_numbers[0..12]: [328, 329, 338, 338, 336, 330, 327, 332, 334, 327, 331, 318]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:19.951350637Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1942,"":"downstairs","":"downstairs","session_id":"48ce02ef-ead7-44c9-96db-9b9e60ad8b76"} {"msg":"[2]R generation[0..12]: [183, 184, 189, 189, 188, 184, 182, 186, 187, 182, 185, 176]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:19.951363905Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1942,"":"downstairs","":"downstairs","session_id":"48ce02ef-ead7-44c9-96db-9b9e60ad8b76"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:19.951377282Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1942,"":"downstairs","":"downstairs","session_id":"48ce02ef-ead7-44c9-96db-9b9e60ad8b76"} {"msg":"Max found gen is 190","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:19.95138976Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1942,"":"downstairs","":"downstairs","session_id":"48ce02ef-ead7-44c9-96db-9b9e60ad8b76"} {"msg":"Generation requested: 190 >= found:190","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:19.951402467Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1942,"":"downstairs","":"downstairs","session_id":"48ce02ef-ead7-44c9-96db-9b9e60ad8b76"} {"msg":"Next flush: 339","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:19.951414855Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1942,"":"downstairs","":"downstairs","session_id":"48ce02ef-ead7-44c9-96db-9b9e60ad8b76"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:19.951427382Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1942,"":"downstairs","":"downstairs","session_id":"48ce02ef-ead7-44c9-96db-9b9e60ad8b76"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:19.951440299Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1942,"session_id":"48ce02ef-ead7-44c9-96db-9b9e60ad8b76"} Wait for a query_work_queue command to finish before sending IO {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:19.95145784Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1942,"client":"0","":"downstairs","session_id":"48ce02ef-ead7-44c9-96db-9b9e60ad8b76"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:19.951470897Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1942,"client":"1","":"downstairs","session_id":"48ce02ef-ead7-44c9-96db-9b9e60ad8b76"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:19.951483505Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1942,"client":"2","":"downstairs","session_id":"48ce02ef-ead7-44c9-96db-9b9e60ad8b76"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:19.951496102Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1942,"session_id":"48ce02ef-ead7-44c9-96db-9b9e60ad8b76"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:19.951507969Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1942,"session_id":"48ce02ef-ead7-44c9-96db-9b9e60ad8b76"} {"msg":"21eb16cc-f26b-481c-9fe4-da1fef40bfd6 is now active with session: 48ce02ef-ead7-44c9-96db-9b9e60ad8b76","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:19.951523158Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1942,"session_id":"48ce02ef-ead7-44c9-96db-9b9e60ad8b76"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:19.951535035Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1942,"session_id":"48ce02ef-ead7-44c9-96db-9b9e60ad8b76"} {"msg":"The guest has finished waiting for activation with:190","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:19.951547032Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1942} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 135 len 32768 data: 7 10 10 10 13 12 12 15 02/30 Write block 95 len 40960 data: 19 19 13 16 15 15 16 14 18 18 03/30 Read block 520 len 32768 04/30 Read block 415 len 4096 05/30 Write block 390 len 28672 data: 9 10 11 10 11 11 15 06/30 Read block 299 len 32768 07/30 Read block 280 len 40960 08/30 Read block 392 len 20480 09/30 Flush 10/30 Write block 384 len 8192 data: 13 12 11/30 Read block 340 len 24576 12/30 Write block 77 len 24576 data: 13 13 13 13 13 10 13/30 Read block 136 len 40960 14/30 Write block 78 len 28672 data: 14 14 14 14 11 10 12 15/30 Write block 355 len 24576 data: 15 18 19 18 18 13 16/30 Read block 543 len 36864 17/30 Read block 286 len 12288 18/30 Read block 195 len 32768 19/30 Flush 20/30 Write block 360 len 16384 data: 14 12 13 14 21/30 Write block 340 len 24576 data: 19 19 14 17 12 12 22/30 Write block 485 len 20480 data: 12 14 14 15 15 23/30 Write block 151 len 24576 data: 16 15 14 15 16 14 24/30 Read block 136 len 20480 25/30 Write block 596 len 4096 data: 10 26/30 Write block 362 len 4096 data: 14 27/30 Write block 221 len 20480 data: 9 11 13 11 13 28/30 Write block 365 len 28672 data: 10 9 9 9 9 14 14 29/30 Write block 545 len 12288 data: 11 13 13 30/30 Read block 523 len 8192 ---------------------------------------------------------------- Crucible gen:190 GIO:true work queues: Upstairs:0 downstairs:11 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1019 Acked Write 4 Done Done Done false 1020 Acked Write 6 Done Done Done false 1021 Acked Write 5 Done Done Done false 1022 Acked Write 6 Done Done Done false 1023 Acked Read 5 Done Done Done false 1024 Acked Write 1 Done Done Done false 1025 Acked Write 1 Done Done Done false 1026 Acked Write 5 Done Done Done false 1027 Acked Write 7 Done Done Done false 1028 Acked Write 3 Done Done Done false 1029 Acked Read 2 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 11 11 11 33 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1018 1018 1018 Downstairs last five completed: 1018 1017 1016 1015 1014 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:19:19.970248652Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1942} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 183 183 183 328 328 328 F F F 1 020-039 184 184 184 329 329 329 F F F 2 040-059 189 189 189 338 338 338 F F F 3 060-079 190 190 190 340 340 340 F F F 4 080-099 190 190 190 340 340 340 F F F 5 100-119 190 190 190 339 339 339 F F F 6 120-139 190 190 190 339 339 339 F F F 7 140-159 190 190 190 339 339 339 T T T 8 160-179 187 187 187 334 334 334 F F F 9 180-199 182 182 182 327 327 327 F F F 10 200-219 185 185 185 331 331 331 F F F 11 220-239 176 176 176 318 318 318 T T T 12 240-259 189 189 189 338 338 338 F F F 13 260-279 189 189 189 338 338 338 F F F 14 280-299 189 189 189 338 338 338 F F F 15 300-319 189 189 189 338 338 338 F F F 16 320-339 181 181 181 324 324 324 F F F 17 340-359 190 190 190 340 340 340 T T T 18 360-379 190 190 190 340 340 340 T T T 19 380-399 190 190 190 340 340 340 F F F 20 400-419 184 184 184 329 329 329 F F F 21 420-439 186 186 186 333 333 333 F F F 22 440-459 186 186 186 332 332 332 F F F 23 460-479 189 189 189 338 338 338 F F F 24 480-499 189 189 189 338 338 338 T T T 25 500-519 179 179 179 322 322 322 F F F 26 520-539 179 179 179 322 322 322 F F F 27 540-559 185 185 185 331 331 331 T T T 28 560-579 185 185 185 331 331 331 F F F 29 580-599 189 189 189 338 338 338 T T T Max gen: 190, Max flush: 340 A Difference in extent metadata is expected here On loop 95 Verifying data now No change after verify Loop: 95 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 183 183 183 328 328 328 F F F 1 020-039 184 184 184 329 329 329 F F F 2 040-059 189 189 189 338 338 338 F F F 3 060-079 190 190 190 340 340 340 F F F 4 080-099 190 190 190 340 340 340 F F F 5 100-119 190 190 190 339 339 339 F F F 6 120-139 190 190 190 339 339 339 F F F 7 140-159 191 191 191 341 341 341 F F F 8 160-179 187 187 187 334 334 334 F F F 9 180-199 182 182 182 327 327 327 F F F 10 200-219 185 185 185 331 331 331 F F F 11 220-239 191 191 191 341 341 341 F F F 12 240-259 189 189 189 338 338 338 F F F 13 260-279 189 189 189 338 338 338 F F F 14 280-299 189 189 189 338 338 338 F F F 15 300-319 189 189 189 338 338 338 F F F 16 320-339 181 181 181 324 324 324 F F F 17 340-359 191 191 191 341 341 341 F F F 18 360-379 191 191 191 341 341 341 F F F 19 380-399 190 190 190 340 340 340 F F F 20 400-419 184 184 184 329 329 329 F F F 21 420-439 186 186 186 333 333 333 F F F 22 440-459 186 186 186 332 332 332 F F F 23 460-479 189 189 189 338 338 338 F F F 24 480-499 191 191 191 341 341 341 F F F 25 500-519 179 179 179 322 322 322 F F F 26 520-539 179 179 179 322 322 322 F F F 27 540-559 191 191 191 341 341 341 F F F 28 560-579 185 185 185 331 331 331 F F F 29 580-599 191 191 191 341 341 341 F F F Max gen: 191, Max flush: 341 Begin loop 96 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.217634317Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1950,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:19:24.261460792Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1950,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.261896356Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1950,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.29567627Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1950,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.296277081Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1950} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"abd5430f1508af0bc008143ea2566277ef714ec2\",\n git_commit_timestamp: \"2025-01-17T20:55:03.000000000Z\",\n git_branch: \"alan/tokio-to-140\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.296315513Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1950} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.296332513Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1950} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.296349873Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1950} {"msg":"Crucible 6f56a947-0808-49d8-8575-8b9456f36dd7 has session id: 467ea12b-3d3a-4e7d-bb16-616ce8c752ba","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.296961271Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1950,"session_id":"467ea12b-3d3a-4e7d-bb16-616ce8c752ba"} {"msg":"Upstairs opts: Upstairs UUID: 6f56a947-0808-49d8-8575-8b9456f36dd7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.297008498Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1950,"session_id":"467ea12b-3d3a-4e7d-bb16-616ce8c752ba"} {"msg":"Crucible stats registered with UUID: 6f56a947-0808-49d8-8575-8b9456f36dd7","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.297036815Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1950,"session_id":"467ea12b-3d3a-4e7d-bb16-616ce8c752ba"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:19:24.29706337Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1950,"session_id":"467ea12b-3d3a-4e7d-bb16-616ce8c752ba"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.297579132Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1950,"task":"crutest"} {"msg":"The guest has requested activation with gen:192","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.297607288Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1950} {"msg":"6f56a947-0808-49d8-8575-8b9456f36dd7 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.297671126Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1950,"session_id":"467ea12b-3d3a-4e7d-bb16-616ce8c752ba"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.297687915Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1950,"client":"0","":"downstairs","session_id":"467ea12b-3d3a-4e7d-bb16-616ce8c752ba"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.29771198Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1950,"client":"0","":"downstairs","session_id":"467ea12b-3d3a-4e7d-bb16-616ce8c752ba"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.29772938Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1950,"client":"1","":"downstairs","session_id":"467ea12b-3d3a-4e7d-bb16-616ce8c752ba"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.297743848Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1950,"client":"1","":"downstairs","session_id":"467ea12b-3d3a-4e7d-bb16-616ce8c752ba"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.297756706Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1950,"client":"2","":"downstairs","session_id":"467ea12b-3d3a-4e7d-bb16-616ce8c752ba"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.297770934Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1950,"client":"2","":"downstairs","session_id":"467ea12b-3d3a-4e7d-bb16-616ce8c752ba"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.297783952Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1950,"":"io task","client":"2","":"downstairs","session_id":"467ea12b-3d3a-4e7d-bb16-616ce8c752ba"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.297797319Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1950,"":"io task","client":"0","":"downstairs","session_id":"467ea12b-3d3a-4e7d-bb16-616ce8c752ba"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.297820993Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1950,"":"io task","client":"1","":"downstairs","session_id":"467ea12b-3d3a-4e7d-bb16-616ce8c752ba"} {"msg":"ds_connection connected from Ok(127.0.0.1:48433)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.297848029Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1950,"":"io task","client":"2","":"downstairs","session_id":"467ea12b-3d3a-4e7d-bb16-616ce8c752ba"} {"msg":"ds_connection connected from Ok(127.0.0.1:39523)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.297894816Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1950,"":"io task","client":"0","":"downstairs","session_id":"467ea12b-3d3a-4e7d-bb16-616ce8c752ba"} {"msg":"ds_connection connected from Ok(127.0.0.1:56521)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.298074622Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1950,"":"io task","client":"1","":"downstairs","session_id":"467ea12b-3d3a-4e7d-bb16-616ce8c752ba"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.298523674Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1950,"client":"1","":"downstairs","session_id":"467ea12b-3d3a-4e7d-bb16-616ce8c752ba"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.298562817Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1950,"client":"2","":"downstairs","session_id":"467ea12b-3d3a-4e7d-bb16-616ce8c752ba"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.29887649Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1950,"client":"0","":"downstairs","session_id":"467ea12b-3d3a-4e7d-bb16-616ce8c752ba"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.299353589Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1950,"client":"1","":"downstairs","session_id":"467ea12b-3d3a-4e7d-bb16-616ce8c752ba"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.299381135Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1950,"client":"2","":"downstairs","session_id":"467ea12b-3d3a-4e7d-bb16-616ce8c752ba"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.299400707Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1950,"client":"0","":"downstairs","session_id":"467ea12b-3d3a-4e7d-bb16-616ce8c752ba"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.299888742Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1950,"session_id":"467ea12b-3d3a-4e7d-bb16-616ce8c752ba"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.299980566Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1950,"session_id":"467ea12b-3d3a-4e7d-bb16-616ce8c752ba"} {"msg":"[0]R flush_numbers[0..12]: [328, 329, 338, 340, 340, 339, 339, 341, 334, 327, 331, 341]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.300048275Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1950,"":"downstairs","":"downstairs","session_id":"467ea12b-3d3a-4e7d-bb16-616ce8c752ba"} {"msg":"[0]R generation[0..12]: [183, 184, 189, 190, 190, 190, 190, 191, 187, 182, 185, 191]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.300067846Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1950,"":"downstairs","":"downstairs","session_id":"467ea12b-3d3a-4e7d-bb16-616ce8c752ba"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.300082105Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1950,"":"downstairs","":"downstairs","session_id":"467ea12b-3d3a-4e7d-bb16-616ce8c752ba"} {"msg":"[1]R flush_numbers[0..12]: [328, 329, 338, 340, 340, 339, 339, 341, 334, 327, 331, 341]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.300095603Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1950,"":"downstairs","":"downstairs","session_id":"467ea12b-3d3a-4e7d-bb16-616ce8c752ba"} {"msg":"[1]R generation[0..12]: [183, 184, 189, 190, 190, 190, 190, 191, 187, 182, 185, 191]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.300109091Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1950,"":"downstairs","":"downstairs","session_id":"467ea12b-3d3a-4e7d-bb16-616ce8c752ba"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.300122558Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1950,"":"downstairs","":"downstairs","session_id":"467ea12b-3d3a-4e7d-bb16-616ce8c752ba"} {"msg":"[2]R flush_numbers[0..12]: [328, 329, 338, 340, 340, 339, 339, 341, 334, 327, 331, 341]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.300135736Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1950,"":"downstairs","":"downstairs","session_id":"467ea12b-3d3a-4e7d-bb16-616ce8c752ba"} {"msg":"[2]R generation[0..12]: [183, 184, 189, 190, 190, 190, 190, 191, 187, 182, 185, 191]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.300148724Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1950,"":"downstairs","":"downstairs","session_id":"467ea12b-3d3a-4e7d-bb16-616ce8c752ba"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.300162092Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1950,"":"downstairs","":"downstairs","session_id":"467ea12b-3d3a-4e7d-bb16-616ce8c752ba"} {"msg":"Max found gen is 192","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.300174609Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1950,"":"downstairs","":"downstairs","session_id":"467ea12b-3d3a-4e7d-bb16-616ce8c752ba"} {"msg":"Generation requested: 192 >= found:192","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.300187307Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1950,"":"downstairs","":"downstairs","session_id":"467ea12b-3d3a-4e7d-bb16-616ce8c752ba"} {"msg":"Next flush: 342","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.300203826Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1950,"":"downstairs","":"downstairs","session_id":"467ea12b-3d3a-4e7d-bb16-616ce8c752ba"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.300217084Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1950,"":"downstairs","":"downstairs","session_id":"467ea12b-3d3a-4e7d-bb16-616ce8c752ba"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.300229901Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1950,"session_id":"467ea12b-3d3a-4e7d-bb16-616ce8c752ba"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-17T21:19:24.300248983Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1950,"client":"0","":"downstairs","session_id":"467ea12b-3d3a-4e7d-bb16-616ce8c752ba"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.300276719Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1950,"client":"1","":"downstairs","session_id":"467ea12b-3d3a-4e7d-bb16-616ce8c752ba"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.300302174Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1950,"client":"2","":"downstairs","session_id":"467ea12b-3d3a-4e7d-bb16-616ce8c752ba"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.300321105Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1950,"session_id":"467ea12b-3d3a-4e7d-bb16-616ce8c752ba"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.300333462Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1950,"session_id":"467ea12b-3d3a-4e7d-bb16-616ce8c752ba"} {"msg":"6f56a947-0808-49d8-8575-8b9456f36dd7 is now active with session: 467ea12b-3d3a-4e7d-bb16-616ce8c752ba","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.300351663Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1950,"session_id":"467ea12b-3d3a-4e7d-bb16-616ce8c752ba"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.300372965Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1950,"session_id":"467ea12b-3d3a-4e7d-bb16-616ce8c752ba"} {"msg":"The guest has finished waiting for activation with:192","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.300396429Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1950} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 266 len 32768 data: 9 10 10 15 15 12 12 11 03/30 Write block 587 len 20480 data: 13 12 12 11 8 04/30 Write block 347 len 12288 data: 12 16 14 05/30 Read block 480 len 12288 06/30 Write block 125 len 28672 data: 12 9 11 12 13 14 12 07/30 Read block 49 len 28672 08/30 Read block 76 len 28672 09/30 Write block 328 len 40960 data: 8 11 10 11 11 11 14 15 14 19 10/30 Read block 583 len 8192 11/30 Flush 12/30 Write block 80 len 40960 data: 15 15 12 11 13 10 11 13 14 17 13/30 Write block 462 len 16384 data: 15 13 16 16 14/30 Write block 132 len 36864 data: 11 8 7 8 11 11 11 14 13 15/30 Read block 90 len 28672 16/30 Read block 464 len 36864 17/30 Read block 52 len 4096 18/30 Write block 525 len 32768 data: 15 16 14 12 10 10 13 14 19/30 Read block 143 len 8192 20/30 Read block 552 len 24576 21/30 Read block 551 len 36864 22/30 Read block 207 len 8192 23/30 Write block 340 len 20480 data: 20 20 15 18 13 24/30 Read block 237 len 40960 25/30 Write block 230 len 12288 data: 19 16 14 26/30 Read block 556 len 36864 27/30 Read block 389 len 40960 28/30 Read block 221 len 12288 29/30 Write block 163 len 16384 data: 13 16 16 16 30/30 Write block 577 len 40960 data: 16 15 15 15 16 15 15 16 14 14 ---------------------------------------------------------------- Crucible gen:192 GIO:true work queues: Upstairs:0 downstairs:19 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1011 Acked Write 10 Done Done Done false 1012 Acked Write 4 Done Done Done false 1013 Acked Write 9 Done Done Done false 1014 Acked Read 7 Done Done Done false 1015 Acked Read 9 Done Done Done false 1016 Acked Read 1 Done Done Done false 1017 Acked Write 8 Done Done Done false 1018 Acked Read 2 Done Done Done false 1019 Acked Read 6 Done Done Done false 1020 Acked Read 9 Done Done Done false 1021 Acked Read 2 Done Done Done false 1022 Acked Write 5 Done Done Done false 1023 Acked Read 10 Done Done Done false 1024 Acked Write 3 Done Done Done false 1025 Acked Read 9 Done Done Done false 1026 Acked Read 10 Done Done Done false 1027 Acked Read 3 Done Done Done false 1028 Acked Write 4 Done Done Done false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 18 18 18 54 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1010 1010 1010 Downstairs last five completed: 1010 1009 1008 1007 1006 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:19:24.321011658Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1950} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 183 183 183 328 328 328 F F F 1 020-039 184 184 184 329 329 329 F F F 2 040-059 189 189 189 338 338 338 F F F 3 060-079 190 190 190 340 340 340 F F F 4 080-099 190 190 190 340 340 340 T T T 5 100-119 190 190 190 339 339 339 F F F 6 120-139 192 192 192 343 343 343 T T T 7 140-159 191 191 191 341 341 341 T T T 8 160-179 187 187 187 334 334 334 T T T 9 180-199 182 182 182 327 327 327 F F F 10 200-219 185 185 185 331 331 331 F F F 11 220-239 191 191 191 341 341 341 T T T 12 240-259 189 189 189 338 338 338 F F F 13 260-279 192 192 192 343 343 343 F F F 14 280-299 189 189 189 338 338 338 F F F 15 300-319 189 189 189 338 338 338 F F F 16 320-339 192 192 192 343 343 343 F F F 17 340-359 192 192 192 343 343 343 T T T 18 360-379 191 191 191 341 341 341 F F F 19 380-399 190 190 190 340 340 340 F F F 20 400-419 184 184 184 329 329 329 F F F 21 420-439 186 186 186 333 333 333 F F F 22 440-459 186 186 186 332 332 332 F F F 23 460-479 189 189 189 338 338 338 T T T 24 480-499 191 191 191 341 341 341 F F F 25 500-519 179 179 179 322 322 322 F F F 26 520-539 179 179 179 322 322 322 T T T 27 540-559 191 191 191 341 341 341 F F F 28 560-579 185 185 185 331 331 331 T T T 29 580-599 192 192 192 343 343 343 T T T Max gen: 192, Max flush: 343 A Difference in extent metadata is expected here On loop 96 Verifying data now No change after verify Loop: 96 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 183 183 183 328 328 328 F F F 1 020-039 184 184 184 329 329 329 F F F 2 040-059 189 189 189 338 338 338 F F F 3 060-079 190 190 190 340 340 340 F F F 4 080-099 193 193 193 344 344 344 F F F 5 100-119 190 190 190 339 339 339 F F F 6 120-139 193 193 193 344 344 344 F F F 7 140-159 193 193 193 344 344 344 F F F 8 160-179 193 193 193 344 344 344 F F F 9 180-199 182 182 182 327 327 327 F F F 10 200-219 185 185 185 331 331 331 F F F 11 220-239 193 193 193 344 344 344 F F F 12 240-259 189 189 189 338 338 338 F F F 13 260-279 192 192 192 343 343 343 F F F 14 280-299 189 189 189 338 338 338 F F F 15 300-319 189 189 189 338 338 338 F F F 16 320-339 192 192 192 343 343 343 F F F 17 340-359 193 193 193 344 344 344 F F F 18 360-379 191 191 191 341 341 341 F F F 19 380-399 190 190 190 340 340 340 F F F 20 400-419 184 184 184 329 329 329 F F F 21 420-439 186 186 186 333 333 333 F F F 22 440-459 186 186 186 332 332 332 F F F 23 460-479 193 193 193 344 344 344 F F F 24 480-499 191 191 191 341 341 341 F F F 25 500-519 179 179 179 322 322 322 F F F 26 520-539 193 193 193 344 344 344 F F F 27 540-559 191 191 191 341 341 341 F F F 28 560-579 193 193 193 344 344 344 F F F 29 580-599 193 193 193 344 344 344 F F F Max gen: 193, Max flush: 344 Begin loop 97 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:28.621666767Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1958,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:28.666708262Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1958,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:28.667375352Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1958} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"abd5430f1508af0bc008143ea2566277ef714ec2\",\n git_commit_timestamp: \"2025-01-17T20:55:03.000000000Z\",\n git_branch: \"alan/tokio-to-140\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:28.667416516Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1958} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:28.667431745Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1958} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:28.667446754Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1958} {"msg":"Crucible dbfa8cb4-8abb-4424-a49b-84c2cd2f4c2e has session id: b01e58e8-dced-4d7c-b3e2-4636e82c1a8e","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:28.668019289Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1958,"session_id":"b01e58e8-dced-4d7c-b3e2-4636e82c1a8e"} {"msg":"Upstairs opts: Upstairs UUID: dbfa8cb4-8abb-4424-a49b-84c2cd2f4c2e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:28.668061813Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1958,"session_id":"b01e58e8-dced-4d7c-b3e2-4636e82c1a8e"} {"msg":"Crucible stats registered with UUID: dbfa8cb4-8abb-4424-a49b-84c2cd2f4c2e","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:28.668091371Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1958,"session_id":"b01e58e8-dced-4d7c-b3e2-4636e82c1a8e"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:19:28.668114144Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1958,"session_id":"b01e58e8-dced-4d7c-b3e2-4636e82c1a8e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:28.66858541Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1958,"task":"crutest"} {"msg":"The guest has requested activation with gen:194","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:28.668607753Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1958} {"msg":"dbfa8cb4-8abb-4424-a49b-84c2cd2f4c2e active request set","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:28.668664956Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1958,"session_id":"b01e58e8-dced-4d7c-b3e2-4636e82c1a8e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:28.668691171Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1958,"client":"0","":"downstairs","session_id":"b01e58e8-dced-4d7c-b3e2-4636e82c1a8e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:28.668716506Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1958,"client":"0","":"downstairs","session_id":"b01e58e8-dced-4d7c-b3e2-4636e82c1a8e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:28.668734567Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1958,"client":"1","":"downstairs","session_id":"b01e58e8-dced-4d7c-b3e2-4636e82c1a8e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:28.668761302Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1958,"client":"1","":"downstairs","session_id":"b01e58e8-dced-4d7c-b3e2-4636e82c1a8e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:28.668786067Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1958,"client":"2","":"downstairs","session_id":"b01e58e8-dced-4d7c-b3e2-4636e82c1a8e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:28.668811402Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1958,"client":"2","":"downstairs","session_id":"b01e58e8-dced-4d7c-b3e2-4636e82c1a8e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:28.668835896Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1958,"":"io task","client":"2","":"downstairs","session_id":"b01e58e8-dced-4d7c-b3e2-4636e82c1a8e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:28.66886048Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1958,"":"io task","client":"0","":"downstairs","session_id":"b01e58e8-dced-4d7c-b3e2-4636e82c1a8e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:28.668887396Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1958,"":"io task","client":"1","":"downstairs","session_id":"b01e58e8-dced-4d7c-b3e2-4636e82c1a8e"} {"msg":"ds_connection connected from Ok(127.0.0.1:52736)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:28.668918314Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1958,"":"io task","client":"2","":"downstairs","session_id":"b01e58e8-dced-4d7c-b3e2-4636e82c1a8e"} {"msg":"ds_connection connected from Ok(127.0.0.1:49059)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:28.6689454Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1958,"":"io task","client":"0","":"downstairs","session_id":"b01e58e8-dced-4d7c-b3e2-4636e82c1a8e"} {"msg":"ds_connection connected from Ok(127.0.0.1:57280)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:28.669182258Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1958,"":"io task","client":"1","":"downstairs","session_id":"b01e58e8-dced-4d7c-b3e2-4636e82c1a8e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:28.669670534Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1958,"client":"0","":"downstairs","session_id":"b01e58e8-dced-4d7c-b3e2-4636e82c1a8e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:28.66969744Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1958,"client":"1","":"downstairs","session_id":"b01e58e8-dced-4d7c-b3e2-4636e82c1a8e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:28.669712168Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1958,"client":"2","":"downstairs","session_id":"b01e58e8-dced-4d7c-b3e2-4636e82c1a8e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:28.670241177Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1958,"client":"0","":"downstairs","session_id":"b01e58e8-dced-4d7c-b3e2-4636e82c1a8e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:28.67031497Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1958,"client":"1","":"downstairs","session_id":"b01e58e8-dced-4d7c-b3e2-4636e82c1a8e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:28.670374045Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1958,"client":"2","":"downstairs","session_id":"b01e58e8-dced-4d7c-b3e2-4636e82c1a8e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:28.67089583Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1958,"session_id":"b01e58e8-dced-4d7c-b3e2-4636e82c1a8e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:28.67093017Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1958,"session_id":"b01e58e8-dced-4d7c-b3e2-4636e82c1a8e"} {"msg":"[0]R flush_numbers[0..12]: [328, 329, 338, 340, 344, 339, 344, 344, 344, 327, 331, 344]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:28.670987434Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1958,"":"downstairs","":"downstairs","session_id":"b01e58e8-dced-4d7c-b3e2-4636e82c1a8e"} {"msg":"[0]R generation[0..12]: [183, 184, 189, 190, 193, 190, 193, 193, 193, 182, 185, 193]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:28.67101528Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1958,"":"downstairs","":"downstairs","session_id":"b01e58e8-dced-4d7c-b3e2-4636e82c1a8e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:28.671030379Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1958,"":"downstairs","":"downstairs","session_id":"b01e58e8-dced-4d7c-b3e2-4636e82c1a8e"} {"msg":"[1]R flush_numbers[0..12]: [328, 329, 338, 340, 344, 339, 344, 344, 344, 327, 331, 344]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:28.671044807Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1958,"":"downstairs","":"downstairs","session_id":"b01e58e8-dced-4d7c-b3e2-4636e82c1a8e"} {"msg":"[1]R generation[0..12]: [183, 184, 189, 190, 193, 190, 193, 193, 193, 182, 185, 193]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:28.671062277Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1958,"":"downstairs","":"downstairs","session_id":"b01e58e8-dced-4d7c-b3e2-4636e82c1a8e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:28.671077306Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1958,"":"downstairs","":"downstairs","session_id":"b01e58e8-dced-4d7c-b3e2-4636e82c1a8e"} {"msg":"[2]R flush_numbers[0..12]: [328, 329, 338, 340, 344, 339, 344, 344, 344, 327, 331, 344]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:28.671091114Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1958,"":"downstairs","":"downstairs","session_id":"b01e58e8-dced-4d7c-b3e2-4636e82c1a8e"} {"msg":"[2]R generation[0..12]: [183, 184, 189, 190, 193, 190, 193, 193, 193, 182, 185, 193]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:28.671104472Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1958,"":"downstairs","":"downstairs","session_id":"b01e58e8-dced-4d7c-b3e2-4636e82c1a8e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:28.67111789Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1958,"":"downstairs","":"downstairs","session_id":"b01e58e8-dced-4d7c-b3e2-4636e82c1a8e"} {"msg":"Max found gen is 194","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:28.671130417Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","Wait for a query_work_queue command to finish before sending IO pid":1958,"":"downstairs","":"downstairs","session_id":"b01e58e8-dced-4d7c-b3e2-4636e82c1a8e"} {"msg":"Generation requested: 194 >= found:194","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:28.671149848Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1958,"":"downstairs","":"downstairs","session_id":"b01e58e8-dced-4d7c-b3e2-4636e82c1a8e"} {"msg":"Next flush: 345","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:28.671162586Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1958,"":"downstairs","":"downstairs","session_id":"b01e58e8-dced-4d7c-b3e2-4636e82c1a8e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:28.671174853Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1958,"":"downstairs","":"downstairs","session_id":"b01e58e8-dced-4d7c-b3e2-4636e82c1a8e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:28.671197766Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1958,"session_id":"b01e58e8-dced-4d7c-b3e2-4636e82c1a8e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:28.671210284Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1958,"client":"0","":"downstairs","session_id":"b01e58e8-dced-4d7c-b3e2-4636e82c1a8e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:28.671223241Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1958,"client":"1","":"downstairs","session_id":"b01e58e8-dced-4d7c-b3e2-4636e82c1a8e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:28.671237119Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1958,"client":"2","":"downstairs","session_id":"b01e58e8-dced-4d7c-b3e2-4636e82c1a8e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:28.671249807Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1958,"session_id":"b01e58e8-dced-4d7c-b3e2-4636e82c1a8e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:28.671263785Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1958,"session_id":"b01e58e8-dced-4d7c-b3e2-4636e82c1a8e"} {"msg":"dbfa8cb4-8abb-4424-a49b-84c2cd2f4c2e is now active with session: b01e58e8-dced-4d7c-b3e2-4636e82c1a8e","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:28.671276592Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1958,"session_id":"b01e58e8-dced-4d7c-b3e2-4636e82c1a8e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:28.671288189Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1958,"session_id":"b01e58e8-dced-4d7c-b3e2-4636e82c1a8e"} {"msg":"The guest has finished waiting for activation with:194","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:28.671301137Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1958} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 433 len 40960 data: 11 14 11 12 12 11 9 8 11 9 02/30 Write block 27 len 40960 data: 18 14 14 17 15 18 20 20 23 18 03/30 Write block 187 len 40960 data: 8 12 14 15 15 17 18 18 17 17 04/30 Write block 392 len 12288 data: 12 11 12 05/30 Write block 516 len 36864 data: 27 22 20 19 16 15 11 19 18 06/30 Read block 396 len 36864 07/30 Flush 08/30 Write block 498 len 16384 data: 20 17 19 21 09/30 Write block 40 len 12288 data: 18 17 14 10/30 Read block 145 len 16384 11/30 Read block 278 len 36864 12/30 Flush 13/30 Write block 68 len 32768 data: 19 17 17 18 17 14 12 14 14/30 Read block 223 len 36864 15/30 Read block 292 len 40960 16/30 Write block 489 len 24576 data: 16 18 17 13 13 14 17/30 Flush 18/30 Write block 22 len 20480 data: 13 11 13 15 19 19/30 Read block 0 len 28672 20/30 Read block 278 len 4096 21/30 Flush 22/30 Write block 299 len 40960 data: 12 12 14 12 16 16 18 16 13 12 23/30 Write block 157 len 24576 data: 14 13 14 15 15 15 24/30 Write block 1 len 16384 data: 10 13 13 16 25/30 Write block 151 len 24576 data: 17 16 15 16 17 15 26/30 Read block 432 len 16384 27/30 Write block 86 len 8192 data: 12 14 28/30 Read block 68 len 12288 29/30 Write block 587 len 8192 data: 14 13 30/30 Read block 553 len 32768 ---------------------------------------------------------------- Crucible gen:194 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 10 Done Done Done false 1022 Acked Write 6 Done Done Done false 1023 Acked Write 4 Done Done Done false 1024 Acked Write 6 Done Done Done false 1025 Acked Read 4 Done Done Done false 1026 Acked Write 2 Done Done Done false 1027 Acked Read 3 Done Done Done false 1028 Acked Write 2 Done Done Done false 1029 Acked Read 8 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 9 9 9 27 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1020 1020 1020 Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:19:28.691258822Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1958} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 183 183 183 328 328 328 T T T 1 020-039 194 194 194 348 348 348 F F F 2 040-059 194 194 194 346 346 346 F F F 3 060-079 194 194 194 347 347 347 F F F 4 080-099 193 193 193 344 344 344 T T T 5 100-119 190 190 190 339 339 339 F F F 6 120-139 193 193 193 344 344 344 F F F 7 140-159 193 193 193 344 344 344 T T T 8 160-179 193 193 193 344 344 344 T T T 9 180-199 194 194 194 345 345 345 F F F 10 200-219 185 185 185 331 331 331 F F F 11 220-239 193 193 193 344 344 344 F F F 12 240-259 189 189 189 338 338 338 F F F 13 260-279 192 192 192 343 343 343 F F F 14 280-299 189 189 189 338 338 338 T T T 15 300-319 189 189 189 338 338 338 T T T 16 320-339 192 192 192 343 343 343 F F F 17 340-359 193 193 193 344 344 344 F F F 18 360-379 191 191 191 341 341 341 F F F 19 380-399 194 194 194 345 345 345 F F F 20 400-419 184 184 184 329 329 329 F F F 21 420-439 194 194 194 345 345 345 F F F 22 440-459 194 194 194 345 345 345 F F F 23 460-479 193 193 193 344 344 344 F F F 24 480-499 194 194 194 347 347 347 F F F 25 500-519 194 194 194 346 346 346 F F F 26 520-539 194 194 194 345 345 345 F F F 27 540-559 191 191 191 341 341 341 F F F 28 560-579 193 193 193 344 344 344 F F F 29 580-599 193 193 193 344 344 344 T T T Max gen: 194, Max flush: 348 A Difference in extent metadata is expected here On loop 97 Verifying data now No change after verify Loop: 97 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 195 195 195 349 349 349 F F F 1 020-039 194 194 194 348 348 348 F F F 2 040-059 194 194 194 346 346 346 F F F 3 060-079 194 194 194 347 347 347 F F F 4 080-099 195 195 195 349 349 349 F F F 5 100-119 190 190 190 339 339 339 F F F 6 120-139 193 193 193 344 344 344 F F F 7 140-159 195 195 195 349 349 349 F F F 8 160-179 195 195 195 349 349 349 F F F 9 180-199 194 194 194 345 345 345 F F F 10 200-219 185 185 185 331 331 331 F F F 11 220-239 193 193 193 344 344 344 F F F 12 240-259 189 189 189 338 338 338 F F F 13 260-279 192 192 192 343 343 343 F F F 14 280-299 195 195 195 349 349 349 F F F 15 300-319 195 195 195 349 349 349 F F F 16 320-339 192 192 192 343 343 343 F F F 17 340-359 193 193 193 344 344 344 F F F 18 360-379 191 191 191 341 341 341 F F F 19 380-399 194 194 194 345 345 345 F F F 20 400-419 184 184 184 329 329 329 F F F 21 420-439 194 194 194 345 345 345 F F F 22 440-459 194 194 194 345 345 345 F F F 23 460-479 193 193 193 344 344 344 F F F 24 480-499 194 194 194 347 347 347 F F F 25 500-519 194 194 194 346 346 346 F F F 26 520-539 194 194 194 345 345 345 F F F 27 540-559 191 191 191 341 341 341 F F F 28 560-579 193 193 193 344 344 344 F F F 29 580-599 195 195 195 349 349 349 F F F Max gen: 195, Max flush: 349 Begin loop 98 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:32.949756242Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1966,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:32.993889777Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1966,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:32.994516393Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1966} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"abd5430f1508af0bc008143ea2566277ef714ec2\",\n git_commit_timestamp: \"2025-01-17T20:55:03.000000000Z\",\n git_branch: \"alan/tokio-to-140\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:32.994560479Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1966} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:32.994575097Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1966} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:32.994589045Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1966} {"msg":"Crucible 0027a692-d575-45e8-a4d9-6dd89ffae800 has session id: 8e301242-2608-4c18-8a03-a555931cf0ba","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:32.995070147Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1966,"session_id":"8e301242-2608-4c18-8a03-a555931cf0ba"} {"msg":"Upstairs opts: Upstairs UUID: 0027a692-d575-45e8-a4d9-6dd89ffae800, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:32.995115953Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1966,"session_id":"8e301242-2608-4c18-8a03-a555931cf0ba"} {"msg":"Crucible stats registered with UUID: 0027a692-d575-45e8-a4d9-6dd89ffae800","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:32.995138747Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1966,"session_id":"8e301242-2608-4c18-8a03-a555931cf0ba"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:19:32.995160039Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1966,"session_id":"8e301242-2608-4c18-8a03-a555931cf0ba"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:32.995548106Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1966,"task":"crutest"} {"msg":"The guest has requested activation with gen:196","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:32.99557199Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1966} {"msg":"0027a692-d575-45e8-a4d9-6dd89ffae800 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:32.995624221Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1966,"session_id":"8e301242-2608-4c18-8a03-a555931cf0ba"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:32.995658901Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1966,"client":"0","":"downstairs","session_id":"8e301242-2608-4c18-8a03-a555931cf0ba"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:32.995700755Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1966,"client":"0","":"downstairs","session_id":"8e301242-2608-4c18-8a03-a555931cf0ba"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:32.995730472Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1966,"client":"1","":"downstairs","session_id":"8e301242-2608-4c18-8a03-a555931cf0ba"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:32.995756568Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1966,"client":"1","":"downstairs","session_id":"8e301242-2608-4c18-8a03-a555931cf0ba"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:32.995786515Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1966,"client":"2","":"downstairs","session_id":"8e301242-2608-4c18-8a03-a555931cf0ba"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:32.995816273Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1966,"client":"2","":"downstairs","session_id":"8e301242-2608-4c18-8a03-a555931cf0ba"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:32.995842278Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1966,"":"io task","client":"2","":"downstairs","session_id":"8e301242-2608-4c18-8a03-a555931cf0ba"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:32.995867883Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1966,"":"io task","client":"0","":"downstairs","session_id":"8e301242-2608-4c18-8a03-a555931cf0ba"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:32.995890636Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1966,"":"io task","client":"1","":"downstairs","session_id":"8e301242-2608-4c18-8a03-a555931cf0ba"} {"msg":"ds_connection connected from Ok(127.0.0.1:39990)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:32.995915651Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1966,"":"io task","client":"2","":"downstairs","session_id":"8e301242-2608-4c18-8a03-a555931cf0ba"} {"msg":"ds_connection connected from Ok(127.0.0.1:57326)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:32.995942006Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1966,"":"io task","client":"0","":"downstairs","session_id":"8e301242-2608-4c18-8a03-a555931cf0ba"} {"msg":"ds_connection connected from Ok(127.0.0.1:59513)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:32.995967791Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1966,"":"io task","client":"1","":"downstairs","session_id":"8e301242-2608-4c18-8a03-a555931cf0ba"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:32.996666029Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1966,"client":"0","":"downstairs","session_id":"8e301242-2608-4c18-8a03-a555931cf0ba"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:32.996705553Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1966,"client":"1","":"downstairs","session_id":"8e301242-2608-4c18-8a03-a555931cf0ba"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:32.997267371Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1966,"client":"1","":"downstairs","session_id":"8e301242-2608-4c18-8a03-a555931cf0ba"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:32.997411455Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1966,"client":"0","":"downstairs","session_id":"8e301242-2608-4c18-8a03-a555931cf0ba"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:32.997444414Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1966,"client":"2","":"downstairs","session_id":"8e301242-2608-4c18-8a03-a555931cf0ba"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:32.997780441Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1966,"session_id":"8e301242-2608-4c18-8a03-a555931cf0ba"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:32.998023503Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1966,"session_id":"8e301242-2608-4c18-8a03-a555931cf0ba"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:32.998117098Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1966,"client":"2","":"downstairs","session_id":"8e301242-2608-4c18-8a03-a555931cf0ba"} {"msg":"[0]R flush_numbers[0..12]: [349, 348, 346, 347, 349, 339, 344, 349, 349, 345, 331, 344]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:32.998631278Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1966,"":"downstairs","":"downstairs","session_id":"8e301242-2608-4c18-8a03-a555931cf0ba"} {"msg":"[0]R generation[0..12]: [195, 194, 194, 194, 195, 190, 193, 195, 195, 194, 185, 193]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:32.99865157Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1966,"":"downstairs","":"downstairs","session_id":"8e301242-2608-4c18-8a03-a555931cf0ba"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:32.998665778Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1966,"":"downstairs","":"downstairs","session_id":"8e301242-2608-4c18-8a03-a555931cf0ba"} {"msg":"[1]R flush_numbers[0..12]: [349, 348, 346, 347, 349, 339, 344, 349, 349, 345, 331, 344]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:32.998679326Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1966,"":"downstairs","":"downstairs","session_id":"8e301242-2608-4c18-8a03-a555931cf0ba"} {"msg":"[1]R generation[0..12]: [195, 194, 194, 194, 195, 190, 193, 195, 195, 194, 185, 193]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:32.998692844Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1966,"":"downstairs","":"downstairs","session_id":"8e301242-2608-4c18-8a03-a555931cf0ba"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:32.998706482Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1966,"":"downstairs","":"downstairs","session_id":"8e301242-2608-4c18-8a03-a555931cf0ba"} {"msg":"[2]R flush_numbers[0..12]: [349, 348, 346, 347, 349, 339, 344, 349, 349, 345, 331, 344]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:32.99871986Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1966,"":"downstairs","":"downstairs","session_id":"8e301242-2608-4c18-8a03-a555931cf0ba"} {"msg":"[2]R generation[0..12]: [195, 194, 194, 194, 195, 190, 193, 195, 195, 194, 185, 193]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:32.998733278Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1966,"":"downstairs","":"downstairs","session_id":"8e301242-2608-4c18-8a03-a555931cf0ba"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:32.998746746Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1966,"":"downstairs","":"downstairs","session_id":"8e301242-2608-4c18-8a03-a555931cf0ba"} {"msg":"Max found gen is 196","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:32.998759253Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1966,"":"downstairs","":"downstairs","session_id":"8e301242-2608-4c18-8a03-a555931cf0ba"} {"msg":"Generation requested: 196 >= found:196","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:32.99877203Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1966,"":"downstairs","":"downstairs","session_id":"8e301242-2608-4c18-8a03-a555931cf0ba"} {"msg":"Next flush: 350","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:32.998784328Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1966,"":"downstairs","":"downstairs","session_id":"8e301242-2608-4c18-8a03-a555931cf0ba"Wait for a query_work_queue command to finish before sending IO } {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:32.998807661Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1966,"":"downstairs","":"downstairs","session_id":"8e301242-2608-4c18-8a03-a555931cf0ba"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:32.99882339Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1966,"session_id":"8e301242-2608-4c18-8a03-a555931cf0ba"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:32.998838169Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1966,"client":"0","":"downstairs","session_id":"8e301242-2608-4c18-8a03-a555931cf0ba"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:32.998854048Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1966,"client":"1","":"downstairs","session_id":"8e301242-2608-4c18-8a03-a555931cf0ba"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:32.998869807Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1966,"client":"2","":"downstairs","session_id":"8e301242-2608-4c18-8a03-a555931cf0ba"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:32.998885877Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1966,"session_id":"8e301242-2608-4c18-8a03-a555931cf0ba"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:32.998900645Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1966,"session_id":"8e301242-2608-4c18-8a03-a555931cf0ba"} {"msg":"0027a692-d575-45e8-a4d9-6dd89ffae800 is now active with session: 8e301242-2608-4c18-8a03-a555931cf0ba","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:32.998916205Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1966,"session_id":"8e301242-2608-4c18-8a03-a555931cf0ba"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:32.998930623Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1966,"session_id":"8e301242-2608-4c18-8a03-a555931cf0ba"} {"msg":"The guest has finished waiting for activation with:196","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:32.998945672Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1966} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 335 len 8192 data: 16 15 02/30 Write block 571 len 12288 data: 17 15 15 03/30 Write block 269 len 28672 data: 16 16 13 13 12 13 15 04/30 Read block 247 len 24576 05/30 Read block 209 len 20480 06/30 Write block 103 len 20480 data: 19 19 17 15 13 07/30 Read block 329 len 24576 08/30 Write block 135 len 32768 data: 9 12 12 12 15 14 13 16 09/30 Read block 7 len 4096 10/30 Flush 11/30 Flush 12/30 Write block 83 len 16384 data: 12 14 11 13 13/30 Write block 174 len 12288 data: 14 16 16 14/30 Read block 11 len 36864 15/30 Flush 16/30 Write block 284 len 40960 data: 18 17 18 17 17 13 14 16 16 15 17/30 Read block 115 len 4096 18/30 Write block 445 len 28672 data: 8 9 6 9 12 10 10 19/30 Read block 283 len 24576 20/30 Write block 209 len 32768 data: 15 15 13 10 13 11 11 12 21/30 Write block 2 len 32768 data: 14 14 17 16 16 17 18 16 22/30 Write block 57 len 8192 data: 20 17 23/30 Write block 31 len 20480 data: 16 19 21 21 24 24/30 Read block 373 len 36864 25/30 Read block 172 len 28672 26/30 Read block 171 len 28672 27/30 Read block 302 len 40960 28/30 Read block 525 len 4096 29/30 Read block 194 len 28672 30/30 Read block 142 len 20480 ---------------------------------------------------------------- Crucible gen:196 GIO:true work queues: Upstairs:0 downstairs:15 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1015 Acked Write 10 Done Done Done false 1016 Acked Read 1 Done Done Done false 1017 Acked Write 7 Done Done Done false 1018 Acked Read 6 Done Done Done false 1019 Acked Write 8 Done Done Done false 1020 Acked Write 8 Done Done Done false 1021 Acked Write 2 Done Done Done false 1022 Acked Write 5 Done Done Done false 1023 Acked Read 9 Done Done Done false 1024 Acked Read 7 Done Done Done false 1025 Acked Read 7 Done Done Done false 1026 Acked Read 10 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Read 7 Done Done Done false 1029 Acked Read 5 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 15 15 14 44 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1014 1014 1014 Downstairs last five completed: 1014 1013 1012 1011 1010 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:19:33.018974558Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1966} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 195 195 195 349 349 349 T T T 1 020-039 194 194 194 348 348 348 T T T 2 040-059 194 194 194 346 346 346 T T T 3 060-079 194 194 194 347 347 347 F F F 4 080-099 196 196 196 352 352 352 F F F 5 100-119 196 196 196 350 350 350 F F F 6 120-139 196 196 196 350 350 350 F F F 7 140-159 196 196 196 350 350 350 F F F 8 160-179 196 196 196 352 352 352 F F F 9 180-199 194 194 194 345 345 345 F F F 10 200-219 185 185 185 331 331 331 T T T 11 220-239 193 193 193 344 344 344 F F F 12 240-259 189 189 189 338 338 338 F F F 13 260-279 196 196 196 350 350 350 F F F 14 280-299 195 195 195 349 349 349 T T T 15 300-319 195 195 195 349 349 349 F F F 16 320-339 196 196 196 350 350 350 F F F 17 340-359 193 193 193 344 344 344 F F F 18 360-379 191 191 191 341 341 341 F F F 19 380-399 194 194 194 345 345 345 F F F 20 400-419 184 184 184 329 329 329 F F F 21 420-439 194 194 194 345 345 345 F F F 22 440-459 194 194 194 345 345 345 T T T 23 460-479 193 193 193 344 344 344 F F F 24 480-499 194 194 194 347 347 347 F F F 25 500-519 194 194 194 346 346 346 F F F 26 520-539 194 194 194 345 345 345 F F F 27 540-559 191 191 191 341 341 341 F F F 28 560-579 196 196 196 350 350 350 F F F 29 580-599 195 195 195 349 349 349 F F F Max gen: 196, Max flush: 352 A Difference in extent metadata is expected here On loop 98 Verifying data now No change after verify Loop: 98 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 197 197 197 353 353 353 F F F 1 020-039 197 197 197 353 353 353 F F F 2 040-059 197 197 197 353 353 353 F F F 3 060-079 194 194 194 347 347 347 F F F 4 080-099 196 196 196 352 352 352 F F F 5 100-119 196 196 196 350 350 350 F F F 6 120-139 196 196 196 350 350 350 F F F 7 140-159 196 196 196 350 350 350 F F F 8 160-179 196 196 196 352 352 352 F F F 9 180-199 194 194 194 345 345 345 F F F 10 200-219 197 197 197 353 353 353 F F F 11 220-239 193 193 193 344 344 344 F F F 12 240-259 189 189 189 338 338 338 F F F 13 260-279 196 196 196 350 350 350 F F F 14 280-299 197 197 197 353 353 353 F F F 15 300-319 195 195 195 349 349 349 F F F 16 320-339 196 196 196 350 350 350 F F F 17 340-359 193 193 193 344 344 344 F F F 18 360-379 191 191 191 341 341 341 F F F 19 380-399 194 194 194 345 345 345 F F F 20 400-419 184 184 184 329 329 329 F F F 21 420-439 194 194 194 345 345 345 F F F 22 440-459 197 197 197 353 353 353 F F F 23 460-479 193 193 193 344 344 344 F F F 24 480-499 194 194 194 347 347 347 F F F 25 500-519 194 194 194 346 346 346 F F F 26 520-539 194 194 194 345 345 345 F F F 27 540-559 191 191 191 341 341 341 F F F 28 560-579 196 196 196 350 350 350 F F F 29 580-599 195 195 195 349 349 349 F F F Max gen: 197, Max flush: 353 Begin loop 99 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.265306107Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1974,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:19:37.308038453Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1974,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.30849491Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1974,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.34020053Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1974,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.340785893Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1974} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"abd5430f1508af0bc008143ea2566277ef714ec2\",\n git_commit_timestamp: \"2025-01-17T20:55:03.000000000Z\",\n git_branch: \"alan/tokio-to-140\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.340819712Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1974} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.34083287Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1974} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.340846978Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1974} {"msg":"Crucible c9f591c0-4528-4e75-9b2a-fad223b5379e has session id: d5170d1b-c9bb-4c15-80ad-fd31d55d99a0","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.341508835Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1974,"session_id":"d5170d1b-c9bb-4c15-80ad-fd31d55d99a0"} {"msg":"Upstairs opts: Upstairs UUID: c9f591c0-4528-4e75-9b2a-fad223b5379e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.341538603Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1974,"session_id":"d5170d1b-c9bb-4c15-80ad-fd31d55d99a0"} {"msg":"Crucible stats registered with UUID: c9f591c0-4528-4e75-9b2a-fad223b5379e","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.341556033Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1974,"session_id":"d5170d1b-c9bb-4c15-80ad-fd31d55d99a0"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:19:37.34156841Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1974,"session_id":"d5170d1b-c9bb-4c15-80ad-fd31d55d99a0"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.34199574Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1974,"task":"crutest"} {"msg":"The guest has requested activation with gen:198","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.342025267Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1974} {"msg":"c9f591c0-4528-4e75-9b2a-fad223b5379e active request set","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.342089554Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1974,"session_id":"d5170d1b-c9bb-4c15-80ad-fd31d55d99a0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.342107215Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1974,"client":"0","":"downstairs","session_id":"d5170d1b-c9bb-4c15-80ad-fd31d55d99a0"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.342129968Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1974,"client":"0","":"downstairs","session_id":"d5170d1b-c9bb-4c15-80ad-fd31d55d99a0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.342143646Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1974,"client":"1","":"downstairs","session_id":"d5170d1b-c9bb-4c15-80ad-fd31d55d99a0"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.342157304Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1974,"client":"1","":"downstairs","session_id":"d5170d1b-c9bb-4c15-80ad-fd31d55d99a0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.342176305Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1974,"client":"2","":"downstairs","session_id":"d5170d1b-c9bb-4c15-80ad-fd31d55d99a0"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.34220186Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1974,"client":"2","":"downstairs","session_id":"d5170d1b-c9bb-4c15-80ad-fd31d55d99a0"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.342226334Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1974,"":"io task","client":"2","":"downstairs","session_id":"d5170d1b-c9bb-4c15-80ad-fd31d55d99a0"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.34225319Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1974,"":"io task","client":"0","":"downstairs","session_id":"d5170d1b-c9bb-4c15-80ad-fd31d55d99a0"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.342279175Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1974,"":"io task","client":"1","":"downstairs","session_id":"d5170d1b-c9bb-4c15-80ad-fd31d55d99a0"} {"msg":"ds_connection connected from Ok(127.0.0.1:59645)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.34230337Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1974,"":"io task","client":"2","":"downstairs","session_id":"d5170d1b-c9bb-4c15-80ad-fd31d55d99a0"} {"msg":"ds_connection connected from Ok(127.0.0.1:46942)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.342330966Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1974,"":"io task","client":"1","":"downstairs","session_id":"d5170d1b-c9bb-4c15-80ad-fd31d55d99a0"} {"msg":"ds_connection connected from Ok(127.0.0.1:57151)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.342357982Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1974,"":"io task","client":"0","":"downstairs","session_id":"d5170d1b-c9bb-4c15-80ad-fd31d55d99a0"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.343019948Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1974,"client":"1","":"downstairs","session_id":"d5170d1b-c9bb-4c15-80ad-fd31d55d99a0"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.343096843Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1974,"client":"2","":"downstairs","session_id":"d5170d1b-c9bb-4c15-80ad-fd31d55d99a0"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.343218995Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1974,"client":"0","":"downstairs","session_id":"d5170d1b-c9bb-4c15-80ad-fd31d55d99a0"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.343737288Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1974,"client":"2","":"downstairs","session_id":"d5170d1b-c9bb-4c15-80ad-fd31d55d99a0"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.343777281Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1974,"client":"1","":"downstairs","session_id":"d5170d1b-c9bb-4c15-80ad-fd31d55d99a0"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.34386049Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1974,"client":"0","":"downstairs","session_id":"d5170d1b-c9bb-4c15-80ad-fd31d55d99a0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.34440861Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1974,"session_id":"d5170d1b-c9bb-4c15-80ad-fd31d55d99a0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.34444253Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1974,"session_id":"d5170d1b-c9bb-4c15-80ad-fd31d55d99a0"} {"msg":"[0]R flush_numbers[0..12]: [353, 353, 353, 347, 352, 350, 350, 350, 352, 345, 353, 344]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.344498142Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1974,"":"downstairs","":"downstairs","session_id":"d5170d1b-c9bb-4c15-80ad-fd31d55d99a0"} {"msg":"[0]R generation[0..12]: [197, 197, 197, 194, 196, 196, 196, 196, 196, 194, 197, 193]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.344525218Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1974,"":"downstairs","":"downstairs","session_id":"d5170d1b-c9bb-4c15-80ad-fd31d55d99a0"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.344541007Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1974,"":"downstairs","":"downstairs","session_id":"d5170d1b-c9bb-4c15-80ad-fd31d55d99a0"} {"msg":"[1]R flush_numbers[0..12]: [353, 353, 353, 347, 352, 350, 350, 350, 352, 345, 353, 344]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.344555236Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1974,"":"downstairs","":"downstairs","session_id":"d5170d1b-c9bb-4c15-80ad-fd31d55d99a0"} {"msg":"[1]R generation[0..12]: [197, 197, 197, 194, 196, 196, 196, 196, 196, 194, 197, 193]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.344572136Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1974,"":"downstairs","":"downstairs","session_id":"d5170d1b-c9bb-4c15-80ad-fd31d55d99a0"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.344586394Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1974,"":"downstairs","":"downstairs","session_id":"d5170d1b-c9bb-4c15-80ad-fd31d55d99a0"} {"msg":"[2]R flush_numbers[0..12]: [353, 353, 353, 347, 352, 350, 350, 350, 352, 345, 353, 344]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.344600012Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1974,"":"downstairs","":"downstairs","session_id":"d5170d1b-c9bb-4c15-80ad-fd31d55d99a0"} {"msg":"[2]R generation[0..12]: [197, 197, 197, 194, 196, 196, 196, 196, 196, 194, 197, 193]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.34461327Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1974,"":"downstairs","":"downstairs","session_id":"d5170d1b-c9bb-4c15-80ad-fd31d55d99a0"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.344626968Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1974,"":"downstairs","":"downstairs","session_id":"d5170d1b-c9bb-4c15-80ad-fd31d55d99a0"} {"msg":"Max found gen is 198","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.344639605Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1974,"":"downstairs","":"downstairs","session_id":"d5170d1b-c9bb-4c15-80ad-fd31d55d99a0"} {"msg":"Generation requested: 198 >= found:198","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.344652352Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1974,"":"downstairs","":"downstairs","session_id":"d5170d1b-c9bb-4c15-80ad-fd31d55d99a0"} {"msg":"Next flush: 354","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.34466483Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1974,"":"downstairs","":"downstairs","session_id":"d5170d1b-c9bb-4c15-80ad-fd31d55d99a0"} {"msg":"All extents match","v":0,"name":"Wait for a query_work_queue command to finish before sending IO crucible","level":30,"time":"2025-01-17T21:19:37.344685442Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1974,"":"downstairs","":"downstairs","session_id":"d5170d1b-c9bb-4c15-80ad-fd31d55d99a0"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.344698339Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1974,"session_id":"d5170d1b-c9bb-4c15-80ad-fd31d55d99a0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.344709916Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1974,"client":"0","":"downstairs","session_id":"d5170d1b-c9bb-4c15-80ad-fd31d55d99a0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.344722954Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1974,"client":"1","":"downstairs","session_id":"d5170d1b-c9bb-4c15-80ad-fd31d55d99a0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.344735641Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1974,"client":"2","":"downstairs","session_id":"d5170d1b-c9bb-4c15-80ad-fd31d55d99a0"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.344748469Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1974,"session_id":"d5170d1b-c9bb-4c15-80ad-fd31d55d99a0"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.344760126Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1974,"session_id":"d5170d1b-c9bb-4c15-80ad-fd31d55d99a0"} {"msg":"c9f591c0-4528-4e75-9b2a-fad223b5379e is now active with session: d5170d1b-c9bb-4c15-80ad-fd31d55d99a0","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.344774224Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1974,"session_id":"d5170d1b-c9bb-4c15-80ad-fd31d55d99a0"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.34478597Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1974,"session_id":"d5170d1b-c9bb-4c15-80ad-fd31d55d99a0"} {"msg":"The guest has finished waiting for activation with:198","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.344798088Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1974} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 559 len 28672 data: 16 21 22 22 22 16 15 02/30 Read block 450 len 20480 03/30 Write block 34 len 4096 data: 22 04/30 Read block 17 len 20480 05/30 Read block 214 len 8192 06/30 Read block 233 len 36864 07/30 Read block 236 len 28672 08/30 Read block 259 len 40960 09/30 Write block 211 len 32768 data: 14 11 14 12 12 13 10 8 10/30 Read block 534 len 28672 11/30 Read block 155 len 8192 12/30 Read block 262 len 20480 13/30 Write block 5 len 28672 data: 17 17 18 19 17 17 18 14/30 Read block 112 len 16384 15/30 Write block 557 len 28672 data: 13 17 17 22 23 23 23 16/30 Read block 502 len 36864 17/30 Write block 541 len 20480 data: 11 11 13 11 12 18/30 Write block 256 len 36864 data: 17 17 15 16 15 14 10 11 8 19/30 Read block 482 len 36864 20/30 Write block 133 len 20480 data: 9 8 10 13 13 21/30 Read block 141 len 20480 22/30 Write block 444 len 8192 data: 10 9 23/30 Flush 24/30 Write block 169 len 40960 data: 14 13 16 13 14 15 17 17 16 15 25/30 Write block 219 len 20480 data: 8 8 10 12 14 26/30 Read block 32 len 24576 27/30 Write block 298 len 40960 data: 11 13 13 15 13 17 17 19 17 14 28/30 Write block 26 len 8192 data: 20 19 29/30 Write block 477 len 4096 data: 14 30/30 Read block 98 len 16384 ---------------------------------------------------------------- Crucible gen:198 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 10 Done Done Done false 1024 Acked Write 5 Done Done Done false 1025 Acked Read 6 Done Done Done false 1026 Acked Write 10 Done Done Done false 1027 Acked Write 2 Done Done Done false 1028 Acked Write 1 Done Done Done false 1029 Acked Read 4 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 7 7 6 20 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:19:37.365748442Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1974} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 198 198 198 354 354 354 F F F 1 020-039 198 198 198 354 354 354 T T T 2 040-059 197 197 197 353 353 353 F F F 3 060-079 194 194 194 347 347 347 F F F 4 080-099 196 196 196 352 352 352 F F F 5 100-119 196 196 196 350 350 350 F F F 6 120-139 198 198 198 354 354 354 F F F 7 140-159 196 196 196 350 350 350 F F F 8 160-179 196 196 196 352 352 352 T T T 9 180-199 194 194 194 345 345 345 F F F 10 200-219 198 198 198 354 354 354 T T T 11 220-239 193 193 193 344 344 344 T T T 12 240-259 198 198 198 354 354 354 F F F 13 260-279 198 198 198 354 354 354 F F F 14 280-299 197 197 197 353 353 353 T T T 15 300-319 195 195 195 349 349 349 T T T 16 320-339 196 196 196 350 350 350 F F F 17 340-359 193 193 193 344 344 344 F F F 18 360-379 191 191 191 341 341 341 F F F 19 380-399 194 194 194 345 345 345 F F F 20 400-419 184 184 184 329 329 329 F F F 21 420-439 194 194 194 345 345 345 F F F 22 440-459 198 198 198 354 354 354 F F F 23 460-479 193 193 193 344 344 344 T T T 24 480-499 194 194 194 347 347 347 F F F 25 500-519 194 194 194 346 346 346 F F F 26 520-539 194 194 194 345 345 345 F F F 27 540-559 198 198 198 354 354 354 F F F 28 560-579 198 198 198 354 354 354 F F F 29 580-599 195 195 195 349 349 349 F F F Max gen: 198, Max flush: 354 A Difference in extent metadata is expected here On loop 99 Verifying data now No change after verify Loop: 99 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 198 198 198 354 354 354 F F F 1 020-039 199 199 199 355 355 355 F F F 2 040-059 197 197 197 353 353 353 F F F 3 060-079 194 194 194 347 347 347 F F F 4 080-099 196 196 196 352 352 352 F F F 5 100-119 196 196 196 350 350 350 F F F 6 120-139 198 198 198 354 354 354 F F F 7 140-159 196 196 196 350 350 350 F F F 8 160-179 199 199 199 355 355 355 F F F 9 180-199 194 194 194 345 345 345 F F F 10 200-219 199 199 199 355 355 355 F F F 11 220-239 199 199 199 355 355 355 F F F 12 240-259 198 198 198 354 354 354 F F F 13 260-279 198 198 198 354 354 354 F F F 14 280-299 199 199 199 355 355 355 F F F 15 300-319 199 199 199 355 355 355 F F F 16 320-339 196 196 196 350 350 350 F F F 17 340-359 193 193 193 344 344 344 F F F 18 360-379 191 191 191 341 341 341 F F F 19 380-399 194 194 194 345 345 345 F F F 20 400-419 184 184 184 329 329 329 F F F 21 420-439 194 194 194 345 345 345 F F F 22 440-459 198 198 198 354 354 354 F F F 23 460-479 199 199 199 355 355 355 F F F 24 480-499 194 194 194 347 347 347 F F F 25 500-519 194 194 194 346 346 346 F F F 26 520-539 194 194 194 345 345 345 F F F 27 540-559 198 198 198 354 354 354 F F F 28 560-579 198 198 198 354 354 354 F F F 29 580-599 195 195 195 349 349 349 F F F Max gen: 199, Max flush: 355 8:44 Test duration Test completed real 8:43.967399808 user 48.553752911 sys 18.717355876 trap 0.063951543 tflt 0.085163379 dflt 0.129860063 kflt 0.007064549 lock 14:01:10.043744254 slp 42:40.651992937 lat 5.592185442 stop 0.060902477