Turn off color for downstairs dump Creating region directory at: /var/tmp/test_repair Creating dsc directory at: /tmp/dsc Downstairs region 0 created at /var/tmp/test_repair/8810 in 0.27715978 Downstairs region 1 created at /var/tmp/test_repair/8820 in 0.019747209 Downstairs region 2 created at /var/tmp/test_repair/8830 in 0.019190365 Created 3 regions /var/tmp/bins/crutest with -t 127.0.0.1:8810 -t 127.0.0.1:8820 -t 127.0.0.1:8830 1184 1185 1186 Dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:11.604794627Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1188,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:11.67740937Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1188,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:11.678653734Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1188} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"26e7d501b33f47379e53601720353c2983e7654b\",\n git_commit_timestamp: \"2024-12-12T20:16:40.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:11.67870653Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1188} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:11.678721299Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1188} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:11.678735668Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1188} {"msg":"Crucible 51b42ba4-7714-4a37-982c-2df21afd2f77 has session id: d29e37a7-31ee-4bfe-b5b8-4ed938f92ca7","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:11.679504395Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1188,"session_id":"d29e37a7-31ee-4bfe-b5b8-4ed938f92ca7"} {"msg":"Upstairs opts: Upstairs UUID: 51b42ba4-7714-4a37-982c-2df21afd2f77, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:11.67957339Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1188,"session_id":"d29e37a7-31ee-4bfe-b5b8-4ed938f92ca7"} {"msg":"Crucible stats registered with UUID: 51b42ba4-7714-4a37-982c-2df21afd2f77","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:11.679601169Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1188,"session_id":"d29e37a7-31ee-4bfe-b5b8-4ed938f92ca7"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:11.713385459Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1188,"task":"crutest"} {"msg":"The guest has requested activation with gen:1","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:11.713424906Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1188} {"msg":"51b42ba4-7714-4a37-982c-2df21afd2f77 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:11.713454584Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1188,"session_id":"d29e37a7-31ee-4bfe-b5b8-4ed938f92ca7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:11.713479112Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1188,"client":"0","":"downstairs","session_id":"d29e37a7-31ee-4bfe-b5b8-4ed938f92ca7"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:11.713556927Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1188,"client":"0","":"downstairs","session_id":"d29e37a7-31ee-4bfe-b5b8-4ed938f92ca7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:11.713582155Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1188,"client":"1","":"downstairs","session_id":"d29e37a7-31ee-4bfe-b5b8-4ed938f92ca7"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:11.713605364Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1188,"client":"1","":"downstairs","session_id":"d29e37a7-31ee-4bfe-b5b8-4ed938f92ca7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:11.713674709Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1188,"client":"2","":"downstairs","session_id":"d29e37a7-31ee-4bfe-b5b8-4ed938f92ca7"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:11.713713236Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1188,"client":"2","":"downstairs","session_id":"d29e37a7-31ee-4bfe-b5b8-4ed938f92ca7"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:11.713739624Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1188,"":"io task","client":"2","":"downstairs","session_id":"d29e37a7-31ee-4bfe-b5b8-4ed938f92ca7"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:11.713765103Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1188,"":"io task","client":"0","":"downstairs","session_id":"d29e37a7-31ee-4bfe-b5b8-4ed938f92ca7"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:11.713789861Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1188,"":"io task","client":"1","":"downstairs","session_id":"d29e37a7-31ee-4bfe-b5b8-4ed938f92ca7"} {"msg":"ds_connection connected from Ok(127.0.0.1:56765)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:11.713925222Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1188,"":"io task","client":"2","":"downstairs","session_id":"d29e37a7-31ee-4bfe-b5b8-4ed938f92ca7"} {"msg":"ds_connection connected from Ok(127.0.0.1:54100)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:11.713963529Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1188,"":"io task","client":"0","":"downstairs","session_id":"d29e37a7-31ee-4bfe-b5b8-4ed938f92ca7"} {"msg":"ds_connection connected from Ok(127.0.0.1:50478)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:11.713990567Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1188,"":"io task","client":"1","":"downstairs","session_id":"d29e37a7-31ee-4bfe-b5b8-4ed938f92ca7"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:11.715639193Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1188,"client":"0","":"downstairs","session_id":"d29e37a7-31ee-4bfe-b5b8-4ed938f92ca7"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:11.715675931Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1188,"client":"1","":"downstairs","session_id":"d29e37a7-31ee-4bfe-b5b8-4ed938f92ca7"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:11.715705679Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1188,"client":"2","":"downstairs","session_id":"d29e37a7-31ee-4bfe-b5b8-4ed938f92ca7"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:11.716743537Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1188,"client":"1","":"downstairs","session_id":"d29e37a7-31ee-4bfe-b5b8-4ed938f92ca7"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:11.716785534Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1188,"client":"2","":"downstairs","session_id":"d29e37a7-31ee-4bfe-b5b8-4ed938f92ca7"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:11.716831601Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1188,"client":"0","":"downstairs","session_id":"d29e37a7-31ee-4bfe-b5b8-4ed938f92ca7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:11.717669724Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1188,"session_id":"d29e37a7-31ee-4bfe-b5b8-4ed938f92ca7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:11.717695752Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1188,"session_id":"d29e37a7-31ee-4bfe-b5b8-4ed938f92ca7"} {"msg":"[0]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:11.717882279Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1188,"":"downstairs","":"downstairs","session_id":"d29e37a7-31ee-4bfe-b5b8-4ed938f92ca7"} {"msg":"[0]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:11.717901937Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1188,"":"downstairs","":"downstairs","session_id":"d29e37a7-31ee-4bfe-b5b8-4ed938f92ca7"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:11.717916347Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1188,"":"downstairs","":"downstairs","session_id":"d29e37a7-31ee-4bfe-b5b8-4ed938f92ca7"} {"msg":"[1]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:11.71800506Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1188,"":"downstairs","":"downstairs","session_id":"d29e37a7-31ee-4bfe-b5b8-4ed938f92ca7"} {"msg":"[1]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:11.718023249Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1188,"":"downstairs","":"downstairs","session_id":"d29e37a7-31ee-4bfe-b5b8-4ed938f92ca7"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:11.718037168Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1188,"":"downstairs","":"downstairs","session_id":"d29e37a7-31ee-4bfe-b5b8-4ed938f92ca7"} {"msg":"[2]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:11.718050887Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1188,"":"downstairs","":"downstairs","session_id":"d29e37a7-31ee-4bfe-b5b8-4ed938f92ca7"} {"msg":"[2]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:11.718064396Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1188,"":"downstairs","":"downstairs","session_id":"d29e37a7-31ee-4bfe-b5b8-4ed938f92ca7"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:11.718078235Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1188,"":"downstairs","":"downstairs","session_id":"d29e37a7-31ee-4bfe-b5b8-4ed938f92ca7"} {"msg":"Max found gen is 1","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:11.718094524Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1188,"":"downstairs","":"downstairs","session_id":"d29e37a7-31ee-4bfe-b5b8-4ed938f92ca7"} {"msg":"Generation requested: 1 >= found:1","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:11.718108063Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1188,"":"downstairs","":"downstairs","session_id":"d29e37a7-31ee-4bfe-b5b8-4ed938f92ca7"} {"msg":"Next flush: 1","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:11.718121062Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1188,"":"downstairs","":"downstairs","session_id":"d29e37a7-31ee-4bfe-b5b8-4ed938f92ca7"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:11.718265362Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1188,"":"downstairs","":"downstairs","session_id":"d29e37a7-31ee-4bfe-b5b8-4ed938f92ca7"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:11.718283361Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1188,"session_id":"d29e37a7-31ee-4bfe-b5b8-4ed938f92ca7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:11.71829565Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1188,"client":"0","":"downstairs","session_id":"d29e37a7-31ee-4bfe-b5b8-4ed938f92ca7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:11.718308999Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1188,"client":"1","":"downstairs","session_id":"d29e37a7-31ee-4bfe-b5b8-4ed938f92ca7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:11.718321988Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1188,"client":"2","":"downstairs","session_id":"d29e37a7-31ee-4bfe-b5b8-4ed938f92ca7"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:11.718334887Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1188,"session_id":"d29e37a7-31ee-4bfe-b5b8-4ed938f92ca7"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:11.718347187Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1188,"session_id":"d29e37a7-31ee-4bfe-b5b8-4ed938f92ca7"} {"msg":"51b42ba4-7714-4a37-982c-2df21afd2f77 is now active with session: d29e37a7-31ee-4bfe-b5b8-4ed938f92ca7","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:11.718360176Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1188,"session_id":"d29e37a7-31ee-4bfe-b5b8-4ed938f92ca7"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:11.718372255Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1188,"session_id":"d29e37a7-31ee-4bfe-b5b8-4ed938f92ca7"} {"msg":"The guest has finished waiting for activation with:1","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:11.718384524Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1188} Wait for a query_work_queue command to finish before sending IO Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Fill test Read and Verify all blocks (0..600 range:false) Wrote out file "/tmp/test_repair_verify.data" CLIENT: Tests done. All submitted work has been ACK'd ---------------------------------------------------------------- Crucible gen:1 GIO:true work queues: Upstairs:0 downstairs:3 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1004 Acked Read 88 Done Done Done false 1005 Acked Read 256 Done Done Done false 1006 Acked Read 256 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1003 1003 1003 Downstairs last five completed: 1003 1002 1001 1000 Upstairs last five completed: 1006 1005 1004 1003 1002 CLIENT: Up:0 ds:3 act:3 ---------------------------------------------------------------- Crucible gen:1 GIO:true work queues: Upstairs:0 downstairs:0 Downstairs last five completed: 1007 1006 1005 1004 1003 Upstairs last five completed: 1007 1006 1005 1004 1003 CLIENT: Up:0 ds:0 act:3 CLIENT: All crucible jobs finished, exiting program {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:33:15.813570286Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1188} {"msg":"client stopping due to Err(RecvError(()))","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:15.813662579Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1188,"":"io task","client":"0","":"downstairs","session_id":"d29e37a7-31ee-4bfe-b5b8-4ed938f92ca7"} {"msg":"client task is sending Done(QueueClosed)","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:33:15.813724865Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1188,"":"io task","client":"0","":"downstairs","session_id":"d29e37a7-31ee-4bfe-b5b8-4ed938f92ca7"} {"msg":"client task could not reply to main task; shutting down?","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:33:15.813765702Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1188,"":"io task","client":"0","":"downstairs","session_id":"d29e37a7-31ee-4bfe-b5b8-4ed938f92ca7"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:15.81380217Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1188,"":"io task","client":"0","":"downstairs","session_id":"d29e37a7-31ee-4bfe-b5b8-4ed938f92ca7"} Begin loop 1 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:15.849136763Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1190,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:33:15.896251074Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1190,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:15.896664786Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1190,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:15.928836257Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1190,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:15.929419317Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1190} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"26e7d501b33f47379e53601720353c2983e7654b\",\n git_commit_timestamp: \"2024-12-12T20:16:40.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:15.929459504Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1190} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:15.929476793Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1190} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:15.929494582Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1190} {"msg":"Crucible d88d26d6-f78d-4c88-bd9a-81ddd054fc79 has session id: 1c5516a3-001b-4543-92b4-5b99a39abc30","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:15.929979008Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1190,"session_id":"1c5516a3-001b-4543-92b4-5b99a39abc30"} {"msg":"Upstairs opts: Upstairs UUID: d88d26d6-f78d-4c88-bd9a-81ddd054fc79, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:15.930034415Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1190,"session_id":"1c5516a3-001b-4543-92b4-5b99a39abc30"} {"msg":"Crucible stats registered with UUID: d88d26d6-f78d-4c88-bd9a-81ddd054fc79","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:15.930064482Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1190,"session_id":"1c5516a3-001b-4543-92b4-5b99a39abc30"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:15.961048796Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1190,"task":"crutest"} {"msg":"The guest has requested activation with gen:2","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:15.961079084Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1190} {"msg":"d88d26d6-f78d-4c88-bd9a-81ddd054fc79 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:15.961153049Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1190,"session_id":"1c5516a3-001b-4543-92b4-5b99a39abc30"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:15.961174567Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1190,"client":"0","":"downstairs","session_id":"1c5516a3-001b-4543-92b4-5b99a39abc30"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:15.961201406Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1190,"client":"0","":"downstairs","session_id":"1c5516a3-001b-4543-92b4-5b99a39abc30"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:15.961218824Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1190,"client":"1","":"downstairs","session_id":"1c5516a3-001b-4543-92b4-5b99a39abc30"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:15.961236563Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1190,"client":"1","":"downstairs","session_id":"1c5516a3-001b-4543-92b4-5b99a39abc30"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:15.961252882Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1190,"client":"2","":"downstairs","session_id":"1c5516a3-001b-4543-92b4-5b99a39abc30"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:15.961270181Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1190,"client":"2","":"downstairs","session_id":"1c5516a3-001b-4543-92b4-5b99a39abc30"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:15.9612866Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1190,"":"io task","client":"2","":"downstairs","session_id":"1c5516a3-001b-4543-92b4-5b99a39abc30"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:15.961310948Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1190,"":"io task","client":"0","":"downstairs","session_id":"1c5516a3-001b-4543-92b4-5b99a39abc30"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:15.961348835Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1190,"":"io task","client":"1","":"downstairs","session_id":"1c5516a3-001b-4543-92b4-5b99a39abc30"} {"msg":"ds_connection connected from Ok(127.0.0.1:61574)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:15.961383723Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1190,"":"io task","client":"1","":"downstairs","session_id":"1c5516a3-001b-4543-92b4-5b99a39abc30"} {"msg":"ds_connection connected from Ok(127.0.0.1:39327)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:15.961416651Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1190,"":"io task","client":"0","":"downstairs","session_id":"1c5516a3-001b-4543-92b4-5b99a39abc30"} {"msg":"ds_connection connected from Ok(127.0.0.1:47547)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:15.961477606Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1190,"":"io task","client":"2","":"downstairs","session_id":"1c5516a3-001b-4543-92b4-5b99a39abc30"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:15.96215341Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1190,"client":"1","":"downstairs","session_id":"1c5516a3-001b-4543-92b4-5b99a39abc30"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:15.962195477Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1190,"client":"2","":"downstairs","session_id":"1c5516a3-001b-4543-92b4-5b99a39abc30"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:15.962228645Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1190,"client":"0","":"downstairs","session_id":"1c5516a3-001b-4543-92b4-5b99a39abc30"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:15.962823304Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1190,"client":"1","":"downstairs","session_id":"1c5516a3-001b-4543-92b4-5b99a39abc30"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:15.962865111Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1190,"client":"2","":"downstairs","session_id":"1c5516a3-001b-4543-92b4-5b99a39abc30"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:15.962897629Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1190,"client":"0","":"downstairs","session_id":"1c5516a3-001b-4543-92b4-5b99a39abc30"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:15.963425002Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1190,"session_id":"1c5516a3-001b-4543-92b4-5b99a39abc30"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:15.96345119Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1190,"session_id":"1c5516a3-001b-4543-92b4-5b99a39abc30"} {"msg":"[0]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:15.963527435Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1190,"":"downstairs","":"downstairs","session_id":"1c5516a3-001b-4543-92b4-5b99a39abc30"} {"msg":"[0]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:15.963546504Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1190,"":"downstairs","":"downstairs","session_id":"1c5516a3-001b-4543-92b4-5b99a39abc30"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:15.963561383Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1190,"":"downstairs","":"downstairs","session_id":"1c5516a3-001b-4543-92b4-5b99a39abc30"} {"msg":"[1]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:15.963579571Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1190,"":"downstairs","":"downstairs","session_id":"1c5516a3-001b-4543-92b4-5b99a39abc30"} {"msg":"[1]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:15.96359347Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1190,"":"downstairs","":"downstairs","session_id":"1c5516a3-001b-4543-92b4-5b99a39abc30"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:15.963607289Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1190,"":"downstairs","":"downstairs","session_id":"1c5516a3-001b-4543-92b4-5b99a39abc30"} {"msg":"[2]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:15.963620729Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1190,"":"downstairs","":"downstairs","session_id":"1c5516a3-001b-4543-92b4-5b99a39abc30"} {"msg":"[2]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:15.963634078Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1190,"":"downstairs","":"downstairs","session_id":"1c5516a3-001b-4543-92b4-5b99a39abc30"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:15.963647897Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1190,"":"downstairs","":"downstairs","session_id":"1c5516a3-001b-4543-92b4-5b99a39abc30"} {"msg":"Max found gen is 2","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:15.963660846Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1190,"":"downstairs","":"downstairs","session_id":"1c5516a3-001b-4543-92b4-5b99a39abc30"} {"msg":"Generation requested: 2 >= found:2","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:15.963673785Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1190,"":"downstairs","":"downstairs","session_id":"1c5516a3-001b-4543-92b4-5b99a39abc30"} {"msg":"Next flush: 2","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:15.963686584Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1190,"":"downstairs","":"downstairs","session_id":"1c5516a3-001b-4543-92b4-5b99a39abc30"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-12-12T20:33:15.963699503Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1190,"":"downstairs","":"downstairs","session_id":"1c5516a3-001b-4543-92b4-5b99a39abc30"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:15.963716822Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1190,"session_id":"1c5516a3-001b-4543-92b4-5b99a39abc30"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:15.963728701Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1190,"client":"0","":"downstairs","session_id":"1c5516a3-001b-4543-92b4-5b99a39abc30"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:15.96374253Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1190,"client":"1","":"downstairs","session_id":"1c5516a3-001b-4543-92b4-5b99a39abc30"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:15.963755789Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1190,"client":"2","":"downstairs","session_id":"1c5516a3-001b-4543-92b4-5b99a39abc30"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:15.963768838Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1190,"session_id":"1c5516a3-001b-4543-92b4-5b99a39abc30"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:15.963782917Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1190,"session_id":"1c5516a3-001b-4543-92b4-5b99a39abc30"} {"msg":"d88d26d6-f78d-4c88-bd9a-81ddd054fc79 is now active with session: 1c5516a3-001b-4543-92b4-5b99a39abc30","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:15.963795736Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1190,"session_id":"1c5516a3-001b-4543-92b4-5b99a39abc30"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:15.963807626Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1190,"session_id":"1c5516a3-001b-4543-92b4-5b99a39abc30"} {"msg":"The guest has finished waiting for activation with:2","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:15.963819755Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1190} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 87 len 32768 data: 2 2 2 2 2 2 2 2 02/30 Write block 45 len 40960 data: 2 2 2 2 2 2 2 2 2 2 03/30 Write block 500 len 4096 data: 2 04/30 Write block 424 len 24576 data: 2 2 2 2 2 2 05/30 Read block 457 len 16384 06/30 Read block 266 len 8192 07/30 Read block 46 len 24576 08/30 Write block 416 len 40960 data: 2 2 2 2 2 2 2 2 3 3 09/30 Write block 162 len 16384 data: 2 2 2 2 10/30 Read block 541 len 32768 11/30 Read block 79 len 24576 12/30 Read block 404 len 12288 13/30 Write block 474 len 24576 data: 2 2 2 2 2 2 14/30 Flush 15/30 Write block 389 len 28672 data: 2 2 2 2 2 2 2 16/30 Read block 142 len 8192 17/30 Read block 325 len 36864 18/30 Read block 448 len 40960 19/30 Read block 516 len 12288 20/30 Flush 21/30 Write block 144 len 20480 data: 2 2 2 2 2 22/30 Read block 138 len 32768 23/30 Read block 174 len 20480 24/30 Flush 25/30 Write block 99 len 28672 data: 2 2 2 2 2 2 2 26/30 Read block 103 len 24576 27/30 Read block 131 len 4096 28/30 Read block 207 len 40960 29/30 Read block 1 len 4096 30/30 Read block 221 len 32768 ---------------------------------------------------------------- Crucible gen:2 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 7 Done Done Done false 1025 Acked Read 6 Done Done Done false 1026 Acked Read 1 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Read 1 Done Done Done false 1029 Acked Read 8 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 1 2 Done 5 6 5 16 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:33:15.984504428Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1190} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 1 1 1 F F F 1 020-039 1 1 1 1 1 1 F F F 2 040-059 2 2 2 2 2 2 F F F 3 060-079 1 1 1 1 1 1 F F F 4 080-099 2 2 2 2 2 2 T T T 5 100-119 1 1 1 1 1 1 T T T 6 120-139 1 1 1 1 1 1 F F F 7 140-159 2 2 2 4 4 4 F F F 8 160-179 2 2 2 2 2 2 F F F 9 180-199 1 1 1 1 1 1 F F F 10 200-219 1 1 1 1 1 1 F F F 11 220-239 1 1 1 1 1 1 F F F 12 240-259 1 1 1 1 1 1 F F F 13 260-279 1 1 1 1 1 1 F F F 14 280-299 1 1 1 1 1 1 F F F 15 300-319 1 1 1 1 1 1 F F F 16 320-339 1 1 1 1 1 1 F F F 17 340-359 1 1 1 1 1 1 F F F 18 360-379 1 1 1 1 1 1 F F F 19 380-399 2 2 2 3 3 3 F F F 20 400-419 2 2 2 2 2 2 F F F 21 420-439 2 2 2 2 2 2 F F F 22 440-459 1 1 1 1 1 1 F F F 23 460-479 2 2 2 2 2 2 F F F 24 480-499 1 1 1 1 1 1 F F F 25 500-519 2 2 2 2 2 2 F F F 26 520-539 1 1 1 1 1 1 F F F 27 540-559 1 1 1 1 1 1 F F F 28 560-579 1 1 1 1 1 1 F F F 29 580-599 1 1 1 1 1 1 F F F Max gen: 2, Max flush: 4 A Difference in extent metadata is expected here On loop 1 Verifying data now No change after verify Loop: 1 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 1 1 1 F F F 1 020-039 1 1 1 1 1 1 F F F 2 040-059 2 2 2 2 2 2 F F F 3 060-079 1 1 1 1 1 1 F F F 4 080-099 3 3 3 5 5 5 F F F 5 100-119 3 3 3 5 5 5 F F F 6 120-139 1 1 1 1 1 1 F F F 7 140-159 2 2 2 4 4 4 F F F 8 160-179 2 2 2 2 2 2 F F F 9 180-199 1 1 1 1 1 1 F F F 10 200-219 1 1 1 1 1 1 F F F 11 220-239 1 1 1 1 1 1 F F F 12 240-259 1 1 1 1 1 1 F F F 13 260-279 1 1 1 1 1 1 F F F 14 280-299 1 1 1 1 1 1 F F F 15 300-319 1 1 1 1 1 1 F F F 16 320-339 1 1 1 1 1 1 F F F 17 340-359 1 1 1 1 1 1 F F F 18 360-379 1 1 1 1 1 1 F F F 19 380-399 2 2 2 3 3 3 F F F 20 400-419 2 2 2 2 2 2 F F F 21 420-439 2 2 2 2 2 2 F F F 22 440-459 1 1 1 1 1 1 F F F 23 460-479 2 2 2 2 2 2 F F F 24 480-499 1 1 1 1 1 1 F F F 25 500-519 2 2 2 2 2 2 F F F 26 520-539 1 1 1 1 1 1 F F F 27 540-559 1 1 1 1 1 1 F F F 28 560-579 1 1 1 1 1 1 F F F 29 580-599 1 1 1 1 1 1 F F F Max gen: 3, Max flush: 5 Begin loop 2 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:20.289723608Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1198,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:20.33464721Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1198,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:20.335252629Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1198} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"26e7d501b33f47379e53601720353c2983e7654b\",\n git_commit_timestamp: \"2024-12-12T20:16:40.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:20.335287466Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1198} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:20.335305205Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1198} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:20.335319734Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1198} {"msg":"Crucible 913523de-a7c1-41e0-9587-7cce3b8f3004 has session id: a3fbd428-d925-44aa-818e-aff470caf04a","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:20.335746994Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1198,"session_id":"a3fbd428-d925-44aa-818e-aff470caf04a"} {"msg":"Upstairs opts: Upstairs UUID: 913523de-a7c1-41e0-9587-7cce3b8f3004, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:20.335778922Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1198,"session_id":"a3fbd428-d925-44aa-818e-aff470caf04a"} {"msg":"Crucible stats registered with UUID: 913523de-a7c1-41e0-9587-7cce3b8f3004","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:20.335793991Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1198,"session_id":"a3fbd428-d925-44aa-818e-aff470caf04a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:20.367910906Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1198,"task":"crutest"} {"msg":"The guest has requested activation with gen:4","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:20.367954723Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1198} {"msg":"913523de-a7c1-41e0-9587-7cce3b8f3004 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:20.368024229Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1198,"session_id":"a3fbd428-d925-44aa-818e-aff470caf04a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:20.368058316Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1198,"client":"0","":"downstairs","session_id":"a3fbd428-d925-44aa-818e-aff470caf04a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:20.368089944Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1198,"client":"0","":"downstairs","session_id":"a3fbd428-d925-44aa-818e-aff470caf04a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:20.368114663Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1198,"client":"1","":"downstairs","session_id":"a3fbd428-d925-44aa-818e-aff470caf04a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:20.36814806Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1198,"client":"1","":"downstairs","session_id":"a3fbd428-d925-44aa-818e-aff470caf04a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:20.368181478Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1198,"client":"2","":"downstairs","session_id":"a3fbd428-d925-44aa-818e-aff470caf04a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:20.368217755Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1198,"client":"2","":"downstairs","session_id":"a3fbd428-d925-44aa-818e-aff470caf04a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:20.368252053Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1198,"":"io task","client":"0","":"downstairs","session_id":"a3fbd428-d925-44aa-818e-aff470caf04a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:20.368285231Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1198,"":"io task","client":"2","":"downstairs","session_id":"a3fbd428-d925-44aa-818e-aff470caf04a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:20.368410262Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1198,"":"io task","client":"1","":"downstairs","session_id":"a3fbd428-d925-44aa-818e-aff470caf04a"} {"msg":"ds_connection connected from Ok(127.0.0.1:36705)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:20.368456159Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1198,"":"io task","client":"2","":"downstairs","session_id":"a3fbd428-d925-44aa-818e-aff470caf04a"} {"msg":"ds_connection connected from Ok(127.0.0.1:49488)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:20.368494606Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1198,"":"io task","client":"1","":"downstairs","session_id":"a3fbd428-d925-44aa-818e-aff470caf04a"} {"msg":"ds_connection connected from Ok(127.0.0.1:35633)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:20.368521314Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1198,"":"io task","client":"0","":"downstairs","session_id":"a3fbd428-d925-44aa-818e-aff470caf04a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:20.369053438Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1198,"client":"0","":"downstairs","session_id":"a3fbd428-d925-44aa-818e-aff470caf04a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:20.369209857Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1198,"client":"1","":"downstairs","session_id":"a3fbd428-d925-44aa-818e-aff470caf04a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:20.369241535Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1198,"client":"2","":"downstairs","session_id":"a3fbd428-d925-44aa-818e-aff470caf04a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:20.369721742Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1198,"client":"0","":"downstairs","session_id":"a3fbd428-d925-44aa-818e-aff470caf04a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:20.36975231Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1198,"client":"2","":"downstairs","session_id":"a3fbd428-d925-44aa-818e-aff470caf04a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:20.369907089Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1198,"client":"1","":"downstairs","session_id":"a3fbd428-d925-44aa-818e-aff470caf04a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:20.370270444Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1198,"session_id":"a3fbd428-d925-44aa-818e-aff470caf04a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:20.370350228Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1198,"session_id":"a3fbd428-d925-44aa-818e-aff470caf04a"} {"msg":"[0]R flush_numbers[0..12]: [1, 1, 2, 1, 5, 5, 1, 4, 2, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:20.37047049Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1198,"":"downstairs","":"downstairs","session_id":"a3fbd428-d925-44aa-818e-aff470caf04a"} {"msg":"[0]R generation[0..12]: [1, 1, 2, 1, 3, 3, 1, 2, 2, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:20.370490409Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1198,"":"downstairs","":"downstairs","session_id":"a3fbd428-d925-44aa-818e-aff470caf04a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:20.370504828Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1198,"":"downstairs","":"downstairs","session_id":"a3fbd428-d925-44aa-818e-aff470caf04a"} {"msg":"[1]R flush_numbers[0..12]: [1, 1, 2, 1, 5, 5, 1, 4, 2, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:20.370518237Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1198,"":"downstairs","":"downstairs","session_id":"a3fbd428-d925-44aa-818e-aff470caf04a"} {"msg":"[1]R generation[0..12]: [1, 1, 2, 1, 3, 3, 1, 2, 2, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:20.370531646Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1198,"":"downstairs","":"downstairs","session_id":"a3fbd428-d925-44aa-818e-aff470caf04a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:20.370545355Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1198,"":"downstairs","":"downstairs","session_id":"a3fbd428-d925-44aa-818e-aff470caf04a"} {"msg":"[2]R flush_numbers[0..12]: [1, 1, 2, 1, 5, 5, 1, 4, 2, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:20.370558744Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1198,"":"downstairs","":"downstairs","session_id":"a3fbd428-d925-44aa-818e-aff470caf04a"} {"msg":"[2]R generation[0..12]: [1, 1, 2, 1, 3, 3, 1, 2, 2, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:20.370571953Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1198,"":"downstairs","":"downstairs","session_id":"a3fbd428-d925-44aa-818e-aff470caf04a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:20.370585792Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1198,"":"downstairs","":"downstairs","session_id":"a3fbd428-d925-44aa-818e-aff470caf04a"} {"msg":"Max found gen is 4","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:20.370598971Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1198,"":"downstairs","":"downstairs","session_id":"a3fbd428-d925-44aa-818e-aff470caf04a"} {"msg":"Generation requested: 4 >= found:4","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:20.37061234Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1198,"":"downstairs","":"downstairs","session_id":"a3fbd428-d925-44aa-818e-aff470caf04a"} {"msg":"Next flush: 6","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:20.370625109Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1198,"":"downstairs","":"downstairs","session_id":"a3fbd428-d925-44aa-818e-aff470caf04a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:20.370637869Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1198,"":"downstairs","":"downstairs","session_id":Wait for a query_work_queue command to finish before sending IO "a3fbd428-d925-44aa-818e-aff470caf04a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:20.370656377Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1198,"session_id":"a3fbd428-d925-44aa-818e-aff470caf04a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:20.370668376Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1198,"client":"0","":"downstairs","session_id":"a3fbd428-d925-44aa-818e-aff470caf04a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:20.370683895Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1198,"client":"1","":"downstairs","session_id":"a3fbd428-d925-44aa-818e-aff470caf04a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:20.370697244Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1198,"client":"2","":"downstairs","session_id":"a3fbd428-d925-44aa-818e-aff470caf04a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:20.370710344Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1198,"session_id":"a3fbd428-d925-44aa-818e-aff470caf04a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:20.370722363Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1198,"session_id":"a3fbd428-d925-44aa-818e-aff470caf04a"} {"msg":"913523de-a7c1-41e0-9587-7cce3b8f3004 is now active with session: a3fbd428-d925-44aa-818e-aff470caf04a","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:20.370734852Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1198,"session_id":"a3fbd428-d925-44aa-818e-aff470caf04a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:20.370746621Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1198,"session_id":"a3fbd428-d925-44aa-818e-aff470caf04a"} {"msg":"The guest has finished waiting for activation with:4","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:20.37075879Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1198} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 105 len 40960 data: 3 2 2 2 2 2 2 2 2 2 02/30 Write block 219 len 12288 data: 2 2 2 03/30 Write block 296 len 32768 data: 2 2 2 2 2 2 2 2 04/30 Read block 234 len 16384 05/30 Read block 500 len 32768 06/30 Read block 5 len 40960 07/30 Read block 392 len 8192 08/30 Read block 596 len 16384 09/30 Read block 389 len 28672 10/30 Write block 551 len 20480 data: 2 2 2 2 2 11/30 Write block 404 len 28672 data: 2 2 2 2 2 2 2 12/30 Write block 178 len 8192 data: 2 2 13/30 Write block 281 len 24576 data: 2 2 2 2 2 2 14/30 Flush 15/30 Write block 489 len 12288 data: 2 2 2 16/30 Read block 541 len 16384 17/30 Write block 188 len 40960 data: 2 2 2 2 2 2 2 2 2 2 18/30 Read block 254 len 40960 19/30 Read block 234 len 32768 20/30 Write block 311 len 24576 data: 2 2 2 2 2 2 21/30 Read block 455 len 32768 22/30 Read block 543 len 12288 23/30 Write block 338 len 8192 data: 2 2 24/30 Write block 311 len 32768 data: 3 3 3 3 3 3 2 2 25/30 Read block 106 len 32768 26/30 Read block 480 len 36864 27/30 Read block 205 len 12288 28/30 Write block 360 len 32768 data: 2 2 2 2 2 2 2 2 29/30 Read block 59 len 24576 30/30 Read block 364 len 32768 ---------------------------------------------------------------- Crucible gen:4 GIO:true work queues: Upstairs:0 downstairs:16 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1014 Acked Write 3 Done Done Done false 1015 Acked Read 4 Done Done Done false 1016 Acked Write 10 Done Done Done false 1017 Acked Read 10 Done Done Done false 1018 Acked Read 8 Done Done Done false 1019 Acked Write 6 Done Done Done false 1020 Acked Read 8 Done Done Done false 1021 Acked Read 3 Done Done Done false 1022 Acked Write 2 Done Done Done false 1023 Acked Write 8 Done Done Done false 1024 Acked Read 8 Done Done Done false 1025 Acked Read 9 Done Done Done false 1026 Acked Read 3 Done Done Done false 1027 Acked Write 8 Done Done Done false 1028 Acked Read 6 Done Done Done false 1029 Acked Read 8 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 0 1 Done 16 15 16 47 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1013 1013 1013 Downstairs last five completed: 1013 1012 1011 1010 1009 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:33:20.390728303Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1198} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 1 1 1 F F F 1 020-039 1 1 1 1 1 1 F F F 2 040-059 2 2 2 2 2 2 F F F 3 060-079 1 1 1 1 1 1 F F F 4 080-099 3 3 3 5 5 5 F F F 5 100-119 4 4 4 6 6 6 F F F 6 120-139 1 1 1 1 1 1 F F F 7 140-159 2 2 2 4 4 4 F F F 8 160-179 4 4 4 6 6 6 F F F 9 180-199 1 1 1 1 1 1 T T T 10 200-219 4 4 4 6 6 6 F F F 11 220-239 4 4 4 6 6 6 F F F 12 240-259 1 1 1 1 1 1 F F F 13 260-279 1 1 1 1 1 1 F F F 14 280-299 4 4 4 6 6 6 F F F 15 300-319 4 4 4 6 6 6 T T T 16 320-339 1 1 1 1 1 1 T T T 17 340-359 1 1 1 1 1 1 F F F 18 360-379 1 1 1 1 1 1 T T T 19 380-399 2 2 2 3 3 3 F F F 20 400-419 4 4 4 6 6 6 F F F 21 420-439 2 2 2 2 2 2 F F F 22 440-459 1 1 1 1 1 1 F F F 23 460-479 2 2 2 2 2 2 F F F 24 480-499 1 1 1 1 1 1 T T T 25 500-519 2 2 2 2 2 2 F F F 26 520-539 1 1 1 1 1 1 F F F 27 540-559 4 4 4 6 6 6 F F F 28 560-579 1 1 1 1 1 1 F F F 29 580-599 1 1 1 1 1 1 F F F Max gen: 4, Max flush: 6 A Difference in extent metadata is expected here On loop 2 Verifying data now No change after verify Loop: 2 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 1 1 1 F F F 1 020-039 1 1 1 1 1 1 F F F 2 040-059 2 2 2 2 2 2 F F F 3 060-079 1 1 1 1 1 1 F F F 4 080-099 3 3 3 5 5 5 F F F 5 100-119 4 4 4 6 6 6 F F F 6 120-139 1 1 1 1 1 1 F F F 7 140-159 2 2 2 4 4 4 F F F 8 160-179 4 4 4 6 6 6 F F F 9 180-199 5 5 5 7 7 7 F F F 10 200-219 4 4 4 6 6 6 F F F 11 220-239 4 4 4 6 6 6 F F F 12 240-259 1 1 1 1 1 1 F F F 13 260-279 1 1 1 1 1 1 F F F 14 280-299 4 4 4 6 6 6 F F F 15 300-319 5 5 5 7 7 7 F F F 16 320-339 5 5 5 7 7 7 F F F 17 340-359 1 1 1 1 1 1 F F F 18 360-379 5 5 5 7 7 7 F F F 19 380-399 2 2 2 3 3 3 F F F 20 400-419 4 4 4 6 6 6 F F F 21 420-439 2 2 2 2 2 2 F F F 22 440-459 1 1 1 1 1 1 F F F 23 460-479 2 2 2 2 2 2 F F F 24 480-499 5 5 5 7 7 7 F F F 25 500-519 2 2 2 2 2 2 F F F 26 520-539 1 1 1 1 1 1 F F F 27 540-559 4 4 4 6 6 6 F F F 28 560-579 1 1 1 1 1 1 F F F 29 580-599 1 1 1 1 1 1 F F F Max gen: 5, Max flush: 7 Begin loop 3 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:24.65594988Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1206,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:24.702010734Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1206,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:24.702758543Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1206} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"26e7d501b33f47379e53601720353c2983e7654b\",\n git_commit_timestamp: \"2024-12-12T20:16:40.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:24.70279379Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1206} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:24.702808019Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1206} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:24.702822168Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1206} {"msg":"Crucible b08de5c4-6d1e-464c-b1ea-6ccee75ac59d has session id: 47fd520d-34b3-4724-a6c5-9a0001afd323","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:24.703298095Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1206,"session_id":"47fd520d-34b3-4724-a6c5-9a0001afd323"} {"msg":"Upstairs opts: Upstairs UUID: b08de5c4-6d1e-464c-b1ea-6ccee75ac59d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:24.703341992Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1206,"session_id":"47fd520d-34b3-4724-a6c5-9a0001afd323"} {"msg":"Crucible stats registered with UUID: b08de5c4-6d1e-464c-b1ea-6ccee75ac59d","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:24.703359201Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1206,"session_id":"47fd520d-34b3-4724-a6c5-9a0001afd323"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:24.735237943Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1206,"task":"crutest"} {"msg":"The guest has requested activation with gen:6","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:24.735265881Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1206} {"msg":"b08de5c4-6d1e-464c-b1ea-6ccee75ac59d active request set","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:24.735350875Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1206,"session_id":"47fd520d-34b3-4724-a6c5-9a0001afd323"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:24.735368624Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1206,"client":"0","":"downstairs","session_id":"47fd520d-34b3-4724-a6c5-9a0001afd323"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:24.735395672Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1206,"client":"0","":"downstairs","session_id":"47fd520d-34b3-4724-a6c5-9a0001afd323"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:24.735410211Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1206,"client":"1","":"downstairs","session_id":"47fd520d-34b3-4724-a6c5-9a0001afd323"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:24.7354247Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1206,"client":"1","":"downstairs","session_id":"47fd520d-34b3-4724-a6c5-9a0001afd323"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:24.735438059Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1206,"client":"2","":"downstairs","session_id":"47fd520d-34b3-4724-a6c5-9a0001afd323"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:24.735452058Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1206,"client":"2","":"downstairs","session_id":"47fd520d-34b3-4724-a6c5-9a0001afd323"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:24.735465107Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1206,"":"io task","client":"2","":"downstairs","session_id":"47fd520d-34b3-4724-a6c5-9a0001afd323"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:24.735478606Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1206,"":"io task","client":"0","":"downstairs","session_id":"47fd520d-34b3-4724-a6c5-9a0001afd323"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:24.735492125Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1206,"":"io task","client":"1","":"downstairs","session_id":"47fd520d-34b3-4724-a6c5-9a0001afd323"} {"msg":"ds_connection connected from Ok(127.0.0.1:52737)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:24.735631086Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1206,"":"io task","client":"0","":"downstairs","session_id":"47fd520d-34b3-4724-a6c5-9a0001afd323"} {"msg":"ds_connection connected from Ok(127.0.0.1:35522)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:24.735676763Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1206,"":"io task","client":"1","":"downstairs","session_id":"47fd520d-34b3-4724-a6c5-9a0001afd323"} {"msg":"ds_connection connected from Ok(127.0.0.1:60937)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:24.735830202Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1206,"":"io task","client":"2","":"downstairs","session_id":"47fd520d-34b3-4724-a6c5-9a0001afd323"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:24.73644044Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1206,"client":"0","":"downstairs","session_id":"47fd520d-34b3-4724-a6c5-9a0001afd323"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:24.736470328Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1206,"client":"1","":"downstairs","session_id":"47fd520d-34b3-4724-a6c5-9a0001afd323"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:24.736486097Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1206,"client":"2","":"downstairs","session_id":"47fd520d-34b3-4724-a6c5-9a0001afd323"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:24.737041239Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1206,"client":"0","":"downstairs","session_id":"47fd520d-34b3-4724-a6c5-9a0001afd323"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:24.737154021Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1206,"client":"2","":"downstairs","session_id":"47fd520d-34b3-4724-a6c5-9a0001afd323"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:24.737182509Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1206,"client":"1","":"downstairs","session_id":"47fd520d-34b3-4724-a6c5-9a0001afd323"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:24.737580102Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1206,"session_id":"47fd520d-34b3-4724-a6c5-9a0001afd323"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:24.737758159Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1206,"session_id":"47fd520d-34b3-4724-a6c5-9a0001afd323"} {"msg":"[0]R flush_numbers[0..12]: [1, 1, 2, 1, 5, 6, 1, 4, 6, 7, 6, 6]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:24.737919648Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1206,"":"downstairs","":"downstairs","session_id":"47fd520d-34b3-4724-a6c5-9a0001afd323"} {"msg":"[0]R generation[0..12]: [1, 1, 2, 1, 3, 4, 1, 2, 4, 5, 4, 4]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:24.737948886Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1206,"":"downstairs","":"downstairs","session_id":"47fd520d-34b3-4724-a6c5-9a0001afd323"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:24.737965025Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1206,"":"downstairs","":"downstairs","session_id":"47fd520d-34b3-4724-a6c5-9a0001afd323"} {"msg":"[1]R flush_numbers[0..12]: [1, 1, 2, 1, 5, 6, 1, 4, 6, 7, 6, 6]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:24.737979374Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1206,"":"downstairs","":"downstairs","session_id":"47fd520d-34b3-4724-a6c5-9a0001afd323"} {"msg":"[1]R generation[0..12]: [1, 1, 2, 1, 3, 4, 1, 2, 4, 5, 4, 4]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:24.737992903Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1206,"":"downstairs","":"downstairs","session_id":"47fd520d-34b3-4724-a6c5-9a0001afd323"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:24.738006982Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1206,"":"downstairs","":"downstairs","session_id":"47fd520d-34b3-4724-a6c5-9a0001afd323"} {"msg":"[2]R flush_numbers[0..12]: [1, 1, 2, 1, 5, 6, 1, 4, 6, 7, 6, 6]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:24.738020511Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1206,"":"downstairs","":"downstairs","session_id":"47fd520d-34b3-4724-a6c5-9a0001afd323"} {"msg":"[2]R generation[0..12]: [1, 1, 2, 1, 3, 4, 1, 2, 4, 5, 4, 4]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:24.73803384Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1206,"":"downstairs","":"downstairs","session_id":"47fd520d-34b3-4724-a6c5-9a0001afd323"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:24.738047849Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1206,"":"downstairs","":"downstairs","session_id":"47fd520d-34b3-4724-a6c5-9a0001afd323"} {"msg":"Max found gen is 6","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:24.738060858Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1206,"":"downstairs","":"downstairs","session_id":"47fd520d-34b3-4724-a6c5-9a0001afd323"} {"msg":"Generation requested: 6 >= found:6","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:24.738078377Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1206,"":"downstairs","":"downstairs","session_id":"47fd520d-34b3-4724-a6c5-9a0001afd323"} {"msg":"Next flush: 8","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:24.738091706Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1206,"":"downstairs","":"downstairs","session_id":"47fd520d-34b3-4724-a6c5-9a0001afd323"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:24.738104805Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":Wait for a query_work_queue command to finish before sending IO 1206,"":"downstairs","":"downstairs","session_id":"47fd520d-34b3-4724-a6c5-9a0001afd323"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:24.738123044Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1206,"session_id":"47fd520d-34b3-4724-a6c5-9a0001afd323"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:24.738135143Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1206,"client":"0","":"downstairs","session_id":"47fd520d-34b3-4724-a6c5-9a0001afd323"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:24.738148422Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1206,"client":"1","":"downstairs","session_id":"47fd520d-34b3-4724-a6c5-9a0001afd323"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:24.738161451Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1206,"client":"2","":"downstairs","session_id":"47fd520d-34b3-4724-a6c5-9a0001afd323"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:24.738174581Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1206,"session_id":"47fd520d-34b3-4724-a6c5-9a0001afd323"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:24.73818658Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1206,"session_id":"47fd520d-34b3-4724-a6c5-9a0001afd323"} {"msg":"b08de5c4-6d1e-464c-b1ea-6ccee75ac59d is now active with session: 47fd520d-34b3-4724-a6c5-9a0001afd323","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:24.738199329Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1206,"session_id":"47fd520d-34b3-4724-a6c5-9a0001afd323"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:24.738211128Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1206,"session_id":"47fd520d-34b3-4724-a6c5-9a0001afd323"} {"msg":"The guest has finished waiting for activation with:6","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:24.738223187Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1206} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 39 len 4096 data: 2 02/30 Write block 272 len 12288 data: 2 2 2 03/30 Write block 227 len 36864 data: 2 2 2 2 2 2 2 2 2 04/30 Write block 327 len 20480 data: 2 2 2 2 2 05/30 Write block 230 len 4096 data: 3 06/30 Read block 442 len 36864 07/30 Write block 517 len 36864 data: 2 2 2 2 2 2 2 2 2 08/30 Read block 229 len 40960 09/30 Write block 184 len 40960 data: 2 2 2 2 3 3 3 3 3 3 10/30 Read block 40 len 36864 11/30 Read block 499 len 16384 12/30 Read block 432 len 12288 13/30 Write block 259 len 12288 data: 2 2 2 14/30 Write block 484 len 40960 data: 2 2 2 2 2 3 3 3 2 2 15/30 Write block 58 len 4096 data: 2 16/30 Read block 517 len 40960 17/30 Flush 18/30 Write block 507 len 32768 data: 2 2 2 2 2 2 2 2 19/30 Read block 528 len 28672 20/30 Write block 325 len 28672 data: 2 2 3 3 3 3 3 21/30 Read block 417 len 8192 22/30 Write block 518 len 28672 data: 3 3 3 3 3 3 3 23/30 Flush 24/30 Flush 25/30 Write block 540 len 4096 data: 2 26/30 Flush 27/30 Write block 86 len 36864 data: 2 3 3 3 3 3 3 3 3 28/30 Write block 507 len 20480 data: 3 3 3 3 3 29/30 Read block 454 len 40960 30/30 Write block 254 len 12288 data: 2 2 2 ---------------------------------------------------------------- Crucible gen:6 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 9 Done Done Done false 1027 Acked Write 5 Done Done Done false 1028 Acked Read 10 Done Done Done false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:33:24.757353178Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1206} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 1 1 1 F F F 1 020-039 6 6 6 8 8 8 F F F 2 040-059 6 6 6 8 8 8 F F F 3 060-079 1 1 1 1 1 1 F F F 4 080-099 3 3 3 5 5 5 T T T 5 100-119 4 4 4 6 6 6 F F F 6 120-139 1 1 1 1 1 1 F F F 7 140-159 2 2 2 4 4 4 F F F 8 160-179 4 4 4 6 6 6 F F F 9 180-199 6 6 6 8 8 8 F F F 10 200-219 4 4 4 6 6 6 F F F 11 220-239 6 6 6 8 8 8 F F F 12 240-259 6 6 6 8 8 8 T T T 13 260-279 6 6 6 8 8 8 F F F 14 280-299 4 4 4 6 6 6 F F F 15 300-319 5 5 5 7 7 7 F F F 16 320-339 6 6 6 9 9 9 F F F 17 340-359 1 1 1 1 1 1 F F F 18 360-379 5 5 5 7 7 7 F F F 19 380-399 2 2 2 3 3 3 F F F 20 400-419 4 4 4 6 6 6 F F F 21 420-439 2 2 2 2 2 2 F F F 22 440-459 1 1 1 1 1 1 F F F 23 460-479 2 2 2 2 2 2 F F F 24 480-499 6 6 6 8 8 8 F F F 25 500-519 6 6 6 9 9 9 T T T 26 520-539 6 6 6 9 9 9 F F F 27 540-559 6 6 6 11 11 11 F F F 28 560-579 1 1 1 1 1 1 F F F 29 580-599 1 1 1 1 1 1 F F F Max gen: 6, Max flush: 11 A Difference in extent metadata is expected here On loop 3 Verifying data now No change after verify Loop: 3 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 1 1 1 F F F 1 020-039 6 6 6 8 8 8 F F F 2 040-059 6 6 6 8 8 8 F F F 3 060-079 1 1 1 1 1 1 F F F 4 080-099 7 7 7 12 12 12 F F F 5 100-119 4 4 4 6 6 6 F F F 6 120-139 1 1 1 1 1 1 F F F 7 140-159 2 2 2 4 4 4 F F F 8 160-179 4 4 4 6 6 6 F F F 9 180-199 6 6 6 8 8 8 F F F 10 200-219 4 4 4 6 6 6 F F F 11 220-239 6 6 6 8 8 8 F F F 12 240-259 7 7 7 12 12 12 F F F 13 260-279 6 6 6 8 8 8 F F F 14 280-299 4 4 4 6 6 6 F F F 15 300-319 5 5 5 7 7 7 F F F 16 320-339 6 6 6 9 9 9 F F F 17 340-359 1 1 1 1 1 1 F F F 18 360-379 5 5 5 7 7 7 F F F 19 380-399 2 2 2 3 3 3 F F F 20 400-419 4 4 4 6 6 6 F F F 21 420-439 2 2 2 2 2 2 F F F 22 440-459 1 1 1 1 1 1 F F F 23 460-479 2 2 2 2 2 2 F F F 24 480-499 6 6 6 8 8 8 F F F 25 500-519 7 7 7 12 12 12 F F F 26 520-539 6 6 6 9 9 9 F F F 27 540-559 6 6 6 11 11 11 F F F 28 560-579 1 1 1 1 1 1 F F F 29 580-599 1 1 1 1 1 1 F F F Max gen: 7, Max flush: 12 Begin loop 4 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:29.01509814Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1214,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:29.060570334Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1214,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:29.061154664Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1214} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"26e7d501b33f47379e53601720353c2983e7654b\",\n git_commit_timestamp: \"2024-12-12T20:16:40.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:29.061190362Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1214} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:29.061205151Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1214} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:29.06121932Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1214} {"msg":"Crucible 60a3e768-c0f8-42ab-9f8f-049291829399 has session id: 487c52c3-58ab-44a9-8a12-4b0c20e8e17a","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:29.061694747Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1214,"session_id":"487c52c3-58ab-44a9-8a12-4b0c20e8e17a"} {"msg":"Upstairs opts: Upstairs UUID: 60a3e768-c0f8-42ab-9f8f-049291829399, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:29.061740714Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1214,"session_id":"487c52c3-58ab-44a9-8a12-4b0c20e8e17a"} {"msg":"Crucible stats registered with UUID: 60a3e768-c0f8-42ab-9f8f-049291829399","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:29.061765652Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1214,"session_id":"487c52c3-58ab-44a9-8a12-4b0c20e8e17a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:29.092748815Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1214,"task":"crutest"} {"msg":"The guest has requested activation with gen:8","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:29.092780083Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1214} {"msg":"60a3e768-c0f8-42ab-9f8f-049291829399 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:29.092849859Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1214,"session_id":"487c52c3-58ab-44a9-8a12-4b0c20e8e17a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:29.092868627Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1214,"client":"0","":"downstairs","session_id":"487c52c3-58ab-44a9-8a12-4b0c20e8e17a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:29.092892166Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1214,"client":"0","":"downstairs","session_id":"487c52c3-58ab-44a9-8a12-4b0c20e8e17a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:29.092910554Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1214,"client":"1","":"downstairs","session_id":"487c52c3-58ab-44a9-8a12-4b0c20e8e17a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:29.092937093Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1214,"client":"1","":"downstairs","session_id":"487c52c3-58ab-44a9-8a12-4b0c20e8e17a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:29.092964281Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1214,"client":"2","":"downstairs","session_id":"487c52c3-58ab-44a9-8a12-4b0c20e8e17a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:29.092992709Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1214,"client":"2","":"downstairs","session_id":"487c52c3-58ab-44a9-8a12-4b0c20e8e17a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:29.093019197Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1214,"":"io task","client":"2","":"downstairs","session_id":"487c52c3-58ab-44a9-8a12-4b0c20e8e17a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:29.093047065Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1214,"":"io task","client":"0","":"downstairs","session_id":"487c52c3-58ab-44a9-8a12-4b0c20e8e17a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:29.093076243Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1214,"":"io task","client":"1","":"downstairs","session_id":"487c52c3-58ab-44a9-8a12-4b0c20e8e17a"} {"msg":"ds_connection connected from Ok(127.0.0.1:64062)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:29.093106641Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1214,"":"io task","client":"2","":"downstairs","session_id":"487c52c3-58ab-44a9-8a12-4b0c20e8e17a"} {"msg":"ds_connection connected from Ok(127.0.0.1:60418)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:29.093136969Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1214,"":"io task","client":"0","":"downstairs","session_id":"487c52c3-58ab-44a9-8a12-4b0c20e8e17a"} {"msg":"ds_connection connected from Ok(127.0.0.1:47515)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:29.093166307Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1214,"":"io task","client":"1","":"downstairs","session_id":"487c52c3-58ab-44a9-8a12-4b0c20e8e17a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:29.093805573Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1214,"client":"2","":"downstairs","session_id":"487c52c3-58ab-44a9-8a12-4b0c20e8e17a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:29.093934764Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1214,"client":"0","":"downstairs","session_id":"487c52c3-58ab-44a9-8a12-4b0c20e8e17a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:29.094095343Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1214,"client":"1","":"downstairs","session_id":"487c52c3-58ab-44a9-8a12-4b0c20e8e17a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:29.094430259Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1214,"client":"2","":"downstairs","session_id":"487c52c3-58ab-44a9-8a12-4b0c20e8e17a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:29.094648435Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1214,"client":"0","":"downstairs","session_id":"487c52c3-58ab-44a9-8a12-4b0c20e8e17a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:29.094834912Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1214,"client":"1","":"downstairs","session_id":"487c52c3-58ab-44a9-8a12-4b0c20e8e17a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:29.094991601Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1214,"session_id":"487c52c3-58ab-44a9-8a12-4b0c20e8e17a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:29.095116472Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1214,"session_id":"487c52c3-58ab-44a9-8a12-4b0c20e8e17a"} {"msg":"[0]R flush_numbers[0..12]: [1, 8, 8, 1, 12, 6, 1, 4, 6, 8, 6, 8]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:29.095377604Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1214,"":"downstairs","":"downstairs","session_id":"487c52c3-58ab-44a9-8a12-4b0c20e8e17a"} {"msg":"[0]R generation[0..12]: [1, 6, 6, 1, 7, 4, 1, 2, 4, 6, 4, 6]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:29.095402092Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1214,"":"downstairs","":"downstairs","session_id":"487c52c3-58ab-44a9-8a12-4b0c20e8e17a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:29.095417571Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1214,"":"downstairs","":"downstairs","session_id":"487c52c3-58ab-44a9-8a12-4b0c20e8e17a"} {"msg":"[1]R flush_numbers[0..12]: [1, 8, 8, 1, 12, 6, 1, 4, 6, 8, 6, 8]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:29.095431561Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1214,"":"downstairs","":"downstairs","session_id":"487c52c3-58ab-44a9-8a12-4b0c20e8e17a"} {"msg":"[1]R generation[0..12]: [1, 6, 6, 1, 7, 4, 1, 2, 4, 6, 4, 6]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:29.09544518Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1214,"":"downstairs","":"downstairs","session_id":"487c52c3-58ab-44a9-8a12-4b0c20e8e17a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:29.095459199Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1214,"":"downstairs","":"downstairs","session_id":"487c52c3-58ab-44a9-8a12-4b0c20e8e17a"} {"msg":"[2]R flush_numbers[0..12]: [1, 8, 8, 1, 12, 6, 1, 4, 6, 8, 6, 8]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:29.095472738Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1214,"":"downstairs","":"downstairs","session_id":"487c52c3-58ab-44a9-8a12-4b0c20e8e17a"} {"msg":"[2]R generation[0..12]: [1, 6, 6, 1, 7, 4, 1, 2, 4, 6, 4, 6]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:29.095489077Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1214,"":"downstairs","":"downstairs","session_id":"487c52c3-58ab-44a9-8a12-4b0c20e8e17a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:29.095503316Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1214,"":"downstairs","":"downstairs","session_id":"487c52c3-58ab-44a9-8a12-4b0c20e8e17a"} {"msg":"Max found gen is 8","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:29.095516225Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1214,"":"downstairs","":"downstairs","session_id":"487c52c3-58ab-44a9-8a12-4b0c20e8e17a"} {"msg":"Generation requested: 8 >= found:8","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:29.095529294Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1214,"":"downstairs","":"downstairs","session_id":"487c52c3-58ab-44a9-8a12-4b0c20e8e17a"} {"msg":"Next flush: 13","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:29.095542113Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1214,"":"downstairs","":"downstairs","session_id":"487c52c3-58ab-44a9-8a12-4b0c20e8e17a"} {Wait for a query_work_queue command to finish before sending IO "msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:29.095559652Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1214,"":"downstairs","":"downstairs","session_id":"487c52c3-58ab-44a9-8a12-4b0c20e8e17a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:29.095572971Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1214,"session_id":"487c52c3-58ab-44a9-8a12-4b0c20e8e17a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:29.09558495Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1214,"client":"0","":"downstairs","session_id":"487c52c3-58ab-44a9-8a12-4b0c20e8e17a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:29.095597999Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1214,"client":"1","":"downstairs","session_id":"487c52c3-58ab-44a9-8a12-4b0c20e8e17a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:29.095611408Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1214,"client":"2","":"downstairs","session_id":"487c52c3-58ab-44a9-8a12-4b0c20e8e17a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:29.095624917Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1214,"session_id":"487c52c3-58ab-44a9-8a12-4b0c20e8e17a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:29.095645746Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1214,"session_id":"487c52c3-58ab-44a9-8a12-4b0c20e8e17a"} {"msg":"60a3e768-c0f8-42ab-9f8f-049291829399 is now active with session: 487c52c3-58ab-44a9-8a12-4b0c20e8e17a","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:29.095670034Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1214,"session_id":"487c52c3-58ab-44a9-8a12-4b0c20e8e17a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:29.095683463Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1214,"session_id":"487c52c3-58ab-44a9-8a12-4b0c20e8e17a"} {"msg":"The guest has finished waiting for activation with:8","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:29.095695672Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1214} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 418 len 4096 data: 3 02/30 Write block 55 len 24576 data: 2 2 2 3 2 2 03/30 Flush 04/30 Write block 378 len 24576 data: 2 2 2 2 2 2 05/30 Read block 346 len 20480 06/30 Read block 579 len 8192 07/30 Write block 148 len 16384 data: 3 2 2 2 08/30 Write block 394 len 40960 data: 3 3 2 2 2 2 2 2 2 2 09/30 Read block 577 len 36864 10/30 Read block 153 len 40960 11/30 Flush 12/30 Write block 401 len 36864 data: 3 3 3 3 3 3 3 3 3 13/30 Read block 148 len 16384 14/30 Write block 224 len 36864 data: 2 2 2 3 3 3 4 3 3 15/30 Write block 446 len 8192 data: 2 2 16/30 Read block 373 len 36864 17/30 Write block 564 len 24576 data: 2 2 2 2 2 2 18/30 Read block 416 len 40960 19/30 Write block 333 len 28672 data: 2 2 2 2 2 3 3 20/30 Read block 223 len 36864 21/30 Write block 314 len 40960 data: 4 4 4 3 3 2 2 2 2 2 22/30 Read block 184 len 24576 23/30 Write block 48 len 12288 data: 3 3 3 24/30 Read block 83 len 28672 25/30 Write block 439 len 36864 data: 2 2 2 2 2 2 2 3 3 26/30 Write block 363 len 32768 data: 3 3 3 3 3 2 2 2 27/30 Write block 518 len 36864 data: 4 4 4 4 4 4 4 3 2 28/30 Write block 369 len 16384 data: 3 3 2 2 29/30 Write block 553 len 28672 data: 3 3 3 2 2 2 2 30/30 Write block 35 len 28672 data: 2 2 2 2 3 2 2 ---------------------------------------------------------------- Crucible gen:8 GIO:true work queues: Upstairs:0 downstairs:19 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1011 Acked Write 9 Done Done Done false 1012 Acked Read 4 Done Done Done false 1013 Acked Write 9 Done Done Done false 1014 Acked Write 2 Done Done Done false 1015 Acked Read 9 Done Done Done false 1016 Acked Write 6 Done Done Done false 1017 Acked Read 10 Done Done Done false 1018 Acked Write 7 Done Done Done false 1019 Acked Read 9 Done Done Done false 1020 Acked Write 10 Done Done Done false 1021 Acked Read 6 Done Done Done false 1022 Acked Write 3 Done Done Done false 1023 Acked Read 7 Done Done Done false 1024 Acked Write 9 Done Done Done false 1025 Acked Write 8 Done Done Done false 1026 Acked Write 9 Done Done Done false 1027 Acked Write 4 Done Done Done false 1028 Acked Write 7 Done Sent Sent false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 2 2 5 Done 18 17 17 52 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1010 1010 1010 Downstairs last five completed: 1010 1009 1008 1007 1006 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:33:29.115468379Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1214} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 1 1 1 F F F 1 020-039 6 6 6 8 8 8 T T T 2 040-059 8 8 8 13 13 13 T T T 3 060-079 8 8 8 13 13 13 F F F 4 080-099 7 7 7 12 12 12 F F F 5 100-119 4 4 4 6 6 6 F F F 6 120-139 1 1 1 1 1 1 F F F 7 140-159 8 8 8 14 14 14 F F F 8 160-179 4 4 4 6 6 6 F F F 9 180-199 6 6 6 8 8 8 F F F 10 200-219 4 4 4 6 6 6 F F F 11 220-239 6 6 6 8 8 8 T T T 12 240-259 7 7 7 12 12 12 F F F 13 260-279 6 6 6 8 8 8 F F F 14 280-299 4 4 4 6 6 6 F F F 15 300-319 5 5 5 7 7 7 T T T 16 320-339 6 6 6 9 9 9 T T T 17 340-359 1 1 1 1 1 1 F F F 18 360-379 8 8 8 14 14 14 T T T 19 380-399 8 8 8 14 14 14 F F F 20 400-419 8 8 8 14 14 14 T T T 21 420-439 2 2 2 2 2 2 T T T 22 440-459 1 1 1 1 1 1 T T T 23 460-479 2 2 2 2 2 2 F F F 24 480-499 6 6 6 8 8 8 F F F 25 500-519 7 7 7 12 12 12 T T T 26 520-539 6 6 6 9 9 9 T T T 27 540-559 6 6 6 11 11 11 T T T 28 560-579 1 1 1 1 1 1 T T T 29 580-599 1 1 1 1 1 1 F F F Max gen: 8, Max flush: 14 A Difference in extent metadata is expected here On loop 4 Verifying data now No change after verify Loop: 4 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 1 1 1 F F F 1 020-039 9 9 9 15 15 15 F F F 2 040-059 9 9 9 15 15 15 F F F 3 060-079 8 8 8 13 13 13 F F F 4 080-099 7 7 7 12 12 12 F F F 5 100-119 4 4 4 6 6 6 F F F 6 120-139 1 1 1 1 1 1 F F F 7 140-159 8 8 8 14 14 14 F F F 8 160-179 4 4 4 6 6 6 F F F 9 180-199 6 6 6 8 8 8 F F F 10 200-219 4 4 4 6 6 6 F F F 11 220-239 9 9 9 15 15 15 F F F 12 240-259 7 7 7 12 12 12 F F F 13 260-279 6 6 6 8 8 8 F F F 14 280-299 4 4 4 6 6 6 F F F 15 300-319 9 9 9 15 15 15 F F F 16 320-339 9 9 9 15 15 15 F F F 17 340-359 1 1 1 1 1 1 F F F 18 360-379 9 9 9 15 15 15 F F F 19 380-399 8 8 8 14 14 14 F F F 20 400-419 9 9 9 15 15 15 F F F 21 420-439 9 9 9 15 15 15 F F F 22 440-459 9 9 9 15 15 15 F F F 23 460-479 2 2 2 2 2 2 F F F 24 480-499 6 6 6 8 8 8 F F F 25 500-519 9 9 9 15 15 15 F F F 26 520-539 9 9 9 15 15 15 F F F 27 540-559 9 9 9 15 15 15 F F F 28 560-579 9 9 9 15 15 15 F F F 29 580-599 1 1 1 1 1 1 F F F Max gen: 9, Max flush: 15 Begin loop 5 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:33.422414279Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1222,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:33.466575334Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1222,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:33.467298174Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1222} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"26e7d501b33f47379e53601720353c2983e7654b\",\n git_commit_timestamp: \"2024-12-12T20:16:40.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:33.467334172Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1222} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:33.46735274Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1222} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:33.467367489Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1222} {"msg":"Crucible 16bd0ac9-fdd9-41ad-a3f3-9d6f6bc814f3 has session id: 157c1398-d77d-46f4-a0c6-de24748e8c31","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:33.467814639Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1222,"session_id":"157c1398-d77d-46f4-a0c6-de24748e8c31"} {"msg":"Upstairs opts: Upstairs UUID: 16bd0ac9-fdd9-41ad-a3f3-9d6f6bc814f3, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:33.467858325Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1222,"session_id":"157c1398-d77d-46f4-a0c6-de24748e8c31"} {"msg":"Crucible stats registered with UUID: 16bd0ac9-fdd9-41ad-a3f3-9d6f6bc814f3","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:33.467882774Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1222,"session_id":"157c1398-d77d-46f4-a0c6-de24748e8c31"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:33.498630634Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1222,"task":"crutest"} {"msg":"The guest has requested activation with gen:10","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:33.498657082Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1222} {"msg":"16bd0ac9-fdd9-41ad-a3f3-9d6f6bc814f3 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:33.498722797Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1222,"session_id":"157c1398-d77d-46f4-a0c6-de24748e8c31"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:33.498740106Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1222,"client":"0","":"downstairs","session_id":"157c1398-d77d-46f4-a0c6-de24748e8c31"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:33.498763125Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1222,"client":"0","":"downstairs","session_id":"157c1398-d77d-46f4-a0c6-de24748e8c31"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:33.498778193Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1222,"client":"1","":"downstairs","session_id":"157c1398-d77d-46f4-a0c6-de24748e8c31"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:33.498795372Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1222,"client":"1","":"downstairs","session_id":"157c1398-d77d-46f4-a0c6-de24748e8c31"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:33.498819801Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1222,"client":"2","":"downstairs","session_id":"157c1398-d77d-46f4-a0c6-de24748e8c31"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:33.498847069Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1222,"client":"2","":"downstairs","session_id":"157c1398-d77d-46f4-a0c6-de24748e8c31"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:33.498873077Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1222,"":"io task","client":"2","":"downstairs","session_id":"157c1398-d77d-46f4-a0c6-de24748e8c31"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:33.498905015Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1222,"":"io task","client":"0","":"downstairs","session_id":"157c1398-d77d-46f4-a0c6-de24748e8c31"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:33.498933353Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1222,"":"io task","client":"1","":"downstairs","session_id":"157c1398-d77d-46f4-a0c6-de24748e8c31"} {"msg":"ds_connection connected from Ok(127.0.0.1:55171)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:33.498961971Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1222,"":"io task","client":"1","":"downstairs","session_id":"157c1398-d77d-46f4-a0c6-de24748e8c31"} {"msg":"ds_connection connected from Ok(127.0.0.1:55517)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:33.499134669Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1222,"":"io task","client":"0","":"downstairs","session_id":"157c1398-d77d-46f4-a0c6-de24748e8c31"} {"msg":"ds_connection connected from Ok(127.0.0.1:63956)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:33.499175446Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1222,"":"io task","client":"2","":"downstairs","session_id":"157c1398-d77d-46f4-a0c6-de24748e8c31"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:33.499715529Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1222,"client":"0","":"downstairs","session_id":"157c1398-d77d-46f4-a0c6-de24748e8c31"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:33.499744287Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1222,"client":"2","":"downstairs","session_id":"157c1398-d77d-46f4-a0c6-de24748e8c31"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:33.499759566Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1222,"client":"1","":"downstairs","session_id":"157c1398-d77d-46f4-a0c6-de24748e8c31"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:33.500353545Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1222,"client":"1","":"downstairs","session_id":"157c1398-d77d-46f4-a0c6-de24748e8c31"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:33.500493375Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1222,"client":"0","":"downstairs","session_id":"157c1398-d77d-46f4-a0c6-de24748e8c31"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:33.500530432Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1222,"client":"2","":"downstairs","session_id":"157c1398-d77d-46f4-a0c6-de24748e8c31"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:33.500823042Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1222,"session_id":"157c1398-d77d-46f4-a0c6-de24748e8c31"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:33.501048437Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1222,"session_id":"157c1398-d77d-46f4-a0c6-de24748e8c31"} {"msg":"[0]R flush_numbers[0..12]: [1, 15, 15, 13, 12, 6, 1, 14, 6, 8, 6, 15]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:33.501197037Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1222,"":"downstairs","":"downstairs","session_id":"157c1398-d77d-46f4-a0c6-de24748e8c31"} {"msg":"[0]R generation[0..12]: [1, 9, 9, 8, 7, 4, 1, 8, 4, 6, 4, 9]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:33.501216625Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1222,"":"downstairs","":"downstairs","session_id":"157c1398-d77d-46f4-a0c6-de24748e8c31"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:33.501233954Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1222,"":"downstairs","":"downstairs","session_id":"157c1398-d77d-46f4-a0c6-de24748e8c31"} {"msg":"[1]R flush_numbers[0..12]: [1, 15, 15, 13, 12, 6, 1, 14, 6, 8, 6, 15]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:33.501247963Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1222,"":"downstairs","":"downstairs","session_id":"157c1398-d77d-46f4-a0c6-de24748e8c31"} {"msg":"[1]R generation[0..12]: [1, 9, 9, 8, 7, 4, 1, 8, 4, 6, 4, 9]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:33.501261632Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1222,"":"downstairs","":"downstairs","session_id":"157c1398-d77d-46f4-a0c6-de24748e8c31"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:33.501275591Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1222,"":"downstairs","":"downstairs","session_id":"157c1398-d77d-46f4-a0c6-de24748e8c31"} {"msg":"[2]R flush_numbers[0..12]: [1, 15, 15, 13, 12, 6, 1, 14, 6, 8, 6, 15]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:33.50128947Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1222,"":"downstairs","":"downstairs","session_id":"157c1398-d77d-46f4-a0c6-de24748e8c31"} {"msg":"[2]R generation[0..12]: [1, 9, 9, 8, 7, 4, 1, 8, 4, 6, 4, 9]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:33.501303089Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1222,"":"downstairs","":"downstairs","session_id":"157c1398-d77d-46f4-a0c6-de24748e8c31"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:33.501317138Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1222,"":"downstairs","":"downstairs","session_id":"157c1398-d77d-46f4-a0c6-de24748e8c31"} {"msg":"Max found gen is 10","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:33.501330057Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1222,"":"downstairs","":"downstairs","session_id":"157c1398-d77d-46f4-a0c6-de24748e8c31"} {"msg":"Generation requested: 10 >= found:10","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:33.501343287Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1222,"":"downstairs","":"downstairs","session_id":"157c1398-d77d-46f4-a0c6-de24748e8c31"} {"msg":"Next flush: 16","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:33.501356246Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1222,"":"downstairs","":"downstairs","session_id":"157c1398-d77d-46f4-a0c6-de24748e8c31"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:33.501369235Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1222,"":"downstairs","":"downstairs","session_id":"157c1398-d77d-46f4-a0c6-de24748e8c31"} Wait for a query_work_queue command to finish before sending IO {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:33.501386204Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1222,"session_id":"157c1398-d77d-46f4-a0c6-de24748e8c31"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:33.501398203Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1222,"client":"0","":"downstairs","session_id":"157c1398-d77d-46f4-a0c6-de24748e8c31"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:33.501411462Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1222,"client":"1","":"downstairs","session_id":"157c1398-d77d-46f4-a0c6-de24748e8c31"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:33.501426061Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1222,"client":"2","":"downstairs","session_id":"157c1398-d77d-46f4-a0c6-de24748e8c31"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:33.50143925Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1222,"session_id":"157c1398-d77d-46f4-a0c6-de24748e8c31"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:33.501451369Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1222,"session_id":"157c1398-d77d-46f4-a0c6-de24748e8c31"} {"msg":"16bd0ac9-fdd9-41ad-a3f3-9d6f6bc814f3 is now active with session: 157c1398-d77d-46f4-a0c6-de24748e8c31","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:33.501463968Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1222,"session_id":"157c1398-d77d-46f4-a0c6-de24748e8c31"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:33.501475767Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1222,"session_id":"157c1398-d77d-46f4-a0c6-de24748e8c31"} {"msg":"The guest has finished waiting for activation with:10","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:33.501487846Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1222} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 37 len 36864 data: 3 3 4 3 3 2 2 2 3 02/30 Read block 432 len 12288 03/30 Write block 549 len 36864 data: 2 2 3 3 4 4 4 3 3 04/30 Write block 206 len 24576 data: 2 2 2 2 2 2 05/30 Write block 469 len 20480 data: 2 2 2 2 2 06/30 Write block 249 len 40960 data: 2 2 2 2 2 3 3 3 2 2 07/30 Read block 190 len 20480 08/30 Flush 09/30 Write block 14 len 8192 data: 2 2 10/30 Write block 187 len 40960 data: 3 4 4 4 4 4 4 3 3 3 11/30 Read block 332 len 12288 12/30 Write block 527 len 12288 data: 2 2 2 13/30 Read block 465 len 20480 14/30 Read block 19 len 28672 15/30 Flush 16/30 Write block 145 len 16384 data: 3 3 3 4 17/30 Read block 25 len 36864 18/30 Flush 19/30 Write block 452 len 24576 data: 2 2 2 2 2 2 20/30 Write block 303 len 4096 data: 3 21/30 Read block 138 len 4096 22/30 Read block 280 len 32768 23/30 Write block 226 len 36864 data: 3 4 4 4 5 4 4 3 3 24/30 Read block 214 len 4096 25/30 Write block 293 len 20480 data: 2 2 2 3 3 26/30 Write block 179 len 32768 data: 3 2 2 2 2 3 3 3 27/30 Write block 353 len 16384 data: 2 2 2 2 28/30 Write block 481 len 28672 data: 2 2 2 3 3 3 3 29/30 Write block 225 len 28672 data: 3 4 5 5 5 6 5 30/30 Write block 18 len 4096 data: 2 ---------------------------------------------------------------- Crucible gen:10 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1018 Acked Write 6 Done Done Done false 1019 Acked Write 1 Done Done Done false 1020 Acked Read 1 Done Done Done false 1021 Acked Read 8 Done Done Done false 1022 Acked Write 9 Done Done Done false 1023 Acked Read 1 Done Done Done false 1024 Acked Write 5 Done Done Done false 1025 Acked Write 8 Done Done Done false 1026 Acked Write 4 Done Done Done false 1027 Acked Write 7 Done Done Done false 1028 Acked Write 7 Sent Sent Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 10 10 10 30 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1017 1017 1017 Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:33:33.519682662Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1222} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 10 10 10 17 17 17 T T T 1 020-039 10 10 10 16 16 16 F F F 2 040-059 10 10 10 16 16 16 F F F 3 060-079 8 8 8 13 13 13 F F F 4 080-099 7 7 7 12 12 12 F F F 5 100-119 4 4 4 6 6 6 F F F 6 120-139 1 1 1 1 1 1 F F F 7 140-159 10 10 10 18 18 18 F F F 8 160-179 4 4 4 6 6 6 T T T 9 180-199 10 10 10 17 17 17 T T T 10 200-219 10 10 10 16 16 16 F F F 11 220-239 9 9 9 15 15 15 T T T 12 240-259 10 10 10 16 16 16 F F F 13 260-279 6 6 6 8 8 8 F F F 14 280-299 4 4 4 6 6 6 T T T 15 300-319 9 9 9 15 15 15 T T T 16 320-339 9 9 9 15 15 15 F F F 17 340-359 1 1 1 1 1 1 T T T 18 360-379 9 9 9 15 15 15 F F F 19 380-399 8 8 8 14 14 14 F F F 20 400-419 9 9 9 15 15 15 F F F 21 420-439 9 9 9 15 15 15 F F F 22 440-459 9 9 9 15 15 15 T T T 23 460-479 10 10 10 16 16 16 F F F 24 480-499 6 6 6 8 8 8 T T T 25 500-519 9 9 9 15 15 15 F F F 26 520-539 10 10 10 17 17 17 F F F 27 540-559 10 10 10 16 16 16 F F F 28 560-579 9 9 9 15 15 15 F F F 29 580-599 1 1 1 1 1 1 F F F Max gen: 10, Max flush: 18 A Difference in extent metadata is expected here On loop 5 Verifying data now No change after verify Loop: 5 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 11 11 11 19 19 19 F F F 1 020-039 10 10 10 16 16 16 F F F 2 040-059 10 10 10 16 16 16 F F F 3 060-079 8 8 8 13 13 13 F F F 4 080-099 7 7 7 12 12 12 F F F 5 100-119 4 4 4 6 6 6 F F F 6 120-139 1 1 1 1 1 1 F F F 7 140-159 10 10 10 18 18 18 F F F 8 160-179 11 11 11 19 19 19 F F F 9 180-199 11 11 11 19 19 19 F F F 10 200-219 10 10 10 16 16 16 F F F 11 220-239 11 11 11 19 19 19 F F F 12 240-259 10 10 10 16 16 16 F F F 13 260-279 6 6 6 8 8 8 F F F 14 280-299 11 11 11 19 19 19 F F F 15 300-319 11 11 11 19 19 19 F F F 16 320-339 9 9 9 15 15 15 F F F 17 340-359 11 11 11 19 19 19 F F F 18 360-379 9 9 9 15 15 15 F F F 19 380-399 8 8 8 14 14 14 F F F 20 400-419 9 9 9 15 15 15 F F F 21 420-439 9 9 9 15 15 15 F F F 22 440-459 11 11 11 19 19 19 F F F 23 460-479 10 10 10 16 16 16 F F F 24 480-499 11 11 11 19 19 19 F F F 25 500-519 9 9 9 15 15 15 F F F 26 520-539 10 10 10 17 17 17 F F F 27 540-559 10 10 10 16 16 16 F F F 28 560-579 9 9 9 15 15 15 F F F 29 580-599 1 1 1 1 1 1 F F F Max gen: 11, Max flush: 19 Begin loop 6 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:37.808262657Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1230,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:33:37.851926376Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1230,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:37.852334578Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1230,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:37.88436383Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1230,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:37.884951559Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1230} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"26e7d501b33f47379e53601720353c2983e7654b\",\n git_commit_timestamp: \"2024-12-12T20:16:40.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:37.884987177Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1230} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:37.885001216Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1230} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:37.885015755Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1230} {"msg":"Crucible 53e104f4-08d6-4bdd-b3c1-8b6418715aa2 has session id: d6f47531-abb6-4206-8b8a-6e5a567ba861","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:37.885436356Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1230,"session_id":"d6f47531-abb6-4206-8b8a-6e5a567ba861"} {"msg":"Upstairs opts: Upstairs UUID: 53e104f4-08d6-4bdd-b3c1-8b6418715aa2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:37.885481093Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1230,"session_id":"d6f47531-abb6-4206-8b8a-6e5a567ba861"} {"msg":"Crucible stats registered with UUID: 53e104f4-08d6-4bdd-b3c1-8b6418715aa2","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:37.885504261Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1230,"session_id":"d6f47531-abb6-4206-8b8a-6e5a567ba861"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:37.916904786Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1230,"task":"crutest"} {"msg":"The guest has requested activation with gen:12","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:37.916947303Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1230} {"msg":"53e104f4-08d6-4bdd-b3c1-8b6418715aa2 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:37.917030617Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1230,"session_id":"d6f47531-abb6-4206-8b8a-6e5a567ba861"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:37.917064725Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1230,"client":"0","":"downstairs","session_id":"d6f47531-abb6-4206-8b8a-6e5a567ba861"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:37.917098732Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1230,"client":"0","":"downstairs","session_id":"d6f47531-abb6-4206-8b8a-6e5a567ba861"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:37.917118741Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1230,"client":"1","":"downstairs","session_id":"d6f47531-abb6-4206-8b8a-6e5a567ba861"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:37.917137Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1230,"client":"1","":"downstairs","session_id":"d6f47531-abb6-4206-8b8a-6e5a567ba861"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:37.917153809Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1230,"client":"2","":"downstairs","session_id":"d6f47531-abb6-4206-8b8a-6e5a567ba861"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:37.917215144Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1230,"client":"2","":"downstairs","session_id":"d6f47531-abb6-4206-8b8a-6e5a567ba861"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:37.917298899Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1230,"":"io task","client":"2","":"downstairs","session_id":"d6f47531-abb6-4206-8b8a-6e5a567ba861"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:37.917342685Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1230,"":"io task","client":"0","":"downstairs","session_id":"d6f47531-abb6-4206-8b8a-6e5a567ba861"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:37.917374583Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1230,"":"io task","client":"1","":"downstairs","session_id":"d6f47531-abb6-4206-8b8a-6e5a567ba861"} {"msg":"ds_connection connected from Ok(127.0.0.1:44047)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:37.917405791Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1230,"":"io task","client":"2","":"downstairs","session_id":"d6f47531-abb6-4206-8b8a-6e5a567ba861"} {"msg":"ds_connection connected from Ok(127.0.0.1:41692)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:37.917485056Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1230,"":"io task","client":"0","":"downstairs","session_id":"d6f47531-abb6-4206-8b8a-6e5a567ba861"} {"msg":"ds_connection connected from Ok(127.0.0.1:37335)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:37.917521603Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1230,"":"io task","client":"1","":"downstairs","session_id":"d6f47531-abb6-4206-8b8a-6e5a567ba861"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:37.918103523Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1230,"client":"1","":"downstairs","session_id":"d6f47531-abb6-4206-8b8a-6e5a567ba861"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:37.9181428Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1230,"client":"2","":"downstairs","session_id":"d6f47531-abb6-4206-8b8a-6e5a567ba861"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:37.918176898Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1230,"client":"0","":"downstairs","session_id":"d6f47531-abb6-4206-8b8a-6e5a567ba861"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:37.91872151Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1230,"client":"1","":"downstairs","session_id":"d6f47531-abb6-4206-8b8a-6e5a567ba861"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:37.918752198Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1230,"client":"0","":"downstairs","session_id":"d6f47531-abb6-4206-8b8a-6e5a567ba861"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:37.918853491Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1230,"client":"2","":"downstairs","session_id":"d6f47531-abb6-4206-8b8a-6e5a567ba861"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:37.91931396Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1230,"session_id":"d6f47531-abb6-4206-8b8a-6e5a567ba861"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:37.919407143Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1230,"session_id":"d6f47531-abb6-4206-8b8a-6e5a567ba861"} {"msg":"[0]R flush_numbers[0..12]: [19, 16, 16, 13, 12, 6, 1, 18, 19, 19, 16, 19]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:37.919499497Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1230,"":"downstairs","":"downstairs","session_id":"d6f47531-abb6-4206-8b8a-6e5a567ba861"} {"msg":"[0]R generation[0..12]: [11, 10, 10, 8, 7, 4, 1, 10, 11, 11, 10, 11]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:37.919519195Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1230,"":"downstairs","":"downstairs","session_id":"d6f47531-abb6-4206-8b8a-6e5a567ba861"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:37.919533705Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1230,"":"downstairs","":"downstairs","session_id":"d6f47531-abb6-4206-8b8a-6e5a567ba861"} {"msg":"[1]R flush_numbers[0..12]: [19, 16, 16, 13, 12, 6, 1, 18, 19, 19, 16, 19]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:37.919547404Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1230,"":"downstairs","":"downstairs","session_id":"d6f47531-abb6-4206-8b8a-6e5a567ba861"} {"msg":"[1]R generation[0..12]: [11, 10, 10, 8, 7, 4, 1, 10, 11, 11, 10, 11]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:37.919560903Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1230,"":"downstairs","":"downstairs","session_id":"d6f47531-abb6-4206-8b8a-6e5a567ba861"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:37.919574762Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1230,"":"downstairs","":"downstairs","session_id":"d6f47531-abb6-4206-8b8a-6e5a567ba861"} {"msg":"[2]R flush_numbers[0..12]: [19, 16, 16, 13, 12, 6, 1, 18, 19, 19, 16, 19]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:37.919588471Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1230,"":"downstairs","":"downstairs","session_id":"d6f47531-abb6-4206-8b8a-6e5a567ba861"} {"msg":"[2]R generation[0..12]: [11, 10, 10, 8, 7, 4, 1, 10, 11, 11, 10, 11]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:37.91960189Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1230,"":"downstairs","":"downstairs","session_id":"d6f47531-abb6-4206-8b8a-6e5a567ba861"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:37.919615569Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1230,"":"downstairs","":"downstairs","session_id":"d6f47531-abb6-4206-8b8a-6e5a567ba861"} {"msg":"Max found gen is 12","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:37.919631578Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1230,"":"downstairs","":"downstairs","session_id":"d6f47531-abb6-4206-8b8a-6e5a567ba861"} {"msg":"Generation requested: 12 >= found:12","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:37.919644957Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1230,"":"downstairs","":"downstairs","session_id":"d6f47531-abb6-4206-8b8a-6e5a567ba861"} {"msg":"Next flush: 20","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:37.919657866Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1230,"":"downstairs","":"downstairs","session_id":"d6f47531-abb6-4206-8b8a-6e5a567ba861"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:37.919670585Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1230,"":"downstairs","":"downstairs","session_id":"d6f47531-abb6-4206-8b8a-6e5a567ba861"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:37.919683844Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF",Wait for a query_work_queue command to finish before sending IO "pid":1230,"session_id":"d6f47531-abb6-4206-8b8a-6e5a567ba861"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:37.919700703Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1230,"client":"0","":"downstairs","session_id":"d6f47531-abb6-4206-8b8a-6e5a567ba861"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:37.919714082Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1230,"client":"1","":"downstairs","session_id":"d6f47531-abb6-4206-8b8a-6e5a567ba861"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:37.919727151Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1230,"client":"2","":"downstairs","session_id":"d6f47531-abb6-4206-8b8a-6e5a567ba861"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:37.91974027Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1230,"session_id":"d6f47531-abb6-4206-8b8a-6e5a567ba861"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:37.919752419Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1230,"session_id":"d6f47531-abb6-4206-8b8a-6e5a567ba861"} {"msg":"53e104f4-08d6-4bdd-b3c1-8b6418715aa2 is now active with session: d6f47531-abb6-4206-8b8a-6e5a567ba861","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:37.919765049Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1230,"session_id":"d6f47531-abb6-4206-8b8a-6e5a567ba861"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:37.919776948Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1230,"session_id":"d6f47531-abb6-4206-8b8a-6e5a567ba861"} {"msg":"The guest has finished waiting for activation with:12","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:37.919788917Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1230} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 485 len 36864 data: 4 4 4 3 4 4 4 3 3 02/30 Read block 538 len 36864 03/30 Read block 330 len 16384 04/30 Write block 265 len 32768 data: 2 2 2 2 2 2 2 3 05/30 Read block 62 len 4096 06/30 Write block 379 len 40960 data: 3 3 3 3 3 2 2 2 2 2 07/30 Write block 8 len 36864 data: 2 2 2 2 2 2 3 3 2 08/30 Read block 36 len 16384 09/30 Read block 400 len 16384 10/30 Read block 44 len 4096 11/30 Read block 262 len 12288 12/30 Write block 531 len 8192 data: 2 2 13/30 Read block 541 len 40960 14/30 Flush 15/30 Write block 38 len 20480 data: 4 5 4 4 3 16/30 Write block 385 len 40960 data: 3 3 3 3 3 3 3 3 3 4 17/30 Read block 123 len 16384 18/30 Write block 24 len 32768 data: 2 2 2 2 2 2 2 2 19/30 Write block 160 len 28672 data: 2 2 3 3 3 3 2 20/30 Read block 135 len 20480 21/30 Read block 595 len 20480 22/30 Write block 58 len 12288 data: 4 3 3 23/30 Read block 352 len 36864 24/30 Write block 344 len 28672 data: 2 2 2 2 2 2 2 25/30 Read block 436 len 24576 26/30 Write block 516 len 36864 data: 2 3 5 5 5 5 5 5 5 27/30 Write block 272 len 12288 data: 4 3 3 28/30 Write block 372 len 20480 data: 3 2 2 2 2 29/30 Read block 353 len 20480 30/30 Read block 427 len 32768 ---------------------------------------------------------------- Crucible gen:12 GIO:true work queues: Upstairs:0 downstairs:16 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1014 Acked Write 5 Done Done Done false 1015 Acked Write 10 Done Done Done false 1016 Acked Read 4 Done Done Done false 1017 Acked Write 8 Done Done Done false 1018 Acked Write 7 Done Done Done false 1019 Acked Read 5 Done Done Done false 1020 Acked Read 5 Done Done Done false 1021 Acked Write 3 Done Done Done false 1022 Acked Read 9 Done Done Done false 1023 Acked Write 7 Done Done Done false 1024 Acked Read 6 Done Done Done false 1025 Acked Write 9 Done Done Done false 1026 Acked Write 3 Done Done Done false 1027 Acked Write 5 Done Done Done false 1028 Acked Read 5 Done Done Done false 1029 Acked Read 8 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 0 1 Done 15 16 16 47 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1013 1013 1013 Downstairs last five completed: 1013 1012 1011 1010 1009 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:33:37.940491879Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1230} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 12 12 12 20 20 20 F F F 1 020-039 10 10 10 16 16 16 T T T 2 040-059 10 10 10 16 16 16 T T T 3 060-079 8 8 8 13 13 13 T T T 4 080-099 7 7 7 12 12 12 F F F 5 100-119 4 4 4 6 6 6 F F F 6 120-139 1 1 1 1 1 1 F F F 7 140-159 10 10 10 18 18 18 F F F 8 160-179 11 11 11 19 19 19 T T T 9 180-199 11 11 11 19 19 19 F F F 10 200-219 10 10 10 16 16 16 F F F 11 220-239 11 11 11 19 19 19 F F F 12 240-259 10 10 10 16 16 16 F F F 13 260-279 12 12 12 20 20 20 T T T 14 280-299 11 11 11 19 19 19 F F F 15 300-319 11 11 11 19 19 19 F F F 16 320-339 9 9 9 15 15 15 F F F 17 340-359 11 11 11 19 19 19 T T T 18 360-379 12 12 12 20 20 20 T T T 19 380-399 12 12 12 20 20 20 T T T 20 400-419 9 9 9 15 15 15 F F F 21 420-439 9 9 9 15 15 15 F F F 22 440-459 11 11 11 19 19 19 F F F 23 460-479 10 10 10 16 16 16 F F F 24 480-499 12 12 12 20 20 20 F F F 25 500-519 9 9 9 15 15 15 T T T 26 520-539 12 12 12 20 20 20 T T T 27 540-559 10 10 10 16 16 16 F F F 28 560-579 9 9 9 15 15 15 F F F 29 580-599 1 1 1 1 1 1 F F F Max gen: 12, Max flush: 20 A Difference in extent metadata is expected here On loop 6 Verifying data now No change after verify Loop: 6 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 12 12 12 20 20 20 F F F 1 020-039 13 13 13 21 21 21 F F F 2 040-059 13 13 13 21 21 21 F F F 3 060-079 13 13 13 21 21 21 F F F 4 080-099 7 7 7 12 12 12 F F F 5 100-119 4 4 4 6 6 6 F F F 6 120-139 1 1 1 1 1 1 F F F 7 140-159 10 10 10 18 18 18 F F F 8 160-179 13 13 13 21 21 21 F F F 9 180-199 11 11 11 19 19 19 F F F 10 200-219 10 10 10 16 16 16 F F F 11 220-239 11 11 11 19 19 19 F F F 12 240-259 10 10 10 16 16 16 F F F 13 260-279 13 13 13 21 21 21 F F F 14 280-299 11 11 11 19 19 19 F F F 15 300-319 11 11 11 19 19 19 F F F 16 320-339 9 9 9 15 15 15 F F F 17 340-359 13 13 13 21 21 21 F F F 18 360-379 13 13 13 21 21 21 F F F 19 380-399 13 13 13 21 21 21 F F F 20 400-419 9 9 9 15 15 15 F F F 21 420-439 9 9 9 15 15 15 F F F 22 440-459 11 11 11 19 19 19 F F F 23 460-479 10 10 10 16 16 16 F F F 24 480-499 12 12 12 20 20 20 F F F 25 500-519 13 13 13 21 21 21 F F F 26 520-539 13 13 13 21 21 21 F F F 27 540-559 10 10 10 16 16 16 F F F 28 560-579 9 9 9 15 15 15 F F F 29 580-599 1 1 1 1 1 1 F F F Max gen: 13, Max flush: 21 Begin loop 7 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:42.265598587Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1238,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:42.310207482Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1238,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:42.310842618Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1238} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"26e7d501b33f47379e53601720353c2983e7654b\",\n git_commit_timestamp: \"2024-12-12T20:16:40.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:42.310875395Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1238} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:42.310889044Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1238} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:42.310903434Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1238} {"msg":"Crucible 2af68e7f-076f-42da-8fa3-93e530e9213e has session id: 35929aaf-91fc-48d6-a8f0-b21bd446aff2","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:42.311351332Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1238,"session_id":"35929aaf-91fc-48d6-a8f0-b21bd446aff2"} {"msg":"Upstairs opts: Upstairs UUID: 2af68e7f-076f-42da-8fa3-93e530e9213e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:42.311400519Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1238,"session_id":"35929aaf-91fc-48d6-a8f0-b21bd446aff2"} {"msg":"Crucible stats registered with UUID: 2af68e7f-076f-42da-8fa3-93e530e9213e","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:42.311428367Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1238,"session_id":"35929aaf-91fc-48d6-a8f0-b21bd446aff2"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:42.342318037Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1238,"task":"crutest"} {"msg":"The guest has requested activation with gen:14","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:42.342345305Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1238} {"msg":"2af68e7f-076f-42da-8fa3-93e530e9213e active request set","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:42.342408731Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1238,"session_id":"35929aaf-91fc-48d6-a8f0-b21bd446aff2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:42.3424267Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1238,"client":"0","":"downstairs","session_id":"35929aaf-91fc-48d6-a8f0-b21bd446aff2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:42.342450838Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1238,"client":"0","":"downstairs","session_id":"35929aaf-91fc-48d6-a8f0-b21bd446aff2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:42.342465217Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1238,"client":"1","":"downstairs","session_id":"35929aaf-91fc-48d6-a8f0-b21bd446aff2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:42.342479746Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1238,"client":"1","":"downstairs","session_id":"35929aaf-91fc-48d6-a8f0-b21bd446aff2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:42.342493235Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1238,"client":"2","":"downstairs","session_id":"35929aaf-91fc-48d6-a8f0-b21bd446aff2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:42.342507684Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1238,"client":"2","":"downstairs","session_id":"35929aaf-91fc-48d6-a8f0-b21bd446aff2"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:42.342530613Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1238,"":"io task","client":"2","":"downstairs","session_id":"35929aaf-91fc-48d6-a8f0-b21bd446aff2"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:42.342559021Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1238,"":"io task","client":"0","":"downstairs","session_id":"35929aaf-91fc-48d6-a8f0-b21bd446aff2"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:42.342594198Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1238,"":"io task","client":"1","":"downstairs","session_id":"35929aaf-91fc-48d6-a8f0-b21bd446aff2"} {"msg":"ds_connection connected from Ok(127.0.0.1:40617)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:42.342682772Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1238,"":"io task","client":"2","":"downstairs","session_id":"35929aaf-91fc-48d6-a8f0-b21bd446aff2"} {"msg":"ds_connection connected from Ok(127.0.0.1:61811)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:42.342729269Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1238,"":"io task","client":"0","":"downstairs","session_id":"35929aaf-91fc-48d6-a8f0-b21bd446aff2"} {"msg":"ds_connection connected from Ok(127.0.0.1:43634)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:42.342972142Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1238,"":"io task","client":"1","":"downstairs","session_id":"35929aaf-91fc-48d6-a8f0-b21bd446aff2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:42.343512995Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1238,"client":"0","":"downstairs","session_id":"35929aaf-91fc-48d6-a8f0-b21bd446aff2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:42.343563651Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1238,"client":"1","":"downstairs","session_id":"35929aaf-91fc-48d6-a8f0-b21bd446aff2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:42.343598189Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1238,"client":"2","":"downstairs","session_id":"35929aaf-91fc-48d6-a8f0-b21bd446aff2"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:42.344139192Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1238,"client":"0","":"downstairs","session_id":"35929aaf-91fc-48d6-a8f0-b21bd446aff2"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:42.344255544Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1238,"client":"1","":"downstairs","session_id":"35929aaf-91fc-48d6-a8f0-b21bd446aff2"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:42.344411193Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1238,"client":"2","":"downstairs","session_id":"35929aaf-91fc-48d6-a8f0-b21bd446aff2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:42.344648767Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1238,"session_id":"35929aaf-91fc-48d6-a8f0-b21bd446aff2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:42.344732471Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1238,"session_id":"35929aaf-91fc-48d6-a8f0-b21bd446aff2"} {"msg":"[0]R flush_numbers[0..12]: [20, 21, 21, 21, 12, 6, 1, 18, 21, 19, 16, 19]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:42.344921938Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1238,"":"downstairs","":"downstairs","session_id":"35929aaf-91fc-48d6-a8f0-b21bd446aff2"} {"msg":"[0]R generation[0..12]: [12, 13, 13, 13, 7, 4, 1, 10, 13, 11, 10, 11]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:42.344941296Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1238,"":"downstairs","":"downstairs","session_id":"35929aaf-91fc-48d6-a8f0-b21bd446aff2"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:42.344956155Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1238,"":"downstairs","":"downstairs","session_id":"35929aaf-91fc-48d6-a8f0-b21bd446aff2"} {"msg":"[1]R flush_numbers[0..12]: [20, 21, 21, 21, 12, 6, 1, 18, 21, 19, 16, 19]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:42.344970034Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1238,"":"downstairs","":"downstairs","session_id":"35929aaf-91fc-48d6-a8f0-b21bd446aff2"} {"msg":"[1]R generation[0..12]: [12, 13, 13, 13, 7, 4, 1, 10, 13, 11, 10, 11]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:42.344987223Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1238,"":"downstairs","":"downstairs","session_id":"35929aaf-91fc-48d6-a8f0-b21bd446aff2"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:42.345001852Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1238,"":"downstairs","":"downstairs","session_id":"35929aaf-91fc-48d6-a8f0-b21bd446aff2"} {"msg":"[2]R flush_numbers[0..12]: [20, 21, 21, 21, 12, 6, 1, 18, 21, 19, 16, 19]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:42.345015421Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1238,"":"downstairs","":"downstairs","session_id":"35929aaf-91fc-48d6-a8f0-b21bd446aff2"} {"msg":"[2]R generation[0..12]: [12, 13, 13, 13, 7, 4, 1, 10, 13, 11, 10, 11]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:42.34502882Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1238,"":"downstairs","":"downstairs","session_id":"35929aaf-91fc-48d6-a8f0-b21bd446aff2"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:42.345042679Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1238,"":"downstairs","":"downstairs","session_id":"35929aaf-91fc-48d6-a8f0-b21bd446aff2"} {"msg":"Max found gen is 14","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:42.345055708Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1238,"":"downstairs","":"downstairs","session_id":"35929aaf-91fc-48d6-a8f0-b21bd446aff2"} {"msg":"Generation requested: 14 >= found:14","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:42.345068887Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1238,"":"downstairs","":"downstairs","session_id":"35929aaf-91fc-48d6-a8f0-b21bd446aff2"} {"msg":"Next flush: 22","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:42.345081727Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1238,"":"downstairs","":"downstairs","session_id":"35929aaf-91fc-48d6-a8f0-b21bd446aff2"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:42.345094526Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1238,"":"Wait for a query_work_queue command to finish before sending IO downstairs","":"downstairs","session_id":"35929aaf-91fc-48d6-a8f0-b21bd446aff2"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:42.345116734Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1238,"session_id":"35929aaf-91fc-48d6-a8f0-b21bd446aff2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:42.345139633Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1238,"client":"0","":"downstairs","session_id":"35929aaf-91fc-48d6-a8f0-b21bd446aff2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:42.345165721Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1238,"client":"1","":"downstairs","session_id":"35929aaf-91fc-48d6-a8f0-b21bd446aff2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:42.345190869Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1238,"client":"2","":"downstairs","session_id":"35929aaf-91fc-48d6-a8f0-b21bd446aff2"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:42.345216117Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1238,"session_id":"35929aaf-91fc-48d6-a8f0-b21bd446aff2"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2024-12-12T20:33:42.345239656Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1238,"session_id":"35929aaf-91fc-48d6-a8f0-b21bd446aff2"} {"msg":"2af68e7f-076f-42da-8fa3-93e530e9213e is now active with session: 35929aaf-91fc-48d6-a8f0-b21bd446aff2","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:42.345271234Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1238,"session_id":"35929aaf-91fc-48d6-a8f0-b21bd446aff2"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:42.345286932Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1238,"session_id":"35929aaf-91fc-48d6-a8f0-b21bd446aff2"} {"msg":"The guest has finished waiting for activation with:14","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:42.345299432Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1238} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 72 len 16384 data: 2 2 2 2 02/30 Flush 03/30 Write block 431 len 40960 data: 2 2 2 2 2 2 2 2 3 3 04/30 Write block 152 len 4096 data: 2 05/30 Flush 06/30 Write block 557 len 4096 data: 4 07/30 Read block 9 len 40960 08/30 Read block 198 len 12288 09/30 Flush 10/30 Write block 275 len 16384 data: 2 2 2 2 11/30 Flush 12/30 Write block 107 len 8192 data: 3 3 13/30 Read block 361 len 12288 14/30 Read block 270 len 24576 15/30 Write block 293 len 12288 data: 3 3 3 16/30 Flush 17/30 Write block 109 len 32768 data: 3 3 3 3 3 3 2 2 18/30 Read block 94 len 4096 19/30 Write block 554 len 32768 data: 5 5 4 5 3 3 2 2 20/30 Write block 162 len 4096 data: 4 21/30 Read block 354 len 32768 22/30 Flush 23/30 Write block 582 len 12288 data: 2 2 2 24/30 Read block 352 len 40960 25/30 Write block 75 len 28672 data: 3 2 2 2 2 2 2 26/30 Read block 403 len 24576 27/30 Write block 371 len 16384 data: 3 4 3 3 28/30 Write block 63 len 24576 data: 2 2 2 2 2 2 29/30 Read block 471 len 8192 30/30 Write block 322 len 8192 data: 3 3 ---------------------------------------------------------------- Crucible gen:14 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 3 Done Done Done false 1023 Acked Read 10 Done Done Done false 1024 Acked Write 7 Done Done Done false 1025 Acked Read 6 Done Done Done false 1026 Acked Write 4 Done Done Done false 1027 Acked Write 6 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 7 7 7 21 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1021 1021 1021 Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:33:42.363263783Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1238} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 12 12 12 20 20 20 F F F 1 020-039 13 13 13 21 21 21 F F F 2 040-059 13 13 13 21 21 21 F F F 3 060-079 14 14 14 22 22 22 T T T 4 080-099 7 7 7 12 12 12 T T T 5 100-119 14 14 14 27 27 27 F F F 6 120-139 1 1 1 1 1 1 F F F 7 140-159 14 14 14 23 23 23 F F F 8 160-179 14 14 14 27 27 27 F F F 9 180-199 11 11 11 19 19 19 F F F 10 200-219 10 10 10 16 16 16 F F F 11 220-239 11 11 11 19 19 19 F F F 12 240-259 10 10 10 16 16 16 F F F 13 260-279 14 14 14 25 25 25 F F F 14 280-299 14 14 14 26 26 26 F F F 15 300-319 11 11 11 19 19 19 F F F 16 320-339 9 9 9 15 15 15 T T T 17 340-359 13 13 13 21 21 21 F F F 18 360-379 13 13 13 21 21 21 T T T 19 380-399 13 13 13 21 21 21 F F F 20 400-419 9 9 9 15 15 15 F F F 21 420-439 14 14 14 23 23 23 F F F 22 440-459 14 14 14 23 23 23 F F F 23 460-479 10 10 10 16 16 16 F F F 24 480-499 12 12 12 20 20 20 F F F 25 500-519 13 13 13 21 21 21 F F F 26 520-539 13 13 13 21 21 21 F F F 27 540-559 14 14 14 27 27 27 F F F 28 560-579 14 14 14 27 27 27 F F F 29 580-599 1 1 1 1 1 1 T T T Max gen: 14, Max flush: 27 A Difference in extent metadata is expected here On loop 7 Verifying data now No change after verify Loop: 7 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 12 12 12 20 20 20 F F F 1 020-039 13 13 13 21 21 21 F F F 2 040-059 13 13 13 21 21 21 F F F 3 060-079 15 15 15 28 28 28 F F F 4 080-099 15 15 15 28 28 28 F F F 5 100-119 14 14 14 27 27 27 F F F 6 120-139 1 1 1 1 1 1 F F F 7 140-159 14 14 14 23 23 23 F F F 8 160-179 14 14 14 27 27 27 F F F 9 180-199 11 11 11 19 19 19 F F F 10 200-219 10 10 10 16 16 16 F F F 11 220-239 11 11 11 19 19 19 F F F 12 240-259 10 10 10 16 16 16 F F F 13 260-279 14 14 14 25 25 25 F F F 14 280-299 14 14 14 26 26 26 F F F 15 300-319 11 11 11 19 19 19 F F F 16 320-339 15 15 15 28 28 28 F F F 17 340-359 13 13 13 21 21 21 F F F 18 360-379 15 15 15 28 28 28 F F F 19 380-399 13 13 13 21 21 21 F F F 20 400-419 9 9 9 15 15 15 F F F 21 420-439 14 14 14 23 23 23 F F F 22 440-459 14 14 14 23 23 23 F F F 23 460-479 10 10 10 16 16 16 F F F 24 480-499 12 12 12 20 20 20 F F F 25 500-519 13 13 13 21 21 21 F F F 26 520-539 13 13 13 21 21 21 F F F 27 540-559 14 14 14 27 27 27 F F F 28 560-579 14 14 14 27 27 27 F F F 29 580-599 15 15 15 28 28 28 F F F Max gen: 15, Max flush: 28 Begin loop 8 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:46.628024492Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1246,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:33:46.671088802Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1246,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:46.671483795Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1246,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:46.703144592Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1246,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:46.703743011Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1246} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"26e7d501b33f47379e53601720353c2983e7654b\",\n git_commit_timestamp: \"2024-12-12T20:16:40.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:46.703777338Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1246} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:46.703796587Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1246} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:46.703811316Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1246} {"msg":"Crucible b2a08e80-2301-4b46-8429-5d5046f45450 has session id: b2eef66a-a573-43f0-80b9-38ce3fa75b1e","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:46.704209429Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1246,"session_id":"b2eef66a-a573-43f0-80b9-38ce3fa75b1e"} {"msg":"Upstairs opts: Upstairs UUID: b2a08e80-2301-4b46-8429-5d5046f45450, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:46.704253555Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1246,"session_id":"b2eef66a-a573-43f0-80b9-38ce3fa75b1e"} {"msg":"Crucible stats registered with UUID: b2a08e80-2301-4b46-8429-5d5046f45450","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:46.704278504Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1246,"session_id":"b2eef66a-a573-43f0-80b9-38ce3fa75b1e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:46.734844146Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1246,"task":"crutest"} {"msg":"The guest has requested activation with gen:16","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:46.734883273Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1246} {"msg":"b2a08e80-2301-4b46-8429-5d5046f45450 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:46.734911501Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1246,"session_id":"b2eef66a-a573-43f0-80b9-38ce3fa75b1e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:46.73493046Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1246,"client":"0","":"downstairs","session_id":"b2eef66a-a573-43f0-80b9-38ce3fa75b1e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:46.734975277Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1246,"client":"0","":"downstairs","session_id":"b2eef66a-a573-43f0-80b9-38ce3fa75b1e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:46.735010805Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1246,"client":"1","":"downstairs","session_id":"b2eef66a-a573-43f0-80b9-38ce3fa75b1e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:46.735045982Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1246,"client":"1","":"downstairs","session_id":"b2eef66a-a573-43f0-80b9-38ce3fa75b1e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:46.73508046Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1246,"client":"2","":"downstairs","session_id":"b2eef66a-a573-43f0-80b9-38ce3fa75b1e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:46.735119687Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1246,"client":"2","":"downstairs","session_id":"b2eef66a-a573-43f0-80b9-38ce3fa75b1e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:46.735156244Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1246,"":"io task","client":"2","":"downstairs","session_id":"b2eef66a-a573-43f0-80b9-38ce3fa75b1e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:46.735193082Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1246,"":"io task","client":"0","":"downstairs","session_id":"b2eef66a-a573-43f0-80b9-38ce3fa75b1e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:46.735243649Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1246,"":"io task","client":"1","":"downstairs","session_id":"b2eef66a-a573-43f0-80b9-38ce3fa75b1e"} {"msg":"ds_connection connected from Ok(127.0.0.1:39186)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:46.735334662Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1246,"":"io task","client":"2","":"downstairs","session_id":"b2eef66a-a573-43f0-80b9-38ce3fa75b1e"} {"msg":"ds_connection connected from Ok(127.0.0.1:34336)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:46.73537048Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1246,"":"io task","client":"1","":"downstairs","session_id":"b2eef66a-a573-43f0-80b9-38ce3fa75b1e"} {"msg":"ds_connection connected from Ok(127.0.0.1:41864)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:46.735397098Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1246,"":"io task","client":"0","":"downstairs","session_id":"b2eef66a-a573-43f0-80b9-38ce3fa75b1e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:46.73594184Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1246,"client":"1","":"downstairs","session_id":"b2eef66a-a573-43f0-80b9-38ce3fa75b1e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:46.736127917Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1246,"client":"0","":"downstairs","session_id":"b2eef66a-a573-43f0-80b9-38ce3fa75b1e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:46.736158355Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1246,"client":"2","":"downstairs","session_id":"b2eef66a-a573-43f0-80b9-38ce3fa75b1e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:46.736594485Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1246,"client":"1","":"downstairs","session_id":"b2eef66a-a573-43f0-80b9-38ce3fa75b1e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:46.736623063Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1246,"client":"2","":"downstairs","session_id":"b2eef66a-a573-43f0-80b9-38ce3fa75b1e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:46.736889375Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1246,"client":"0","":"downstairs","session_id":"b2eef66a-a573-43f0-80b9-38ce3fa75b1e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:46.737091101Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1246,"session_id":"b2eef66a-a573-43f0-80b9-38ce3fa75b1e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:46.737215173Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1246,"session_id":"b2eef66a-a573-43f0-80b9-38ce3fa75b1e"} {"msg":"[0]R flush_numbers[0..12]: [20, 21, 21, 28, 28, 27, 1, 23, 27, 19, 16, 19]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:46.737425948Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1246,"":"downstairs","":"downstairs","session_id":"b2eef66a-a573-43f0-80b9-38ce3fa75b1e"} {"msg":"[0]R generation[0..12]: [12, 13, 13, 15, 15, 14, 1, 14, 14, 11, 10, 11]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:46.737446177Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1246,"":"downstairs","":"downstairs","session_id":"b2eef66a-a573-43f0-80b9-38ce3fa75b1e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:46.737461056Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1246,"":"downstairs","":"downstairs","session_id":"b2eef66a-a573-43f0-80b9-38ce3fa75b1e"} {"msg":"[1]R flush_numbers[0..12]: [20, 21, 21, 28, 28, 27, 1, 23, 27, 19, 16, 19]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:46.737475075Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1246,"":"downstairs","":"downstairs","session_id":"b2eef66a-a573-43f0-80b9-38ce3fa75b1e"} {"msg":"[1]R generation[0..12]: [12, 13, 13, 15, 15, 14, 1, 14, 14, 11, 10, 11]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:46.737488924Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1246,"":"downstairs","":"downstairs","session_id":"b2eef66a-a573-43f0-80b9-38ce3fa75b1e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:46.737502803Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1246,"":"downstairs","":"downstairs","session_id":"b2eef66a-a573-43f0-80b9-38ce3fa75b1e"} {"msg":"[2]R flush_numbers[0..12]: [20, 21, 21, 28, 28, 27, 1, 23, 27, 19, 16, 19]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:46.737516262Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1246,"":"downstairs","":"downstairs","session_id":"b2eef66a-a573-43f0-80b9-38ce3fa75b1e"} {"msg":"[2]R generation[0..12]: [12, 13, 13, 15, 15, 14, 1, 14, 14, 11, 10, 11]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:46.737529601Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1246,"":"downstairs","":"downstairs","session_id":"b2eef66a-a573-43f0-80b9-38ce3fa75b1e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:46.73754445Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1246,"":"downstairs","":"downstairs","session_id":"b2eef66a-a573-43f0-80b9-38ce3fa75b1e"} {"msg":"Max found gen is 16","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:46.737557799Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1246,"":"downstairs","":"downstairs","session_id":"b2eef66a-a573-43f0-80b9-38ce3fa75b1e"} {"msg":"Generation requested: 16 >= found:16","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:46.737571018Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1246,"":"downstairs","":"downstairs","session_id":"b2eef66a-a573-43f0-80b9-38ce3fa75b1e"} {"msg":"Next flush: 29","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:46.737583817Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1246,"":"downstairs","":"downstairs","session_id":"b2eef66a-a573-43f0-80b9-38ce3fa75b1e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:46.737596606Z","Wait for a query_work_queue command to finish before sending IO hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1246,"":"downstairs","":"downstairs","session_id":"b2eef66a-a573-43f0-80b9-38ce3fa75b1e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:46.737620435Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1246,"session_id":"b2eef66a-a573-43f0-80b9-38ce3fa75b1e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:46.737647153Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1246,"client":"0","":"downstairs","session_id":"b2eef66a-a573-43f0-80b9-38ce3fa75b1e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:46.737672351Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1246,"client":"1","":"downstairs","session_id":"b2eef66a-a573-43f0-80b9-38ce3fa75b1e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 :30,"time":"2024-12-12T20:33:46.737699549Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1246,"client":"2","":"downstairs","session_id":"b2eef66a-a573-43f0-80b9-38ce3fa75b1e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:46.737714088Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1246,"session_id":"b2eef66a-a573-43f0-80b9-38ce3fa75b1e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:46.737726337Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1246,"session_id":"b2eef66a-a573-43f0-80b9-38ce3fa75b1e"} {"msg":"b2a08e80-2301-4b46-8429-5d5046f45450 is now active with session: b2eef66a-a573-43f0-80b9-38ce3fa75b1e","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:46.737739166Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1246,"session_id":"b2eef66a-a573-43f0-80b9-38ce3fa75b1e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:46.737751296Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1246,"session_id":"b2eef66a-a573-43f0-80b9-38ce3fa75b1e"} {"msg":"The guest has finished waiting for activation with:16","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:46.737763305Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1246} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 315 len 32768 data: 5 5 4 4 3 3 3 4 02/30 Write block 2 len 12288 data: 2 2 2 03/30 Read block 365 len 40960 04/30 Read block 236 len 24576 05/30 Read block 39 len 32768 06/30 Write block 346 len 12288 data: 3 3 3 07/30 Write block 511 len 20480 data: 4 3 3 3 2 08/30 Write block 157 len 28672 data: 2 2 2 3 3 5 4 09/30 Write block 17 len 24576 data: 2 3 2 2 2 2 10/30 Read block 491 len 36864 11/30 Write block 576 len 16384 data: 2 2 2 2 12/30 Write block 352 len 16384 data: 2 3 3 3 13/30 Read block 418 len 12288 14/30 Write block 551 len 36864 data: 4 4 5 6 6 5 6 4 4 15/30 Read block 156 len 32768 16/30 Write block 239 len 20480 data: 2 2 2 2 2 17/30 Write block 457 len 12288 data: 3 2 2 18/30 Flush 19/30 Write block 418 len 20480 data: 4 3 3 3 3 20/30 Read block 240 len 16384 21/30 Read block 331 len 4096 22/30 Flush 23/30 Write block 518 len 8192 data: 6 6 24/30 Read block 145 len 8192 25/30 Write block 258 len 36864 data: 3 3 3 3 2 2 2 3 3 26/30 Read block 81 len 32768 27/30 Write block 43 len 4096 data: 3 28/30 Read block 559 len 28672 29/30 Read block 187 len 24576 30/30 Write block 362 len 12288 data: 3 4 4 ---------------------------------------------------------------- Crucible gen:16 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 2 Done Done Done false 1023 Acked Read 2 Done Done Done false 1024 Acked Write 9 Done Done Done false 1025 Acked Read 8 Done Done Done false 1026 Acked Write 1 Done Done Done false 1027 Acked Read 7 Done Done Done false 1028 Acked Read 6 Done Done Done false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 7 7 7 21 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1021 1021 1021 Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:33:46.756285178Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1246} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 16 16 16 29 29 29 F F F 1 020-039 16 16 16 29 29 29 F F F 2 040-059 13 13 13 21 21 21 T T T 3 060-079 15 15 15 28 28 28 F F F 4 080-099 15 15 15 28 28 28 F F F 5 100-119 14 14 14 27 27 27 F F F 6 120-139 1 1 1 1 1 1 F F F 7 140-159 16 16 16 29 29 29 F F F 8 160-179 16 16 16 29 29 29 F F F 9 180-199 11 11 11 19 19 19 F F F 10 200-219 10 10 10 16 16 16 F F F 11 220-239 16 16 16 29 29 29 F F F 12 240-259 16 16 16 29 29 29 T T T 13 260-279 14 14 14 25 25 25 T T T 14 280-299 14 14 14 26 26 26 F F F 15 300-319 16 16 16 29 29 29 F F F 16 320-339 16 16 16 29 29 29 F F F 17 340-359 16 16 16 29 29 29 F F F 18 360-379 15 15 15 28 28 28 T T T 19 380-399 13 13 13 21 21 21 F F F 20 400-419 16 16 16 30 30 30 F F F 21 420-439 16 16 16 30 30 30 F F F 22 440-459 16 16 16 29 29 29 F F F 23 460-479 10 10 10 16 16 16 F F F 24 480-499 12 12 12 20 20 20 F F F 25 500-519 16 16 16 29 29 29 T T T 26 520-539 13 13 13 21 21 21 F F F 27 540-559 16 16 16 29 29 29 F F F 28 560-579 16 16 16 29 29 29 F F F 29 580-599 15 15 15 28 28 28 F F F Max gen: 16, Max flush: 30 A Difference in extent metadata is expected here On loop 8 Verifying data now No change after verify Loop: 8 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 16 16 16 29 29 29 F F F 1 020-039 16 16 16 29 29 29 F F F 2 040-059 17 17 17 31 31 31 F F F 3 060-079 15 15 15 28 28 28 F F F 4 080-099 15 15 15 28 28 28 F F F 5 100-119 14 14 14 27 27 27 F F F 6 120-139 1 1 1 1 1 1 F F F 7 140-159 16 16 16 29 29 29 F F F 8 160-179 16 16 16 29 29 29 F F F 9 180-199 11 11 11 19 19 19 F F F 10 200-219 10 10 10 16 16 16 F F F 11 220-239 16 16 16 29 29 29 F F F 12 240-259 17 17 17 31 31 31 F F F 13 260-279 17 17 17 31 31 31 F F F 14 280-299 14 14 14 26 26 26 F F F 15 300-319 16 16 16 29 29 29 F F F 16 320-339 16 16 16 29 29 29 F F F 17 340-359 16 16 16 29 29 29 F F F 18 360-379 17 17 17 31 31 31 F F F 19 380-399 13 13 13 21 21 21 F F F 20 400-419 16 16 16 30 30 30 F F F 21 420-439 16 16 16 30 30 30 F F F 22 440-459 16 16 16 29 29 29 F F F 23 460-479 10 10 10 16 16 16 F F F 24 480-499 12 12 12 20 20 20 F F F 25 500-519 17 17 17 31 31 31 F F F 26 520-539 13 13 13 21 21 21 F F F 27 540-559 16 16 16 29 29 29 F F F 28 560-579 16 16 16 29 29 29 F F F 29 580-599 15 15 15 28 28 28 F F F Max gen: 17, Max flush: 31 Begin loop 9 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:51.05710425Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1254,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:51.101286274Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1254,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:51.10192213Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1254} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"26e7d501b33f47379e53601720353c2983e7654b\",\n git_commit_timestamp: \"2024-12-12T20:16:40.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:51.101958677Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1254} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:51.101972716Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1254} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:51.101986965Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1254} {"msg":"Crucible 480a5db9-7cbf-484b-ab21-dec92c1fc92d has session id: a376cb60-84cc-402e-8bef-b389346325cf","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:51.102414536Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1254,"session_id":"a376cb60-84cc-402e-8bef-b389346325cf"} {"msg":"Upstairs opts: Upstairs UUID: 480a5db9-7cbf-484b-ab21-dec92c1fc92d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:51.102458773Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1254,"session_id":"a376cb60-84cc-402e-8bef-b389346325cf"} {"msg":"Crucible stats registered with UUID: 480a5db9-7cbf-484b-ab21-dec92c1fc92d","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:51.102478771Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1254,"session_id":"a376cb60-84cc-402e-8bef-b389346325cf"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:51.133424328Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1254,"task":"crutest"} {"msg":"The guest has requested activation with gen:18","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:51.133449896Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1254} {"msg":"480a5db9-7cbf-484b-ab21-dec92c1fc92d active request set","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:51.13352733Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1254,"session_id":"a376cb60-84cc-402e-8bef-b389346325cf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:51.133544109Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1254,"client":"0","":"downstairs","session_id":"a376cb60-84cc-402e-8bef-b389346325cf"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:51.133566898Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1254,"client":"0","":"downstairs","session_id":"a376cb60-84cc-402e-8bef-b389346325cf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:51.133584966Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1254,"client":"1","":"downstairs","session_id":"a376cb60-84cc-402e-8bef-b389346325cf"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:51.133599596Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1254,"client":"1","":"downstairs","session_id":"a376cb60-84cc-402e-8bef-b389346325cf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:51.133618334Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1254,"client":"2","":"downstairs","session_id":"a376cb60-84cc-402e-8bef-b389346325cf"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:51.133645282Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1254,"client":"2","":"downstairs","session_id":"a376cb60-84cc-402e-8bef-b389346325cf"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:51.133669791Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1254,"":"io task","client":"2","":"downstairs","session_id":"a376cb60-84cc-402e-8bef-b389346325cf"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:51.133695469Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1254,"":"io task","client":"0","":"downstairs","session_id":"a376cb60-84cc-402e-8bef-b389346325cf"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:51.133718117Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1254,"":"io task","client":"1","":"downstairs","session_id":"a376cb60-84cc-402e-8bef-b389346325cf"} {"msg":"ds_connection connected from Ok(127.0.0.1:50204)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:51.133744615Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1254,"":"io task","client":"2","":"downstairs","session_id":"a376cb60-84cc-402e-8bef-b389346325cf"} {"msg":"ds_connection connected from Ok(127.0.0.1:43451)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:51.133787192Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1254,"":"io task","client":"0","":"downstairs","session_id":"a376cb60-84cc-402e-8bef-b389346325cf"} {"msg":"ds_connection connected from Ok(127.0.0.1:37443)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:51.13382313Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1254,"":"io task","client":"1","":"downstairs","session_id":"a376cb60-84cc-402e-8bef-b389346325cf"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:51.134667242Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1254,"client":"0","":"downstairs","session_id":"a376cb60-84cc-402e-8bef-b389346325cf"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:51.13469691Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1254,"client":"1","":"downstairs","session_id":"a376cb60-84cc-402e-8bef-b389346325cf"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:51.134713209Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1254,"client":"2","":"downstairs","session_id":"a376cb60-84cc-402e-8bef-b389346325cf"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:51.135326806Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1254,"client":"2","":"downstairs","session_id":"a376cb60-84cc-402e-8bef-b389346325cf"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:51.135359134Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1254,"client":"1","":"downstairs","session_id":"a376cb60-84cc-402e-8bef-b389346325cf"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:51.135375433Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1254,"client":"0","":"downstairs","session_id":"a376cb60-84cc-402e-8bef-b389346325cf"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:51.135892927Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1254,"session_id":"a376cb60-84cc-402e-8bef-b389346325cf"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:51.136036307Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1254,"session_id":"a376cb60-84cc-402e-8bef-b389346325cf"} {"msg":"[0]R flush_numbers[0..12]: [29, 29, 31, 28, 28, 27, 1, 29, 29, 19, 16, 29]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:51.136089724Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1254,"":"downstairs","":"downstairs","session_id":"a376cb60-84cc-402e-8bef-b389346325cf"} {"msg":"[0]R generation[0..12]: [16, 16, 17, 15, 15, 14, 1, 16, 16, 11, 10, 16]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:51.136108872Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1254,"":"downstairs","":"downstairs","session_id":"a376cb60-84cc-402e-8bef-b389346325cf"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:51.136124201Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1254,"":"downstairs","":"downstairs","session_id":"a376cb60-84cc-402e-8bef-b389346325cf"} {"msg":"[1]R flush_numbers[0..12]: [29, 29, 31, 28, 28, 27, 1, 29, 29, 19, 16, 29]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:51.136152679Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1254,"":"downstairs","":"downstairs","session_id":"a376cb60-84cc-402e-8bef-b389346325cf"} {"msg":"[1]R generation[0..12]: [16, 16, 17, 15, 15, 14, 1, 16, 16, 11, 10, 16]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:51.136166698Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1254,"":"downstairs","":"downstairs","session_id":"a376cb60-84cc-402e-8bef-b389346325cf"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:51.136180747Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1254,"":"downstairs","":"downstairs","session_id":"a376cb60-84cc-402e-8bef-b389346325cf"} {"msg":"[2]R flush_numbers[0..12]: [29, 29, 31, 28, 28, 27, 1, 29, 29, 19, 16, 29]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:51.136194306Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1254,"":"downstairs","":"downstairs","session_id":"a376cb60-84cc-402e-8bef-b389346325cf"} {"msg":"[2]R generation[0..12]: [16, 16, 17, 15, 15, 14, 1, 16, 16, 11, 10, 16]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:51.136207976Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1254,"":"downstairs","":"downstairs","session_id":"a376cb60-84cc-402e-8bef-b389346325cf"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:51.136221904Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1254,"":"downstairs","":"downstairs","session_id":"a376cb60-84cc-402e-8bef-b389346325cf"} {"msg":"Max found gen is 18","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:51.136234904Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1254,"":"downstairs","":"downstairs","session_id":"a376cb60-84cc-402e-8bef-b389346325cf"} {"msg":"Generation requested: 18 >= found:18","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:51.136251613Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1254,"":"downstairs","":"downstairs","session_id":"a376cb60-84cc-402e-8bef-b389346325cf"} {"msg":"Next flush: 32","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:51.136265022Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1254,"":"downstairs","":"downstairs","session_id":"a376cb60-84cc-402e-8bef-b389346325cf"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-12-12T20:33:51.136277931Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1254,"":"downstairs","":"downstairs","session_id":"a376cb60-84cc-402e-8bef-b389346325cf"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:51.13629337Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1254,"session_id":"a376cb60-84cc-402e-8bef-b389346325cf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:51.136305619Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1254,"client":"0","":"downstairs","session_id":"a376cb60-84cc-402e-8bef-b389346325cf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:51.136319398Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1254,"client":"1","":"downstairs","session_id":"a376cb60-84cc-402e-8bef-b389346325cf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:51.136332777Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1254,"client":"2","":"downstairs","session_id":"a376cb60-84cc-402e-8bef-b389346325cf"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:51.136346206Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1254,"session_id":"a376cb60-84cc-402e-8bef-b389346325cf"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:51.136358415Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1254,"session_id":"a376cb60-84cc-402e-8bef-b389346325cf"} {"msg":"480a5db9-7cbf-484b-ab21-dec92c1fc92d is now active with session: a376cb60-84cc-402e-8bef-b389346325cf","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:51.136371124Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1254,"session_id":"a376cb60-84cc-402e-8bef-b389346325cf"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:51.136382924Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1254,"session_id":"a376cb60-84cc-402e-8bef-b389346325cf"} {"msg":"The guest has finished waiting for activation with:18","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:51.136395173Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1254} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 125 len 28672 data: 2 2 2 2 2 2 2 03/30 Read block 196 len 28672 04/30 Read block 563 len 8192 05/30 Write block 476 len 4096 data: 3 06/30 Write block 263 len 24576 data: 3 3 4 4 3 3 07/30 Read block 141 len 36864 08/30 Write block 389 len 24576 data: 4 4 4 4 4 5 09/30 Read block 504 len 16384 10/30 Read block 140 len 40960 11/30 Write block 184 len 12288 data: 4 4 4 12/30 Read block 286 len 36864 13/30 Read block 46 len 24576 14/30 Flush 15/30 Write block 265 len 32768 data: 5 5 4 4 3 3 3 5 16/30 Write block 562 len 12288 data: 2 2 3 17/30 Write block 156 len 4096 data: 2 18/30 Read block 206 len 28672 19/30 Read block 406 len 40960 20/30 Read block 501 len 40960 21/30 Write block 53 len 28672 data: 3 3 3 3 3 5 4 22/30 Read block 215 len 16384 23/30 Flush 24/30 Write block 140 len 4096 data: 2 25/30 Write block 251 len 40960 data: 3 3 3 4 4 4 3 4 4 4 26/30 Read block 124 len 20480 27/30 Read block 385 len 4096 28/30 Read block 269 len 16384 29/30 Write block 308 len 40960 data: 2 2 2 4 4 4 5 6 6 5 30/30 Write block 235 len 8192 data: 3 2 ---------------------------------------------------------------- Crucible gen:18 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 1 Done Done Done false 1024 Acked Write 10 Done Done Done false 1025 Acked Read 5 Done Done Done false 1026 Acked Read 1 Done Done Done false 1027 Acked Read 4 Done Done Done false 1028 Acked Write 10 Sent Sent Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:33:51.155827923Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1254} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 16 16 16 29 29 29 F F F 1 020-039 16 16 16 29 29 29 F F F 2 040-059 18 18 18 34 34 34 F F F 3 060-079 15 15 15 28 28 28 F F F 4 080-099 15 15 15 28 28 28 F F F 5 100-119 14 14 14 27 27 27 F F F 6 120-139 18 18 18 33 33 33 F F F 7 140-159 18 18 18 34 34 34 T T T 8 160-179 16 16 16 29 29 29 F F F 9 180-199 18 18 18 33 33 33 F F F 10 200-219 10 10 10 16 16 16 F F F 11 220-239 16 16 16 29 29 29 T T T 12 240-259 17 17 17 31 31 31 T T T 13 260-279 18 18 18 34 34 34 T T T 14 280-299 14 14 14 26 26 26 F F F 15 300-319 16 16 16 29 29 29 T T T 16 320-339 16 16 16 29 29 29 F F F 17 340-359 16 16 16 29 29 29 F F F 18 360-379 17 17 17 31 31 31 F F F 19 380-399 18 18 18 33 33 33 F F F 20 400-419 16 16 16 30 30 30 F F F 21 420-439 16 16 16 30 30 30 F F F 22 440-459 16 16 16 29 29 29 F F F 23 460-479 18 18 18 33 33 33 F F F 24 480-499 12 12 12 20 20 20 F F F 25 500-519 17 17 17 31 31 31 F F F 26 520-539 13 13 13 21 21 21 F F F 27 540-559 16 16 16 29 29 29 F F F 28 560-579 18 18 18 34 34 34 F F F 29 580-599 15 15 15 28 28 28 F F F Max gen: 18, Max flush: 34 A Difference in extent metadata is expected here On loop 9 Verifying data now No change after verify Loop: 9 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 16 16 16 29 29 29 F F F 1 020-039 16 16 16 29 29 29 F F F 2 040-059 18 18 18 34 34 34 F F F 3 060-079 15 15 15 28 28 28 F F F 4 080-099 15 15 15 28 28 28 F F F 5 100-119 14 14 14 27 27 27 F F F 6 120-139 18 18 18 33 33 33 F F F 7 140-159 19 19 19 35 35 35 F F F 8 160-179 16 16 16 29 29 29 F F F 9 180-199 18 18 18 33 33 33 F F F 10 200-219 10 10 10 16 16 16 F F F 11 220-239 19 19 19 35 35 35 F F F 12 240-259 19 19 19 35 35 35 F F F 13 260-279 19 19 19 35 35 35 F F F 14 280-299 14 14 14 26 26 26 F F F 15 300-319 19 19 19 35 35 35 F F F 16 320-339 16 16 16 29 29 29 F F F 17 340-359 16 16 16 29 29 29 F F F 18 360-379 17 17 17 31 31 31 F F F 19 380-399 18 18 18 33 33 33 F F F 20 400-419 16 16 16 30 30 30 F F F 21 420-439 16 16 16 30 30 30 F F F 22 440-459 16 16 16 29 29 29 F F F 23 460-479 18 18 18 33 33 33 F F F 24 480-499 12 12 12 20 20 20 F F F 25 500-519 17 17 17 31 31 31 F F F 26 520-539 13 13 13 21 21 21 F F F 27 540-559 16 16 16 29 29 29 F F F 28 560-579 18 18 18 34 34 34 F F F 29 580-599 15 15 15 28 28 28 F F F Max gen: 19, Max flush: 35 Begin loop 10 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:55.427097703Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1262,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:55.47152163Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1262,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:55.472213182Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1262} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"26e7d501b33f47379e53601720353c2983e7654b\",\n git_commit_timestamp: \"2024-12-12T20:16:40.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:55.47224501Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1262} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:55.472259439Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1262} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:55.472273598Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1262} {"msg":"Crucible bbf79a83-7f38-4d66-8317-f50aebcc4570 has session id: 122ce771-863a-4b62-b011-3267e8db4b0a","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:55.47267323Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1262,"session_id":"122ce771-863a-4b62-b011-3267e8db4b0a"} {"msg":"Upstairs opts: Upstairs UUID: bbf79a83-7f38-4d66-8317-f50aebcc4570, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:55.472717867Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1262,"session_id":"122ce771-863a-4b62-b011-3267e8db4b0a"} {"msg":"Crucible stats registered with UUID: bbf79a83-7f38-4d66-8317-f50aebcc4570","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:55.472742566Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1262,"session_id":"122ce771-863a-4b62-b011-3267e8db4b0a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:55.503899077Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1262,"task":"crutest"} {"msg":"The guest has requested activation with gen:20","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:55.503924355Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1262} {"msg":"bbf79a83-7f38-4d66-8317-f50aebcc4570 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:55.504009119Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1262,"session_id":"122ce771-863a-4b62-b011-3267e8db4b0a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:55.504029278Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1262,"client":"0","":"downstairs","session_id":"122ce771-863a-4b62-b011-3267e8db4b0a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:55.504056686Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1262,"client":"0","":"downstairs","session_id":"122ce771-863a-4b62-b011-3267e8db4b0a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:55.504082985Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1262,"client":"1","":"downstairs","session_id":"122ce771-863a-4b62-b011-3267e8db4b0a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:55.504110463Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1262,"client":"1","":"downstairs","session_id":"122ce771-863a-4b62-b011-3267e8db4b0a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:55.504138491Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1262,"client":"2","":"downstairs","session_id":"122ce771-863a-4b62-b011-3267e8db4b0a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:55.504168199Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1262,"client":"2","":"downstairs","session_id":"122ce771-863a-4b62-b011-3267e8db4b0a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:55.504196377Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1262,"":"io task","client":"2","":"downstairs","session_id":"122ce771-863a-4b62-b011-3267e8db4b0a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:55.504225964Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1262,"":"io task","client":"0","":"downstairs","session_id":"122ce771-863a-4b62-b011-3267e8db4b0a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:55.504254443Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1262,"":"io task","client":"1","":"downstairs","session_id":"122ce771-863a-4b62-b011-3267e8db4b0a"} {"msg":"ds_connection connected from Ok(127.0.0.1:62826)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:55.504427741Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1262,"":"io task","client":"2","":"downstairs","session_id":"122ce771-863a-4b62-b011-3267e8db4b0a"} {"msg":"ds_connection connected from Ok(127.0.0.1:57295)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:55.504462218Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1262,"":"io task","client":"0","":"downstairs","session_id":"122ce771-863a-4b62-b011-3267e8db4b0a"} {"msg":"ds_connection connected from Ok(127.0.0.1:51822)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:55.504489026Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1262,"":"io task","client":"1","":"downstairs","session_id":"122ce771-863a-4b62-b011-3267e8db4b0a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:55.504976493Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1262,"client":"0","":"downstairs","session_id":"122ce771-863a-4b62-b011-3267e8db4b0a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:55.50501272Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1262,"client":"1","":"downstairs","session_id":"122ce771-863a-4b62-b011-3267e8db4b0a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:55.505200147Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1262,"client":"2","":"downstairs","session_id":"122ce771-863a-4b62-b011-3267e8db4b0a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:55.505632498Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1262,"client":"1","":"downstairs","session_id":"122ce771-863a-4b62-b011-3267e8db4b0a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:55.505691643Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1262,"client":"0","":"downstairs","session_id":"122ce771-863a-4b62-b011-3267e8db4b0a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:55.505966235Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1262,"client":"2","":"downstairs","session_id":"122ce771-863a-4b62-b011-3267e8db4b0a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:55.506195399Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1262,"session_id":"122ce771-863a-4b62-b011-3267e8db4b0a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:55.506272654Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1262,"session_id":"122ce771-863a-4b62-b011-3267e8db4b0a"} {"msg":"[0]R flush_numbers[0..12]: [29, 29, 34, 28, 28, 27, 33, 35, 29, 33, 16, 35]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:55.506487499Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1262,"":"downstairs","":"downstairs","session_id":"122ce771-863a-4b62-b011-3267e8db4b0a"} {"msg":"[0]R generation[0..12]: [16, 16, 18, 15, 15, 14, 18, 19, 16, 18, 10, 19]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:55.506506167Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1262,"":"downstairs","":"downstairs","session_id":"122ce771-863a-4b62-b011-3267e8db4b0a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:55.506520736Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1262,"":"downstairs","":"downstairs","session_id":"122ce771-863a-4b62-b011-3267e8db4b0a"} {"msg":"[1]R flush_numbers[0..12]: [29, 29, 34, 28, 28, 27, 33, 35, 29, 33, 16, 35]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:55.506534495Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1262,"":"downstairs","":"downstairs","session_id":"122ce771-863a-4b62-b011-3267e8db4b0a"} {"msg":"[1]R generation[0..12]: [16, 16, 18, 15, 15, 14, 18, 19, 16, 18, 10, 19]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:55.506548115Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1262,"":"downstairs","":"downstairs","session_id":"122ce771-863a-4b62-b011-3267e8db4b0a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:55.506562254Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1262,"":"downstairs","":"downstairs","session_id":"122ce771-863a-4b62-b011-3267e8db4b0a"} {"msg":"[2]R flush_numbers[0..12]: [29, 29, 34, 28, 28, 27, 33, 35, 29, 33, 16, 35]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:55.506575853Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1262,"":"downstairs","":"downstairs","session_id":"122ce771-863a-4b62-b011-3267e8db4b0a"} {"msg":"[2]R generation[0..12]: [16, 16, 18, 15, 15, 14, 18, 19, 16, 18, 10, 19]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:55.506593241Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1262,"":"downstairs","":"downstairs","session_id":"122ce771-863a-4b62-b011-3267e8db4b0a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:55.50660764Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1262,"":"downstairs","":"downstairs","session_id":"122ce771-863a-4b62-b011-3267e8db4b0a"} {"msg":"Max found gen is 20","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:55.506620699Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1262,"":"downstairs","":"downstairs","session_id":"122ce771-863a-4b62-b011-3267e8db4b0a"} {"msg":"Generation requested: 20 >= found:20","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:55.506633729Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1262,"":"downstairs","":"downstairs","session_id":"122ce771-863a-4b62-b011-3267e8db4b0a"} {"msg":"Next flush: 36","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:55.506646658Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1262,"":"downstairs","":"downstairs","session_id":"122ce771-863a-4b62-b011-3267e8db4b0a"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-12-12T20:33:55.506659497Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1262,"":"downstairs","":"downstairs","session_id":"122ce771-863a-4b62-b011-3267e8db4b0a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:55.506675696Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1262,"session_id":"122ce771-863a-4b62-b011-3267e8db4b0a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:55.506687575Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1262,"client":"0","":"downstairs","session_id":"122ce771-863a-4b62-b011-3267e8db4b0a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:55.506701114Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1262,"client":"1","":"downstairs","session_id":"122ce771-863a-4b62-b011-3267e8db4b0a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:55.506714143Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1262,"client":"2","":"downstairs","session_id":"122ce771-863a-4b62-b011-3267e8db4b0a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:55.506727362Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1262,"session_id":"122ce771-863a-4b62-b011-3267e8db4b0a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:55.506739601Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1262,"session_id":"122ce771-863a-4b62-b011-3267e8db4b0a"} {"msg":"bbf79a83-7f38-4d66-8317-f50aebcc4570 is now active with session: 122ce771-863a-4b62-b011-3267e8db4b0a","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:55.50675232Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1262,"session_id":"122ce771-863a-4b62-b011-3267e8db4b0a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:55.50676418Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1262,"session_id":"122ce771-863a-4b62-b011-3267e8db4b0a"} {"msg":"The guest has finished waiting for activation with:20","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:55.506776139Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1262} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 15 len 20480 data: 4 3 3 4 3 03/30 Write block 32 len 40960 data: 2 2 2 3 3 4 5 6 5 5 04/30 Write block 559 len 40960 data: 5 3 3 3 3 4 3 3 3 3 05/30 Read block 285 len 8192 06/30 Write block 386 len 12288 data: 4 4 4 07/30 Write block 279 len 16384 data: 2 2 3 3 08/30 Write block 90 len 24576 data: 4 4 4 4 4 2 09/30 Read block 217 len 12288 10/30 Read block 346 len 12288 11/30 Write block 382 len 8192 data: 4 4 12/30 Flush 13/30 Write block 422 len 20480 data: 4 3 4 4 3 14/30 Read block 347 len 40960 15/30 Write block 357 len 12288 data: 2 2 2 16/30 Write block 385 len 12288 data: 4 5 5 17/30 Flush 18/30 Write block 93 len 12288 data: 5 5 3 19/30 Write block 311 len 32768 data: 5 5 5 6 7 7 6 5 20/30 Read block 581 len 32768 21/30 Write block 466 len 16384 data: 2 2 2 3 22/30 Read block 430 len 16384 23/30 Read block 283 len 4096 24/30 Read block 270 len 40960 25/30 Write block 540 len 28672 data: 3 2 2 2 2 2 2 26/30 Write block 31 len 36864 data: 3 3 3 3 4 4 5 6 7 27/30 Write block 544 len 4096 data: 3 28/30 Read block 161 len 8192 29/30 Read block 20 len 40960 30/30 Write block 15 len 4096 data: 5 ---------------------------------------------------------------- Crucible gen:20 GIO:true work queues: Upstairs:0 downstairs:13 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1017 Acked Write 3 Done Done Done false 1018 Acked Write 8 Done Done Done false 1019 Acked Read 8 Done Done Done false 1020 Acked Write 4 Done Done Done false 1021 Acked Read 4 Done Done Done false 1022 Acked Read 1 Done Done Done false 1023 Acked Read 10 Done Done Done false 1024 Acked Write 7 Done Done Done false 1025 Acked Write 9 Done Done Done false 1026 Acked Write 1 Done Done Done false 1027 Acked Read 2 Done Done Done false 1028 Acked Read 10 Done Done Done false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 12 12 12 36 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1016 1016 1016 Downstairs last five completed: 1016 1015 1014 1013 1012 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:33:55.524761948Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1262} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 20 20 20 37 37 37 T T T 1 020-039 20 20 20 37 37 37 T T T 2 040-059 20 20 20 37 37 37 F F F 3 060-079 15 15 15 28 28 28 F F F 4 080-099 20 20 20 37 37 37 T T T 5 100-119 14 14 14 27 27 27 F F F 6 120-139 18 18 18 33 33 33 F F F 7 140-159 19 19 19 35 35 35 F F F 8 160-179 16 16 16 29 29 29 F F F 9 180-199 18 18 18 33 33 33 F F F 10 200-219 10 10 10 16 16 16 F F F 11 220-239 19 19 19 35 35 35 F F F 12 240-259 19 19 19 35 35 35 F F F 13 260-279 20 20 20 37 37 37 F F F 14 280-299 20 20 20 37 37 37 F F F 15 300-319 19 19 19 35 35 35 T T T 16 320-339 16 16 16 29 29 29 F F F 17 340-359 20 20 20 38 38 38 F F F 18 360-379 17 17 17 31 31 31 F F F 19 380-399 20 20 20 38 38 38 F F F 20 400-419 16 16 16 30 30 30 F F F 21 420-439 20 20 20 38 38 38 F F F 22 440-459 16 16 16 29 29 29 F F F 23 460-479 18 18 18 33 33 33 T T T 24 480-499 12 12 12 20 20 20 F F F 25 500-519 17 17 17 31 31 31 F F F 26 520-539 13 13 13 21 21 21 F F F 27 540-559 20 20 20 37 37 37 T T T 28 560-579 20 20 20 37 37 37 F F F 29 580-599 15 15 15 28 28 28 F F F Max gen: 20, Max flush: 38 A Difference in extent metadata is expected here On loop 10 Verifying data now No change after verify Loop: 10 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 21 21 21 39 39 39 F F F 1 020-039 21 21 21 39 39 39 F F F 2 040-059 20 20 20 37 37 37 F F F 3 060-079 15 15 15 28 28 28 F F F 4 080-099 21 21 21 39 39 39 F F F 5 100-119 14 14 14 27 27 27 F F F 6 120-139 18 18 18 33 33 33 F F F 7 140-159 19 19 19 35 35 35 F F F 8 160-179 16 16 16 29 29 29 F F F 9 180-199 18 18 18 33 33 33 F F F 10 200-219 10 10 10 16 16 16 F F F 11 220-239 19 19 19 35 35 35 F F F 12 240-259 19 19 19 35 35 35 F F F 13 260-279 20 20 20 37 37 37 F F F 14 280-299 20 20 20 37 37 37 F F F 15 300-319 21 21 21 39 39 39 F F F 16 320-339 16 16 16 29 29 29 F F F 17 340-359 20 20 20 38 38 38 F F F 18 360-379 17 17 17 31 31 31 F F F 19 380-399 20 20 20 38 38 38 F F F 20 400-419 16 16 16 30 30 30 F F F 21 420-439 20 20 20 38 38 38 F F F 22 440-459 16 16 16 29 29 29 F F F 23 460-479 21 21 21 39 39 39 F F F 24 480-499 12 12 12 20 20 20 F F F 25 500-519 17 17 17 31 31 31 F F F 26 520-539 13 13 13 21 21 21 F F F 27 540-559 21 21 21 39 39 39 F F F 28 560-579 20 20 20 37 37 37 F F F 29 580-599 15 15 15 28 28 28 F F F Max gen: 21, Max flush: 39 Begin loop 11 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:59.801344722Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1270,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:33:59.845083386Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1270,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:59.845440761Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1270,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:59.877863635Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1270,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:59.878434356Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1270} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"26e7d501b33f47379e53601720353c2983e7654b\",\n git_commit_timestamp: \"2024-12-12T20:16:40.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:59.878469104Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1270} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:59.878482763Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1270} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:59.878501581Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1270} {"msg":"Crucible 0489becf-8926-4f6f-b518-dcbb6ace117e has session id: 237720cb-fff0-4469-a026-ee65c532cf3d","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:59.878920082Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1270,"session_id":"237720cb-fff0-4469-a026-ee65c532cf3d"} {"msg":"Upstairs opts: Upstairs UUID: 0489becf-8926-4f6f-b518-dcbb6ace117e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:59.878966949Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1270,"session_id":"237720cb-fff0-4469-a026-ee65c532cf3d"} {"msg":"Crucible stats registered with UUID: 0489becf-8926-4f6f-b518-dcbb6ace117e","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:59.878992388Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1270,"session_id":"237720cb-fff0-4469-a026-ee65c532cf3d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:59.90985194Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1270,"task":"crutest"} {"msg":"The guest has requested activation with gen:22","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:59.909885507Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1270} {"msg":"0489becf-8926-4f6f-b518-dcbb6ace117e active request set","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:59.909919055Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1270,"session_id":"237720cb-fff0-4469-a026-ee65c532cf3d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:59.909935604Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1270,"client":"0","":"downstairs","session_id":"237720cb-fff0-4469-a026-ee65c532cf3d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:59.909964262Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1270,"client":"0","":"downstairs","session_id":"237720cb-fff0-4469-a026-ee65c532cf3d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:59.909982011Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1270,"client":"1","":"downstairs","session_id":"237720cb-fff0-4469-a026-ee65c532cf3d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:59.909999519Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1270,"client":"1","":"downstairs","session_id":"237720cb-fff0-4469-a026-ee65c532cf3d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:59.910015848Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1270,"client":"2","":"downstairs","session_id":"237720cb-fff0-4469-a026-ee65c532cf3d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:59.910033207Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1270,"client":"2","":"downstairs","session_id":"237720cb-fff0-4469-a026-ee65c532cf3d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:59.910049456Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1270,"":"io task","client":"2","":"downstairs","session_id":"237720cb-fff0-4469-a026-ee65c532cf3d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:59.910070145Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1270,"":"io task","client":"0","":"downstairs","session_id":"237720cb-fff0-4469-a026-ee65c532cf3d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:59.910092183Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1270,"":"io task","client":"1","":"downstairs","session_id":"237720cb-fff0-4469-a026-ee65c532cf3d"} {"msg":"ds_connection connected from Ok(127.0.0.1:56546)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:59.910125341Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1270,"":"io task","client":"2","":"downstairs","session_id":"237720cb-fff0-4469-a026-ee65c532cf3d"} {"msg":"ds_connection connected from Ok(127.0.0.1:60744)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:59.910158778Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1270,"":"io task","client":"0","":"downstairs","session_id":"237720cb-fff0-4469-a026-ee65c532cf3d"} {"msg":"ds_connection connected from Ok(127.0.0.1:47031)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:59.910399332Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1270,"":"io task","client":"1","":"downstairs","session_id":"237720cb-fff0-4469-a026-ee65c532cf3d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:59.910884558Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1270,"client":"2","":"downstairs","session_id":"237720cb-fff0-4469-a026-ee65c532cf3d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:59.911088394Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1270,"client":"1","":"downstairs","session_id":"237720cb-fff0-4469-a026-ee65c532cf3d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:59.911132581Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1270,"client":"0","":"downstairs","session_id":"237720cb-fff0-4469-a026-ee65c532cf3d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:59.911606099Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1270,"client":"2","":"downstairs","session_id":"237720cb-fff0-4469-a026-ee65c532cf3d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:59.911645716Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1270,"client":"1","":"downstairs","session_id":"237720cb-fff0-4469-a026-ee65c532cf3d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:59.911820874Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1270,"client":"0","":"downstairs","session_id":"237720cb-fff0-4469-a026-ee65c532cf3d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:59.91216115Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1270,"session_id":"237720cb-fff0-4469-a026-ee65c532cf3d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:59.912235365Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1270,"session_id":"237720cb-fff0-4469-a026-ee65c532cf3d"} {"msg":"[0]R flush_numbers[0..12]: [39, 39, 37, 28, 39, 27, 33, 35, 29, 33, 16, 35]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:59.912352067Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1270,"":"downstairs","":"downstairs","session_id":"237720cb-fff0-4469-a026-ee65c532cf3d"} {"msg":"[0]R generation[0..12]: [21, 21, 20, 15, 21, 14, 18, 19, 16, 18, 10, 19]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:59.912372886Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1270,"":"downstairs","":"downstairs","session_id":"237720cb-fff0-4469-a026-ee65c532cf3d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:59.912390954Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1270,"":"downstairs","":"downstairs","session_id":"237720cb-fff0-4469-a026-ee65c532cf3d"} {"msg":"[1]R flush_numbers[0..12]: [39, 39, 37, 28, 39, 27, 33, 35, 29, 33, 16, 35]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:59.912405303Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1270,"":"downstairs","":"downstairs","session_id":"237720cb-fff0-4469-a026-ee65c532cf3d"} {"msg":"[1]R generation[0..12]: [21, 21, 20, 15, 21, 14, 18, 19, 16, 18, 10, 19]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:59.912419143Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1270,"":"downstairs","":"downstairs","session_id":"237720cb-fff0-4469-a026-ee65c532cf3d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:59.912433042Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1270,"":"downstairs","":"downstairs","session_id":"237720cb-fff0-4469-a026-ee65c532cf3d"} {"msg":"[2]R flush_numbers[0..12]: [39, 39, 37, 28, 39, 27, 33, 35, 29, 33, 16, 35]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:59.91244681Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1270,"":"downstairs","":"downstairs","session_id":"237720cb-fff0-4469-a026-ee65c532cf3d"} {"msg":"[2]R generation[0..12]: [21, 21, 20, 15, 21, 14, 18, 19, 16, 18, 10, 19]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:59.91246047Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1270,"":"downstairs","":"downstairs","session_id":"237720cb-fff0-4469-a026-ee65c532cf3d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:59.912474429Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1270,"":"downstairs","":"downstairs","session_id":"237720cb-fff0-4469-a026-ee65c532cf3d"} {"msg":"Max found gen is 22","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:59.912487428Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1270,"":"downstairs","":"downstairs","session_id":"237720cb-fff0-4469-a026-ee65c532cf3d"} {"msg":"Generation requested: 22 >= found:22","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:59.912500607Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1270,"":"downstairs","":"downstairs","session_id":"237720cb-fff0-4469-a026-ee65c532cf3d"} {"msg":"Next flush: 40","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:59.912513466Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1270,"":"downstairs","":"downstairs","session_id":"237720cb-fff0-4469-a026-ee65c532cf3d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:59.912526435Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1270,"":"downstairs","":"downstairs","session_id":"237720cb-fff0-4469-a026-ee65c532cf3d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:59.912539624Z","hostnameWait for a query_work_queue command to finish before sending IO ":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1270,"session_id":"237720cb-fff0-4469-a026-ee65c532cf3d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:59.912558383Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1270,"client":"0","":"downstairs","session_id":"237720cb-fff0-4469-a026-ee65c532cf3d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:59.912571602Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1270,"client":"1","":"downstairs","session_id":"237720cb-fff0-4469-a026-ee65c532cf3d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:59.912586381Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1270,"client":"2","":"downstairs","session_id":"237720cb-fff0-4469-a026-ee65c532cf3d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:59.91259956Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1270,"session_id":"237720cb-fff0-4469-a026-ee65c532cf3d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:59.912611809Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1270,"session_id":"237720cb-fff0-4469-a026-ee65c532cf3d"} {"msg":"0489becf-8926-4f6f-b518-dcbb6ace117e is now active with session: 237720cb-fff0-4469-a026-ee65c532cf3d","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:59.912624398Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1270,"session_id":"237720cb-fff0-4469-a026-ee65c532cf3d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:59.912636177Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1270,"session_id":"237720cb-fff0-4469-a026-ee65c532cf3d"} {"msg":"The guest has finished waiting for activation with:22","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:33:59.912648267Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1270} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 187 len 8192 data: 4 5 02/30 Read block 395 len 8192 03/30 Read block 453 len 24576 04/30 Read block 563 len 32768 05/30 Write block 501 len 40960 data: 2 2 2 2 2 2 4 4 4 4 06/30 Write block 55 len 24576 data: 4 4 4 6 5 4 07/30 Read block 527 len 40960 08/30 Read block 455 len 32768 09/30 Read block 106 len 32768 10/30 Read block 359 len 8192 11/30 Read block 336 len 28672 12/30 Write block 460 len 24576 data: 2 2 2 2 2 2 13/30 Read block 252 len 16384 14/30 Write block 285 len 40960 data: 3 3 2 2 2 2 2 2 4 4 15/30 Write block 42 len 40960 data: 4 4 3 4 3 3 4 4 4 3 16/30 Write block 253 len 28672 data: 4 5 5 5 4 5 5 17/30 Read block 351 len 16384 18/30 Read block 147 len 32768 19/30 Flush 20/30 Write block 357 len 4096 data: 3 21/30 Flush 22/30 Write block 87 len 8192 data: 4 4 23/30 Read block 504 len 16384 24/30 Read block 454 len 24576 25/30 Write block 506 len 40960 data: 3 5 5 5 5 5 4 4 4 3 26/30 Read block 353 len 28672 27/30 Read block 66 len 20480 28/30 Read block 77 len 16384 29/30 Read block 2 len 24576 30/30 Write block 354 len 16384 data: 4 4 3 4 ---------------------------------------------------------------- Crucible gen:22 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 2 Done Done Done false 1022 Acked Read 4 Done Done Done false 1023 Acked Read 6 Done Done Done false 1024 Acked Write 10 Done Done Done false 1025 Acked Read 7 Done Done Done false 1026 Acked Read 5 Done Done Done false 1027 Acked Read 4 Done Done Done false 1028 Acked Read 6 Done Done Done false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 8 8 8 24 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1020 1020 1020 Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:33:59.933247616Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1270} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 21 21 21 39 39 39 F F F 1 020-039 21 21 21 39 39 39 F F F 2 040-059 22 22 22 40 40 40 F F F 3 060-079 22 22 22 40 40 40 F F F 4 080-099 21 21 21 39 39 39 T T T 5 100-119 14 14 14 27 27 27 F F F 6 120-139 18 18 18 33 33 33 F F F 7 140-159 19 19 19 35 35 35 F F F 8 160-179 16 16 16 29 29 29 F F F 9 180-199 22 22 22 40 40 40 F F F 10 200-219 10 10 10 16 16 16 F F F 11 220-239 19 19 19 35 35 35 F F F 12 240-259 22 22 22 40 40 40 F F F 13 260-279 20 20 20 37 37 37 F F F 14 280-299 22 22 22 40 40 40 F F F 15 300-319 21 21 21 39 39 39 F F F 16 320-339 16 16 16 29 29 29 F F F 17 340-359 22 22 22 41 41 41 T T T 18 360-379 17 17 17 31 31 31 F F F 19 380-399 20 20 20 38 38 38 F F F 20 400-419 16 16 16 30 30 30 F F F 21 420-439 20 20 20 38 38 38 F F F 22 440-459 16 16 16 29 29 29 F F F 23 460-479 22 22 22 40 40 40 F F F 24 480-499 12 12 12 20 20 20 F F F 25 500-519 22 22 22 40 40 40 T T T 26 520-539 13 13 13 21 21 21 F F F 27 540-559 21 21 21 39 39 39 F F F 28 560-579 20 20 20 37 37 37 F F F 29 580-599 15 15 15 28 28 28 F F F Max gen: 22, Max flush: 41 A Difference in extent metadata is expected here On loop 11 Verifying data now No change after verify Loop: 11 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 21 21 21 39 39 39 F F F 1 020-039 21 21 21 39 39 39 F F F 2 040-059 22 22 22 40 40 40 F F F 3 060-079 22 22 22 40 40 40 F F F 4 080-099 23 23 23 42 42 42 F F F 5 100-119 14 14 14 27 27 27 F F F 6 120-139 18 18 18 33 33 33 F F F 7 140-159 19 19 19 35 35 35 F F F 8 160-179 16 16 16 29 29 29 F F F 9 180-199 22 22 22 40 40 40 F F F 10 200-219 10 10 10 16 16 16 F F F 11 220-239 19 19 19 35 35 35 F F F 12 240-259 22 22 22 40 40 40 F F F 13 260-279 20 20 20 37 37 37 F F F 14 280-299 22 22 22 40 40 40 F F F 15 300-319 21 21 21 39 39 39 F F F 16 320-339 16 16 16 29 29 29 F F F 17 340-359 23 23 23 42 42 42 F F F 18 360-379 17 17 17 31 31 31 F F F 19 380-399 20 20 20 38 38 38 F F F 20 400-419 16 16 16 30 30 30 F F F 21 420-439 20 20 20 38 38 38 F F F 22 440-459 16 16 16 29 29 29 F F F 23 460-479 22 22 22 40 40 40 F F F 24 480-499 12 12 12 20 20 20 F F F 25 500-519 23 23 23 42 42 42 F F F 26 520-539 13 13 13 21 21 21 F F F 27 540-559 21 21 21 39 39 39 F F F 28 560-579 20 20 20 37 37 37 F F F 29 580-599 15 15 15 28 28 28 F F F Max gen: 23, Max flush: 42 Begin loop 12 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:04.228340094Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1278,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:04.272422964Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1278,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:04.273109127Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1278} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"26e7d501b33f47379e53601720353c2983e7654b\",\n git_commit_timestamp: \"2024-12-12T20:16:40.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:04.273160013Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1278} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:04.273182171Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1278} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:04.27320674Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1278} {"msg":"Crucible dee0478e-885e-4879-899b-b8f62cfcb229 has session id: f599586b-8294-45a3-981c-623b07900f8e","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:04.27363422Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1278,"session_id":"f599586b-8294-45a3-981c-623b07900f8e"} {"msg":"Upstairs opts: Upstairs UUID: dee0478e-885e-4879-899b-b8f62cfcb229, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:04.273672988Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1278,"session_id":"f599586b-8294-45a3-981c-623b07900f8e"} {"msg":"Crucible stats registered with UUID: dee0478e-885e-4879-899b-b8f62cfcb229","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:04.273689796Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1278,"session_id":"f599586b-8294-45a3-981c-623b07900f8e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:04.304508571Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1278,"task":"crutest"} {"msg":"The guest has requested activation with gen:24","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:04.304565427Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1278} {"msg":"dee0478e-885e-4879-899b-b8f62cfcb229 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:04.304588006Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1278,"session_id":"f599586b-8294-45a3-981c-623b07900f8e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:04.304601675Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1278,"client":"0","":"downstairs","session_id":"f599586b-8294-45a3-981c-623b07900f8e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:04.304623533Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1278,"client":"0","":"downstairs","session_id":"f599586b-8294-45a3-981c-623b07900f8e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:04.304638032Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1278,"client":"1","":"downstairs","session_id":"f599586b-8294-45a3-981c-623b07900f8e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:04.304652181Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1278,"client":"1","":"downstairs","session_id":"f599586b-8294-45a3-981c-623b07900f8e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:04.30466904Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1278,"client":"2","":"downstairs","session_id":"f599586b-8294-45a3-981c-623b07900f8e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:04.304683479Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1278,"client":"2","":"downstairs","session_id":"f599586b-8294-45a3-981c-623b07900f8e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:04.304696748Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1278,"":"io task","client":"2","":"downstairs","session_id":"f599586b-8294-45a3-981c-623b07900f8e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:04.304712217Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1278,"":"io task","client":"0","":"downstairs","session_id":"f599586b-8294-45a3-981c-623b07900f8e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:04.304726026Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1278,"":"io task","client":"1","":"downstairs","session_id":"f599586b-8294-45a3-981c-623b07900f8e"} {"msg":"ds_connection connected from Ok(127.0.0.1:34297)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:04.304891345Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1278,"":"io task","client":"0","":"downstairs","session_id":"f599586b-8294-45a3-981c-623b07900f8e"} {"msg":"ds_connection connected from Ok(127.0.0.1:63322)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:04.305072262Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1278,"":"io task","client":"2","":"downstairs","session_id":"f599586b-8294-45a3-981c-623b07900f8e"} {"msg":"ds_connection connected from Ok(127.0.0.1:33369)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:04.30510468Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1278,"":"io task","client":"1","":"downstairs","session_id":"f599586b-8294-45a3-981c-623b07900f8e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:04.305670911Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1278,"client":"2","":"downstairs","session_id":"f599586b-8294-45a3-981c-623b07900f8e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:04.305814171Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1278,"client":"1","":"downstairs","session_id":"f599586b-8294-45a3-981c-623b07900f8e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:04.305843559Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1278,"client":"0","":"downstairs","session_id":"f599586b-8294-45a3-981c-623b07900f8e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:04.306382802Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1278,"client":"2","":"downstairs","session_id":"f599586b-8294-45a3-981c-623b07900f8e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:04.30641321Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1278,"client":"0","":"downstairs","session_id":"f599586b-8294-45a3-981c-623b07900f8e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:04.306653653Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1278,"client":"1","":"downstairs","session_id":"f599586b-8294-45a3-981c-623b07900f8e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:04.306922905Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1278,"session_id":"f599586b-8294-45a3-981c-623b07900f8e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:04.306984371Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1278,"session_id":"f599586b-8294-45a3-981c-623b07900f8e"} {"msg":"[0]R flush_numbers[0..12]: [39, 39, 40, 40, 42, 27, 33, 35, 29, 40, 16, 35]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:04.307202485Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1278,"":"downstairs","":"downstairs","session_id":"f599586b-8294-45a3-981c-623b07900f8e"} {"msg":"[0]R generation[0..12]: [21, 21, 22, 22, 23, 14, 18, 19, 16, 22, 10, 19]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:04.307223094Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1278,"":"downstairs","":"downstairs","session_id":"f599586b-8294-45a3-981c-623b07900f8e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:04.307238243Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1278,"":"downstairs","":"downstairs","session_id":"f599586b-8294-45a3-981c-623b07900f8e"} {"msg":"[1]R flush_numbers[0..12]: [39, 39, 40, 40, 42, 27, 33, 35, 29, 40, 16, 35]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:04.307252372Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1278,"":"downstairs","":"downstairs","session_id":"f599586b-8294-45a3-981c-623b07900f8e"} {"msg":"[1]R generation[0..12]: [21, 21, 22, 22, 23, 14, 18, 19, 16, 22, 10, 19]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:04.307266271Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1278,"":"downstairs","":"downstairs","session_id":"f599586b-8294-45a3-981c-623b07900f8e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:04.30728042Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1278,"":"downstairs","":"downstairs","session_id":"f599586b-8294-45a3-981c-623b07900f8e"} {"msg":"[2]R flush_numbers[0..12]: [39, 39, 40, 40, 42, 27, 33, 35, 29, 40, 16, 35]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:04.307294219Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1278,"":"downstairs","":"downstairs","session_id":"f599586b-8294-45a3-981c-623b07900f8e"} {"msg":"[2]R generation[0..12]: [21, 21, 22, 22, 23, 14, 18, 19, 16, 22, 10, 19]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:04.307307858Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1278,"":"downstairs","":"downstairs","session_id":"f599586b-8294-45a3-981c-623b07900f8e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:04.307321727Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1278,"":"downstairs","":"downstairs","session_id":"f599586b-8294-45a3-981c-623b07900f8e"} {"msg":"Max found gen is 24","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:04.307334836Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1278,"":"downstairs","":"downstairs","session_id":"f599586b-8294-45a3-981c-623b07900f8e"} {"msg":"Generation requested: 24 >= found:24","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:04.307347955Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1278,"":"downstairs","":"downstairs","session_id":"f599586b-8294-45a3-981c-623b07900f8e"} {"msg":"Next flush: 43","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:04.307361934Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1278,"":"downstairs","":"downstairs","session_id":"f599586b-8294-45a3-981c-623b07900f8e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time"Wait for a query_work_queue command to finish before sending IO :"2024-12-12T20:34:04.307378393Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1278,"":"downstairs","":"downstairs","session_id":"f599586b-8294-45a3-981c-623b07900f8e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:04.307397532Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1278,"session_id":"f599586b-8294-45a3-981c-623b07900f8e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:04.307409601Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1278,"client":"0","":"downstairs","session_id":"f599586b-8294-45a3-981c-623b07900f8e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:04.30742292Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1278,"client":"1","":"downstairs","session_id":"f599586b-8294-45a3-981c-623b07900f8e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:04.307436099Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1278,"client":"2","":"downstairs","session_id":"f599586b-8294-45a3-981c-623b07900f8e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:04.307449268Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1278,"session_id":"f599586b-8294-45a3-981c-623b07900f8e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:04.307461498Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1278,"session_id":"f599586b-8294-45a3-981c-623b07900f8e"} {"msg":"dee0478e-885e-4879-899b-b8f62cfcb229 is now active with session: f599586b-8294-45a3-981c-623b07900f8e","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:04.307474187Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1278,"session_id":"f599586b-8294-45a3-981c-623b07900f8e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:04.307486276Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1278,"session_id":"f599586b-8294-45a3-981c-623b07900f8e"} {"msg":"The guest has finished waiting for activation with:24","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:04.307498465Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1278} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 0 len 28672 data: 2 2 3 3 3 2 2 02/30 Write block 551 len 20480 data: 5 5 6 7 7 03/30 Read block 454 len 4096 04/30 Read block 526 len 32768 05/30 Read block 78 len 28672 06/30 Write block 111 len 12288 data: 4 4 4 07/30 Write block 542 len 28672 data: 3 3 4 3 3 2 2 08/30 Write block 93 len 20480 data: 6 6 4 2 2 09/30 Read block 218 len 24576 10/30 Write block 177 len 4096 data: 2 11/30 Write block 312 len 28672 data: 6 6 7 8 8 7 6 12/30 Read block 498 len 32768 13/30 Read block 435 len 8192 14/30 Write block 373 len 40960 data: 4 4 3 3 2 3 4 4 4 5 15/30 Write block 178 len 36864 data: 3 4 3 3 3 3 5 5 5 16/30 Write block 67 len 8192 data: 3 3 17/30 Read block 128 len 24576 18/30 Read block 478 len 24576 19/30 Write block 84 len 24576 data: 2 2 3 5 5 4 20/30 Flush 21/30 Write block 337 len 36864 data: 3 4 4 2 2 2 2 3 3 22/30 Flush 23/30 Write block 438 len 40960 data: 3 4 4 3 3 3 3 3 4 4 24/30 Write block 253 len 16384 data: 5 6 6 6 25/30 Read block 224 len 4096 26/30 Read block 73 len 8192 27/30 Read block 214 len 16384 28/30 Read block 143 len 36864 29/30 Write block 27 len 36864 data: 3 3 3 3 4 4 4 4 5 30/30 Read block 518 len 8192 ---------------------------------------------------------------- Crucible gen:24 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 10 Done Done Done false 1023 Acked Write 4 Done Done Done false 1024 Acked Read 1 Done Done Done false 1025 Acked Read 2 Done Done Done false 1026 Acked Read 4 Done Done Done false 1027 Acked Read 9 Done Done Done false 1028 Acked Write 9 Sent Sent Done false 1029 Acked Read 2 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 0 4 Done 6 6 8 20 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1021 1021 1021 Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:34:04.328321819Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1278} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 24 24 24 43 43 43 F F F 1 020-039 21 21 21 39 39 39 T T T 2 040-059 22 22 22 40 40 40 F F F 3 060-079 24 24 24 43 43 43 F F F 4 080-099 24 24 24 43 43 43 F F F 5 100-119 24 24 24 43 43 43 F F F 6 120-139 18 18 18 33 33 33 F F F 7 140-159 19 19 19 35 35 35 F F F 8 160-179 24 24 24 43 43 43 F F F 9 180-199 24 24 24 43 43 43 F F F 10 200-219 10 10 10 16 16 16 F F F 11 220-239 19 19 19 35 35 35 F F F 12 240-259 22 22 22 40 40 40 T T T 13 260-279 20 20 20 37 37 37 F F F 14 280-299 22 22 22 40 40 40 F F F 15 300-319 24 24 24 43 43 43 F F F 16 320-339 24 24 24 44 44 44 F F F 17 340-359 24 24 24 44 44 44 F F F 18 360-379 24 24 24 43 43 43 F F F 19 380-399 24 24 24 43 43 43 F F F 20 400-419 16 16 16 30 30 30 F F F 21 420-439 20 20 20 38 38 38 T T T 22 440-459 16 16 16 29 29 29 T T T 23 460-479 22 22 22 40 40 40 F F F 24 480-499 12 12 12 20 20 20 F F F 25 500-519 23 23 23 42 42 42 F F F 26 520-539 13 13 13 21 21 21 F F F 27 540-559 24 24 24 43 43 43 F F F 28 560-579 20 20 20 37 37 37 F F F 29 580-599 15 15 15 28 28 28 F F F Max gen: 24, Max flush: 44 A Difference in extent metadata is expected here On loop 12 Verifying data now No change after verify Loop: 12 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 24 24 24 43 43 43 F F F 1 020-039 25 25 25 45 45 45 F F F 2 040-059 22 22 22 40 40 40 F F F 3 060-079 24 24 24 43 43 43 F F F 4 080-099 24 24 24 43 43 43 F F F 5 100-119 24 24 24 43 43 43 F F F 6 120-139 18 18 18 33 33 33 F F F 7 140-159 19 19 19 35 35 35 F F F 8 160-179 24 24 24 43 43 43 F F F 9 180-199 24 24 24 43 43 43 F F F 10 200-219 10 10 10 16 16 16 F F F 11 220-239 19 19 19 35 35 35 F F F 12 240-259 25 25 25 45 45 45 F F F 13 260-279 20 20 20 37 37 37 F F F 14 280-299 22 22 22 40 40 40 F F F 15 300-319 24 24 24 43 43 43 F F F 16 320-339 24 24 24 44 44 44 F F F 17 340-359 24 24 24 44 44 44 F F F 18 360-379 24 24 24 43 43 43 F F F 19 380-399 24 24 24 43 43 43 F F F 20 400-419 16 16 16 30 30 30 F F F 21 420-439 25 25 25 45 45 45 F F F 22 440-459 25 25 25 45 45 45 F F F 23 460-479 22 22 22 40 40 40 F F F 24 480-499 12 12 12 20 20 20 F F F 25 500-519 23 23 23 42 42 42 F F F 26 520-539 13 13 13 21 21 21 F F F 27 540-559 24 24 24 43 43 43 F F F 28 560-579 20 20 20 37 37 37 F F F 29 580-599 15 15 15 28 28 28 F F F Max gen: 25, Max flush: 45 Begin loop 13 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:08.593421124Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1286,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:34:08.637843681Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1286,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:08.638257792Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1286,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:08.670816467Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1286,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:08.671391408Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1286} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"26e7d501b33f47379e53601720353c2983e7654b\",\n git_commit_timestamp: \"2024-12-12T20:16:40.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:08.671427195Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1286} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:08.671441224Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1286} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:08.671455473Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1286} {"msg":"Crucible 4074208e-a177-4286-9daf-9862ad81a95e has session id: 21edc154-5657-4be6-93b9-0c06c2ebe796","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:08.671830697Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1286,"session_id":"21edc154-5657-4be6-93b9-0c06c2ebe796"} {"msg":"Upstairs opts: Upstairs UUID: 4074208e-a177-4286-9daf-9862ad81a95e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:08.671871785Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1286,"session_id":"21edc154-5657-4be6-93b9-0c06c2ebe796"} {"msg":"Crucible stats registered with UUID: 4074208e-a177-4286-9daf-9862ad81a95e","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:08.671888353Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1286,"session_id":"21edc154-5657-4be6-93b9-0c06c2ebe796"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:08.702605985Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1286,"task":"crutest"} {"msg":"The guest has requested activation with gen:26","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:08.702636503Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1286} {"msg":"4074208e-a177-4286-9daf-9862ad81a95e active request set","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:08.702702479Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1286,"session_id":"21edc154-5657-4be6-93b9-0c06c2ebe796"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:08.702737926Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1286,"client":"0","":"downstairs","session_id":"21edc154-5657-4be6-93b9-0c06c2ebe796"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:08.702767614Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1286,"client":"0","":"downstairs","session_id":"21edc154-5657-4be6-93b9-0c06c2ebe796"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:08.702789213Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1286,"client":"1","":"downstairs","session_id":"21edc154-5657-4be6-93b9-0c06c2ebe796"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:08.70282225Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1286,"client":"1","":"downstairs","session_id":"21edc154-5657-4be6-93b9-0c06c2ebe796"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:08.702854408Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1286,"client":"2","":"downstairs","session_id":"21edc154-5657-4be6-93b9-0c06c2ebe796"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:08.702890086Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1286,"client":"2","":"downstairs","session_id":"21edc154-5657-4be6-93b9-0c06c2ebe796"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:08.702923644Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1286,"":"io task","client":"2","":"downstairs","session_id":"21edc154-5657-4be6-93b9-0c06c2ebe796"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:08.702950951Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1286,"":"io task","client":"0","":"downstairs","session_id":"21edc154-5657-4be6-93b9-0c06c2ebe796"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:08.70297252Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1286,"":"io task","client":"1","":"downstairs","session_id":"21edc154-5657-4be6-93b9-0c06c2ebe796"} {"msg":"ds_connection connected from Ok(127.0.0.1:48248)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:08.703064164Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1286,"":"io task","client":"1","":"downstairs","session_id":"21edc154-5657-4be6-93b9-0c06c2ebe796"} {"msg":"ds_connection connected from Ok(127.0.0.1:53693)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:08.703105611Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1286,"":"io task","client":"0","":"downstairs","session_id":"21edc154-5657-4be6-93b9-0c06c2ebe796"} {"msg":"ds_connection connected from Ok(127.0.0.1:35465)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:08.703167987Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1286,"":"io task","client":"2","":"downstairs","session_id":"21edc154-5657-4be6-93b9-0c06c2ebe796"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:08.703567079Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1286,"client":"1","":"downstairs","session_id":"21edc154-5657-4be6-93b9-0c06c2ebe796"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:08.70369659Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1286,"client":"2","":"downstairs","session_id":"21edc154-5657-4be6-93b9-0c06c2ebe796"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:08.703933764Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1286,"client":"0","":"downstairs","session_id":"21edc154-5657-4be6-93b9-0c06c2ebe796"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:08.704110671Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1286,"client":"1","":"downstairs","session_id":"21edc154-5657-4be6-93b9-0c06c2ebe796"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:08.704265781Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1286,"client":"2","":"downstairs","session_id":"21edc154-5657-4be6-93b9-0c06c2ebe796"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:08.704580379Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1286,"client":"0","":"downstairs","session_id":"21edc154-5657-4be6-93b9-0c06c2ebe796"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:08.704659694Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1286,"session_id":"21edc154-5657-4be6-93b9-0c06c2ebe796"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:08.704747278Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1286,"session_id":"21edc154-5657-4be6-93b9-0c06c2ebe796"} {"msg":"[0]R flush_numbers[0..12]: [43, 45, 40, 43, 43, 43, 33, 35, 43, 43, 16, 35]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:08.705012659Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1286,"":"downstairs","":"downstairs","session_id":"21edc154-5657-4be6-93b9-0c06c2ebe796"} {"msg":"[0]R generation[0..12]: [24, 25, 22, 24, 24, 24, 18, 19, 24, 24, 10, 19]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:08.705032998Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1286,"":"downstairs","":"downstairs","session_id":"21edc154-5657-4be6-93b9-0c06c2ebe796"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:08.705047687Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1286,"":"downstairs","":"downstairs","session_id":"21edc154-5657-4be6-93b9-0c06c2ebe796"} {"msg":"[1]R flush_numbers[0..12]: [43, 45, 40, 43, 43, 43, 33, 35, 43, 43, 16, 35]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:08.705061206Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1286,"":"downstairs","":"downstairs","session_id":"21edc154-5657-4be6-93b9-0c06c2ebe796"} {"msg":"[1]R generation[0..12]: [24, 25, 22, 24, 24, 24, 18, 19, 24, 24, 10, 19]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:08.705074805Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1286,"":"downstairs","":"downstairs","session_id":"21edc154-5657-4be6-93b9-0c06c2ebe796"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:08.705088734Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1286,"":"downstairs","":"downstairs","session_id":"21edc154-5657-4be6-93b9-0c06c2ebe796"} {"msg":"[2]R flush_numbers[0..12]: [43, 45, 40, 43, 43, 43, 33, 35, 43, 43, 16, 35]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:08.705102573Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1286,"":"downstairs","":"downstairs","session_id":"21edc154-5657-4be6-93b9-0c06c2ebe796"} {"msg":"[2]R generation[0..12]: [24, 25, 22, 24, 24, 24, 18, 19, 24, 24, 10, 19]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:08.705119382Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1286,"":"downstairs","":"downstairs","session_id":"21edc154-5657-4be6-93b9-0c06c2ebe796"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:08.705133741Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1286,"":"downstairs","":"downstairs","session_id":"21edc154-5657-4be6-93b9-0c06c2ebe796"} {"msg":"Max found gen is 26","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:08.70514716Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1286,"":"downstairs","":"downstairs","session_id":"21edc154-5657-4be6-93b9-0c06c2ebe796"} {"msg":"Generation requested: 26 >= found:26","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:08.705160429Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1286,"":"downstairs","":"downstairs","session_id":"21edc154-5657-4be6-93b9-0c06c2ebe796"} {"msg":"Next flush: 46","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:08.705173428Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1286,"":"downstairs","":"downstairs","session_id":"21edc154-5657-4be6-93b9-0c06c2ebe796"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-12-12T20:34:08.705186407Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1286,"":"downstairs","":"downstairs","session_id":"21edc154-5657-4be6-93b9-0c06c2ebe796"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:08.705203776Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1286,"session_id":"21edc154-5657-4be6-93b9-0c06c2ebe796"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:08.705215975Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1286,"client":"0","":"downstairs","session_id":"21edc154-5657-4be6-93b9-0c06c2ebe796"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:08.705230644Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1286,"client":"1","":"downstairs","session_id":"21edc154-5657-4be6-93b9-0c06c2ebe796"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:08.705244223Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1286,"client":"2","":"downstairs","session_id":"21edc154-5657-4be6-93b9-0c06c2ebe796"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:08.705257652Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1286,"session_id":"21edc154-5657-4be6-93b9-0c06c2ebe796"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:08.705269972Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1286,"session_id":"21edc154-5657-4be6-93b9-0c06c2ebe796"} {"msg":"4074208e-a177-4286-9daf-9862ad81a95e is now active with session: 21edc154-5657-4be6-93b9-0c06c2ebe796","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:08.705282751Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1286,"session_id":"21edc154-5657-4be6-93b9-0c06c2ebe796"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:08.70529459Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1286,"session_id":"21edc154-5657-4be6-93b9-0c06c2ebe796"} {"msg":"The guest has finished waiting for activation with:26","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2024-12-12T20:34:08.705306719Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1286} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 502 len 4096 data: 3 02/30 Write block 543 len 32768 data: 4 5 4 4 3 3 3 3 03/30 Read block 131 len 12288 04/30 Read block 287 len 28672 05/30 Write block 384 len 4096 data: 3 06/30 Read block 305 len 12288 07/30 Read block 353 len 32768 08/30 Read block 195 len 16384 09/30 Read block 39 len 12288 10/30 Read block 347 len 40960 11/30 Write block 533 len 32768 data: 2 2 2 2 2 2 2 4 12/30 Read block 145 len 24576 13/30 Write block 446 len 40960 data: 5 5 2 2 2 2 3 3 3 3 14/30 Read block 411 len 24576 15/30 Read block 191 len 24576 16/30 Read block 536 len 28672 17/30 Write block 136 len 20480 data: 2 2 2 2 3 18/30 Flush 19/30 Write block 49 len 36864 data: 5 5 4 3 4 4 5 5 5 20/30 Read block 526 len 28672 21/30 Read block 481 len 28672 22/30 Read block 221 len 28672 23/30 Read block 450 len 28672 24/30 Read block 461 len 24576 25/30 Flush 26/30 Write block 68 len 24576 data: 4 2 2 2 3 3 27/30 Read block 479 len 36864 28/30 Read block 181 len 24576 29/30 Read block 59 len 24576 30/30 Read block 87 len 20480 ---------------------------------------------------------------- Crucible gen:26 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 6 Done Done Done false 1026 Acked Read 9 Done Done Done false 1027 Acked Read 6 Done Done Done false 1028 Acked Read 6 Done Done Done false 1029 Acked Read 5 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:34:08.726522666Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1286} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 24 24 24 43 43 43 F F F 1 020-039 25 25 25 45 45 45 F F F 2 040-059 26 26 26 47 47 47 F F F 3 060-079 24 24 24 43 43 43 T T T 4 080-099 24 24 24 43 43 43 F F F 5 100-119 24 24 24 43 43 43 F F F 6 120-139 26 26 26 46 46 46 F F F 7 140-159 26 26 26 46 46 46 F F F 8 160-179 24 24 24 43 43 43 F F F 9 180-199 24 24 24 43 43 43 F F F 10 200-219 10 10 10 16 16 16 F F F 11 220-239 19 19 19 35 35 35 F F F 12 240-259 25 25 25 45 45 45 F F F 13 260-279 20 20 20 37 37 37 F F F 14 280-299 22 22 22 40 40 40 F F F 15 300-319 24 24 24 43 43 43 F F F 16 320-339 24 24 24 44 44 44 F F F 17 340-359 24 24 24 44 44 44 F F F 18 360-379 24 24 24 43 43 43 F F F 19 380-399 26 26 26 46 46 46 F F F 20 400-419 16 16 16 30 30 30 F F F 21 420-439 25 25 25 45 45 45 F F F 22 440-459 26 26 26 46 46 46 F F F 23 460-479 22 22 22 40 40 40 F F F 24 480-499 12 12 12 20 20 20 F F F 25 500-519 26 26 26 46 46 46 F F F 26 520-539 26 26 26 46 46 46 F F F 27 540-559 26 26 26 46 46 46 F F F 28 560-579 20 20 20 37 37 37 F F F 29 580-599 15 15 15 28 28 28 F F F Max gen: 26, Max flush: 47 A Difference in extent metadata is expected here On loop 13 Verifying data now No change after verify Loop: 13 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 24 24 24 43 43 43 F F F 1 020-039 25 25 25 45 45 45 F F F 2 040-059 26 26 26 47 47 47 F F F 3 060-079 27 27 27 48 48 48 F F F 4 080-099 24 24 24 43 43 43 F F F 5 100-119 24 24 24 43 43 43 F F F 6 120-139 26 26 26 46 46 46 F F F 7 140-159 26 26 26 46 46 46 F F F 8 160-179 24 24 24 43 43 43 F F F 9 180-199 24 24 24 43 43 43 F F F 10 200-219 10 10 10 16 16 16 F F F 11 220-239 19 19 19 35 35 35 F F F 12 240-259 25 25 25 45 45 45 F F F 13 260-279 20 20 20 37 37 37 F F F 14 280-299 22 22 22 40 40 40 F F F 15 300-319 24 24 24 43 43 43 F F F 16 320-339 24 24 24 44 44 44 F F F 17 340-359 24 24 24 44 44 44 F F F 18 360-379 24 24 24 43 43 43 F F F 19 380-399 26 26 26 46 46 46 F F F 20 400-419 16 16 16 30 30 30 F F F 21 420-439 25 25 25 45 45 45 F F F 22 440-459 26 26 26 46 46 46 F F F 23 460-479 22 22 22 40 40 40 F F F 24 480-499 12 12 12 20 20 20 F F F 25 500-519 26 26 26 46 46 46 F F F 26 520-539 26 26 26 46 46 46 F F F 27 540-559 26 26 26 46 46 46 F F F 28 560-579 20 20 20 37 37 37 F F F 29 580-599 15 15 15 28 28 28 F F F Max gen: 27, Max flush: 48 Begin loop 14 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:13.004148738Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1294,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:34:13.047701655Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1294,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:13.04806078Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1294,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:13.080412229Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1294,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:13.081007278Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1294} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"26e7d501b33f47379e53601720353c2983e7654b\",\n git_commit_timestamp: \"2024-12-12T20:16:40.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:13.081042636Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1294} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:13.081056675Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1294} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:13.081070994Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1294} {"msg":"Crucible 8e897076-76b1-4a62-8a36-44b624e9f2c7 has session id: 3d85ce56-1f78-498f-bdad-b28e370608f7","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:13.081518483Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1294,"session_id":"3d85ce56-1f78-498f-bdad-b28e370608f7"} {"msg":"Upstairs opts: Upstairs UUID: 8e897076-76b1-4a62-8a36-44b624e9f2c7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:13.081552171Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1294,"session_id":"3d85ce56-1f78-498f-bdad-b28e370608f7"} {"msg":"Crucible stats registered with UUID: 8e897076-76b1-4a62-8a36-44b624e9f2c7","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:13.081566899Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1294,"session_id":"3d85ce56-1f78-498f-bdad-b28e370608f7"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:13.11229704Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1294,"task":"crutest"} {"msg":"The guest has requested activation with gen:28","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:13.112326569Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1294} {"msg":"8e897076-76b1-4a62-8a36-44b624e9f2c7 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:13.112400513Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1294,"session_id":"3d85ce56-1f78-498f-bdad-b28e370608f7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:13.112421642Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1294,"client":"0","":"downstairs","session_id":"3d85ce56-1f78-498f-bdad-b28e370608f7"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:13.11245036Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1294,"client":"0","":"downstairs","session_id":"3d85ce56-1f78-498f-bdad-b28e370608f7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:13.112475598Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1294,"client":"1","":"downstairs","session_id":"3d85ce56-1f78-498f-bdad-b28e370608f7"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:13.112509906Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1294,"client":"1","":"downstairs","session_id":"3d85ce56-1f78-498f-bdad-b28e370608f7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:13.112540134Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1294,"client":"2","":"downstairs","session_id":"3d85ce56-1f78-498f-bdad-b28e370608f7"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:13.112564122Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1294,"client":"2","":"downstairs","session_id":"3d85ce56-1f78-498f-bdad-b28e370608f7"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:13.11259452Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1294,"":"io task","client":"2","":"downstairs","session_id":"3d85ce56-1f78-498f-bdad-b28e370608f7"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:13.112630697Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1294,"":"io task","client":"0","":"downstairs","session_id":"3d85ce56-1f78-498f-bdad-b28e370608f7"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:13.112665415Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1294,"":"io task","client":"1","":"downstairs","session_id":"3d85ce56-1f78-498f-bdad-b28e370608f7"} {"msg":"ds_connection connected from Ok(127.0.0.1:51160)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:13.112839443Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1294,"":"io task","client":"1","":"downstairs","session_id":"3d85ce56-1f78-498f-bdad-b28e370608f7"} {"msg":"ds_connection connected from Ok(127.0.0.1:45360)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:13.11287768Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1294,"":"io task","client":"0","":"downstairs","session_id":"3d85ce56-1f78-498f-bdad-b28e370608f7"} {"msg":"ds_connection connected from Ok(127.0.0.1:35816)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:13.112903859Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1294,"":"io task","client":"2","":"downstairs","session_id":"3d85ce56-1f78-498f-bdad-b28e370608f7"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:13.113556214Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1294,"client":"1","":"downstairs","session_id":"3d85ce56-1f78-498f-bdad-b28e370608f7"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:13.11360534Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1294,"client":"2","":"downstairs","session_id":"3d85ce56-1f78-498f-bdad-b28e370608f7"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:13.113631548Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1294,"client":"0","":"downstairs","session_id":"3d85ce56-1f78-498f-bdad-b28e370608f7"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:13.114137124Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1294,"client":"1","":"downstairs","session_id":"3d85ce56-1f78-498f-bdad-b28e370608f7"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:13.114243416Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1294,"client":"2","":"downstairs","session_id":"3d85ce56-1f78-498f-bdad-b28e370608f7"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:13.114307352Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1294,"client":"0","":"downstairs","session_id":"3d85ce56-1f78-498f-bdad-b28e370608f7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:13.114721583Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1294,"session_id":"3d85ce56-1f78-498f-bdad-b28e370608f7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:13.114804528Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1294,"session_id":"3d85ce56-1f78-498f-bdad-b28e370608f7"} {"msg":"[0]R flush_numbers[0..12]: [43, 45, 47, 48, 43, 43, 46, 46, 43, 43, 16, 35]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:13.114922839Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1294,"":"downstairs","":"downstairs","session_id":"3d85ce56-1f78-498f-bdad-b28e370608f7"} {"msg":"[0]R generation[0..12]: [24, 25, 26, 27, 24, 24, 26, 26, 24, 24, 10, 19]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:13.114942148Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1294,"":"downstairs","":"downstairs","session_id":"3d85ce56-1f78-498f-bdad-b28e370608f7"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:13.114956977Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1294,"":"downstairs","":"downstairs","session_id":"3d85ce56-1f78-498f-bdad-b28e370608f7"} {"msg":"[1]R flush_numbers[0..12]: [43, 45, 47, 48, 43, 43, 46, 46, 43, 43, 16, 35]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:13.114974166Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1294,"":"downstairs","":"downstairs","session_id":"3d85ce56-1f78-498f-bdad-b28e370608f7"} {"msg":"[1]R generation[0..12]: [24, 25, 26, 27, 24, 24, 26, 26, 24, 24, 10, 19]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:13.114988335Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1294,"":"downstairs","":"downstairs","session_id":"3d85ce56-1f78-498f-bdad-b28e370608f7"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:13.115002334Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1294,"":"downstairs","":"downstairs","session_id":"3d85ce56-1f78-498f-bdad-b28e370608f7"} {"msg":"[2]R flush_numbers[0..12]: [43, 45, 47, 48, 43, 43, 46, 46, 43, 43, 16, 35]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:13.115016203Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1294,"":"downstairs","":"downstairs","session_id":"3d85ce56-1f78-498f-bdad-b28e370608f7"} {"msg":"[2]R generation[0..12]: [24, 25, 26, 27, 24, 24, 26, 26, 24, 24, 10, 19]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:13.115029752Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1294,"":"downstairs","":"downstairs","session_id":"3d85ce56-1f78-498f-bdad-b28e370608f7"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:13.115043691Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1294,"":"downstairs","":"downstairs","session_id":"3d85ce56-1f78-498f-bdad-b28e370608f7"} {"msg":"Max found gen is 28","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:13.11505658Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1294,"":"downstairs","":"downstairs","session_id":"3d85ce56-1f78-498f-bdad-b28e370608f7"} {"msg":"Generation requested: 28 >= found:28","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:13.115069789Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1294,"":"downstairs","":"downstairs","session_id":"3d85ce56-1f78-498f-bdad-b28e370608f7"} {"msg":"Next flush: 49","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:13.115082879Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1294,"":"downstairs","":"downstairs","session_id":"3d85ce56-1f78-498f-bdad-b28e370608f7"} {"msg":"All extents match","v":0,"Wait for a query_work_queue command to finish before sending IO name":"crucible","level":30,"time":"2024-12-12T20:34:13.115099747Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1294,"":"downstairs","":"downstairs","session_id":"3d85ce56-1f78-498f-bdad-b28e370608f7"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:13.115113066Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1294,"session_id":"3d85ce56-1f78-498f-bdad-b28e370608f7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:13.115125076Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1294,"client":"0","":"downstairs","session_id":"3d85ce56-1f78-498f-bdad-b28e370608f7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:13.115138315Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1294,"client":"1","":"downstairs","session_id":"3d85ce56-1f78-498f-bdad-b28e370608f7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:13.115151444Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1294,"client":"2","":"downstairs","session_id":"3d85ce56-1f78-498f-bdad-b28e370608f7"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:13.115164753Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1294,"session_id":"3d85ce56-1f78-498f-bdad-b28e370608f7"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:13.115178992Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1294,"session_id":"3d85ce56-1f78-498f-bdad-b28e370608f7"} {"msg":"8e897076-76b1-4a62-8a36-44b624e9f2c7 is now active with session: 3d85ce56-1f78-498f-bdad-b28e370608f7","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:13.115191991Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1294,"session_id":"3d85ce56-1f78-498f-bdad-b28e370608f7"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:13.1152041Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1294,"session_id":"3d85ce56-1f78-498f-bdad-b28e370608f7"} {"msg":"The guest has finished waiting for activation with:28","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:13.115216379Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1294} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 577 len 40960 data: 3 3 3 2 2 3 3 3 2 2 02/30 Write block 153 len 40960 data: 2 2 2 3 3 3 3 4 4 6 03/30 Write block 478 len 4096 data: 3 04/30 Write block 290 len 36864 data: 3 3 3 5 5 4 4 4 3 05/30 Read block 471 len 32768 06/30 Read block 56 len 36864 07/30 Flush 08/30 Write block 565 len 8192 data: 4 4 09/30 Write block 590 len 4096 data: 2 10/30 Read block 405 len 12288 11/30 Read block 212 len 8192 12/30 Write block 204 len 20480 data: 2 2 3 3 3 13/30 Read block 19 len 12288 14/30 Read block 540 len 16384 15/30 Flush 16/30 Flush 17/30 Write block 197 len 32768 data: 3 2 2 2 2 2 2 3 18/30 Write block 116 len 24576 data: 3 2 2 2 2 2 19/30 Write block 122 len 12288 data: 2 2 2 20/30 Write block 328 len 36864 data: 4 4 4 4 2 3 3 3 3 21/30 Write block 114 len 20480 data: 4 3 4 3 3 22/30 Read block 398 len 8192 23/30 Write block 107 len 32768 data: 4 4 4 4 5 5 5 5 24/30 Read block 535 len 32768 25/30 Write block 187 len 28672 data: 5 6 5 5 5 5 5 26/30 Write block 523 len 4096 data: 6 27/30 Read block 74 len 12288 28/30 Write block 196 len 4096 data: 4 29/30 Read block 529 len 4096 30/30 Write block 154 len 28672 data: 3 3 4 4 4 4 5 ---------------------------------------------------------------- Crucible gen:28 GIO:true work queues: Upstairs:0 downstairs:14 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1016 Acked Write 8 Done Done Done false 1017 Acked Write 6 Done Done Done false 1018 Acked Write 3 Done Done Done false 1019 Acked Write 9 Done Done Done false 1020 Acked Write 5 Done Done Done false 1021 Acked Read 2 Done Done Done false 1022 Acked Write 8 Done Done Done false 1023 Acked Read 8 Done Done Done false 1024 Acked Write 7 Done Done Done false 1025 Acked Write 1 Done Done Done false 1026 Acked Read 3 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Read 1 Done Done Done false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 13 13 13 39 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1015 1015 1015 Downstairs last five completed: 1015 1014 1013 1012 1011 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:34:13.132816756Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1294} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 24 24 24 43 43 43 F F F 1 020-039 25 25 25 45 45 45 F F F 2 040-059 26 26 26 47 47 47 F F F 3 060-079 27 27 27 48 48 48 F F F 4 080-099 24 24 24 43 43 43 F F F 5 100-119 24 24 24 43 43 43 T T T 6 120-139 26 26 26 46 46 46 T T T 7 140-159 28 28 28 49 49 49 T T T 8 160-179 28 28 28 49 49 49 T T T 9 180-199 24 24 24 43 43 43 T T T 10 200-219 28 28 28 50 50 50 T T T 11 220-239 19 19 19 35 35 35 F F F 12 240-259 25 25 25 45 45 45 F F F 13 260-279 20 20 20 37 37 37 F F F 14 280-299 28 28 28 49 49 49 F F F 15 300-319 24 24 24 43 43 43 F F F 16 320-339 24 24 24 44 44 44 T T T 17 340-359 24 24 24 44 44 44 F F F 18 360-379 24 24 24 43 43 43 F F F 19 380-399 26 26 26 46 46 46 F F F 20 400-419 16 16 16 30 30 30 F F F 21 420-439 25 25 25 45 45 45 F F F 22 440-459 26 26 26 46 46 46 F F F 23 460-479 28 28 28 49 49 49 F F F 24 480-499 12 12 12 20 20 20 F F F 25 500-519 26 26 26 46 46 46 F F F 26 520-539 26 26 26 46 46 46 T T T 27 540-559 26 26 26 46 46 46 F F F 28 560-579 28 28 28 50 50 50 F F F 29 580-599 28 28 28 50 50 50 F F F Max gen: 28, Max flush: 50 A Difference in extent metadata is expected here On loop 14 Verifying data now No change after verify Loop: 14 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 24 24 24 43 43 43 F F F 1 020-039 25 25 25 45 45 45 F F F 2 040-059 26 26 26 47 47 47 F F F 3 060-079 27 27 27 48 48 48 F F F 4 080-099 24 24 24 43 43 43 F F F 5 100-119 29 29 29 51 51 51 F F F 6 120-139 29 29 29 51 51 51 F F F 7 140-159 29 29 29 51 51 51 F F F 8 160-179 29 29 29 51 51 51 F F F 9 180-199 29 29 29 51 51 51 F F F 10 200-219 29 29 29 51 51 51 F F F 11 220-239 19 19 19 35 35 35 F F F 12 240-259 25 25 25 45 45 45 F F F 13 260-279 20 20 20 37 37 37 F F F 14 280-299 28 28 28 49 49 49 F F F 15 300-319 24 24 24 43 43 43 F F F 16 320-339 29 29 29 51 51 51 F F F 17 340-359 24 24 24 44 44 44 F F F 18 360-379 24 24 24 43 43 43 F F F 19 380-399 26 26 26 46 46 46 F F F 20 400-419 16 16 16 30 30 30 F F F 21 420-439 25 25 25 45 45 45 F F F 22 440-459 26 26 26 46 46 46 F F F 23 460-479 28 28 28 49 49 49 F F F 24 480-499 12 12 12 20 20 20 F F F 25 500-519 26 26 26 46 46 46 F F F 26 520-539 29 29 29 51 51 51 F F F 27 540-559 26 26 26 46 46 46 F F F 28 560-579 28 28 28 50 50 50 F F F 29 580-599 28 28 28 50 50 50 F F F Max gen: 29, Max flush: 51 Begin loop 15 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:17.445234668Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1302,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:34:17.488345706Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1302,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:17.488741758Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1302,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:17.520374867Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1302,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:17.520940668Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1302} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"26e7d501b33f47379e53601720353c2983e7654b\",\n git_commit_timestamp: \"2024-12-12T20:16:40.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:17.520976666Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1302} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:17.520990445Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1302} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:17.521005433Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1302} {"msg":"Crucible e269f593-1b09-454c-91ae-a46110454c01 has session id: 974cda1a-fdf4-431f-8a49-50076d3d0a54","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:17.521473791Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1302,"session_id":"974cda1a-fdf4-431f-8a49-50076d3d0a54"} {"msg":"Upstairs opts: Upstairs UUID: e269f593-1b09-454c-91ae-a46110454c01, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:17.521520808Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1302,"session_id":"974cda1a-fdf4-431f-8a49-50076d3d0a54"} {"msg":"Crucible stats registered with UUID: e269f593-1b09-454c-91ae-a46110454c01","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:17.521545716Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1302,"session_id":"974cda1a-fdf4-431f-8a49-50076d3d0a54"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:17.552217641Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1302,"task":"crutest"} {"msg":"The guest has requested activation with gen:30","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:17.552248969Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1302} {"msg":"e269f593-1b09-454c-91ae-a46110454c01 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:17.552283457Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1302,"session_id":"974cda1a-fdf4-431f-8a49-50076d3d0a54"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:17.552301106Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1302,"client":"0","":"downstairs","session_id":"974cda1a-fdf4-431f-8a49-50076d3d0a54"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:17.552329673Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1302,"client":"0","":"downstairs","session_id":"974cda1a-fdf4-431f-8a49-50076d3d0a54"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:17.552359602Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1302,"client":"1","":"downstairs","session_id":"974cda1a-fdf4-431f-8a49-50076d3d0a54"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:17.552394989Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1302,"client":"1","":"downstairs","session_id":"974cda1a-fdf4-431f-8a49-50076d3d0a54"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:17.552466754Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1302,"client":"2","":"downstairs","session_id":"974cda1a-fdf4-431f-8a49-50076d3d0a54"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:17.552506871Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1302,"client":"2","":"downstairs","session_id":"974cda1a-fdf4-431f-8a49-50076d3d0a54"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:17.552535979Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1302,"":"io task","client":"2","":"downstairs","session_id":"974cda1a-fdf4-431f-8a49-50076d3d0a54"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:17.552562977Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1302,"":"io task","client":"0","":"downstairs","session_id":"974cda1a-fdf4-431f-8a49-50076d3d0a54"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:17.552592575Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1302,"":"io task","client":"1","":"downstairs","session_id":"974cda1a-fdf4-431f-8a49-50076d3d0a54"} {"msg":"ds_connection connected from Ok(127.0.0.1:33330)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:17.552661201Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1302,"":"io task","client":"2","":"downstairs","session_id":"974cda1a-fdf4-431f-8a49-50076d3d0a54"} {"msg":"ds_connection connected from Ok(127.0.0.1:33007)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:17.552751535Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1302,"":"io task","client":"0","":"downstairs","session_id":"974cda1a-fdf4-431f-8a49-50076d3d0a54"} {"msg":"ds_connection connected from Ok(127.0.0.1:45809)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:17.552787662Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1302,"":"io task","client":"1","":"downstairs","session_id":"974cda1a-fdf4-431f-8a49-50076d3d0a54"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:17.553204053Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1302,"client":"1","":"downstairs","session_id":"974cda1a-fdf4-431f-8a49-50076d3d0a54"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:17.553239021Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1302,"client":"2","":"downstairs","session_id":"974cda1a-fdf4-431f-8a49-50076d3d0a54"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:17.553484984Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1302,"client":"0","":"downstairs","session_id":"974cda1a-fdf4-431f-8a49-50076d3d0a54"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:17.553953952Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1302,"client":"1","":"downstairs","session_id":"974cda1a-fdf4-431f-8a49-50076d3d0a54"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:17.553983739Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1302,"client":"2","":"downstairs","session_id":"974cda1a-fdf4-431f-8a49-50076d3d0a54"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:17.554072223Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1302,"client":"0","":"downstairs","session_id":"974cda1a-fdf4-431f-8a49-50076d3d0a54"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:17.554537191Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1302,"session_id":"974cda1a-fdf4-431f-8a49-50076d3d0a54"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:17.554638054Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1302,"session_id":"974cda1a-fdf4-431f-8a49-50076d3d0a54"} {"msg":"[0]R flush_numbers[0..12]: [43, 45, 47, 48, 43, 51, 51, 51, 51, 51, 51, 35]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:17.554711879Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1302,"":"downstairs","":"downstairs","session_id":"974cda1a-fdf4-431f-8a49-50076d3d0a54"} {"msg":"[0]R generation[0..12]: [24, 25, 26, 27, 24, 29, 29, 29, 29, 29, 29, 19]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:17.554733138Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1302,"":"downstairs","":"downstairs","session_id":"974cda1a-fdf4-431f-8a49-50076d3d0a54"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:17.554748527Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1302,"":"downstairs","":"downstairs","session_id":"974cda1a-fdf4-431f-8a49-50076d3d0a54"} {"msg":"[1]R flush_numbers[0..12]: [43, 45, 47, 48, 43, 51, 51, 51, 51, 51, 51, 35]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:17.554762636Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1302,"":"downstairs","":"downstairs","session_id":"974cda1a-fdf4-431f-8a49-50076d3d0a54"} {"msg":"[1]R generation[0..12]: [24, 25, 26, 27, 24, 29, 29, 29, 29, 29, 29, 19]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:17.554776585Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1302,"":"downstairs","":"downstairs","session_id":"974cda1a-fdf4-431f-8a49-50076d3d0a54"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:17.554790794Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1302,"":"downstairs","":"downstairs","session_id":"974cda1a-fdf4-431f-8a49-50076d3d0a54"} {"msg":"[2]R flush_numbers[0..12]: [43, 45, 47, 48, 43, 51, 51, 51, 51, 51, 51, 35]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:17.554804373Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1302,"":"downstairs","":"downstairs","session_id":"974cda1a-fdf4-431f-8a49-50076d3d0a54"} {"msg":"[2]R generation[0..12]: [24, 25, 26, 27, 24, 29, 29, 29, 29, 29, 29, 19]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:17.554817922Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1302,"":"downstairs","":"downstairs","session_id":"974cda1a-fdf4-431f-8a49-50076d3d0a54"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:17.554832121Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1302,"":"downstairs","":"downstairs","session_id":"974cda1a-fdf4-431f-8a49-50076d3d0a54"} {"msg":"Max found gen is 30","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:17.55484528Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1302,"":"downstairs","":"downstairs","session_id":"974cda1a-fdf4-431f-8a49-50076d3d0a54"} {"msg":"Generation requested: 30 >= found:30","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:17.554858489Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1302,"":"downstairs","":"downstairs","session_id":"974cda1a-fdf4-431f-8a49-50076d3d0a54"} {"msg":"Next flush: 52","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:17.554874928Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1302,"":"downstairs","":"downstairs","session_id":"974cda1a-fdf4-431f-8a49-50076d3d0a54"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:17.554889087Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1302,"":"downstairs","":"downstairs","session_id":"974cda1a-fdf4-431f-8a49-50076d3d0a54"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:17.554902656Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1302,"session_id":"974cda1a-fdf4-431f-8a49-50076d3d0a54"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:17.554914685Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1302,"client":"0","":"downstairs","session_id":"974cda1a-fdf4-431f-8a49-50076d3d0a54"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-12-12T20:34:17.554928324Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1302,"client":"1","":"downstairs","session_id":"974cda1a-fdf4-431f-8a49-50076d3d0a54"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:17.554949433Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1302,"client":"2","":"downstairs","session_id":"974cda1a-fdf4-431f-8a49-50076d3d0a54"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:17.554962942Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1302,"session_id":"974cda1a-fdf4-431f-8a49-50076d3d0a54"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:17.554975121Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1302,"session_id":"974cda1a-fdf4-431f-8a49-50076d3d0a54"} {"msg":"e269f593-1b09-454c-91ae-a46110454c01 is now active with session: 974cda1a-fdf4-431f-8a49-50076d3d0a54","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:17.55498803Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1302,"session_id":"974cda1a-fdf4-431f-8a49-50076d3d0a54"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:17.555000179Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1302,"session_id":"974cda1a-fdf4-431f-8a49-50076d3d0a54"} {"msg":"The guest has finished waiting for activation with:30","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:17.555012479Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1302} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 402 len 8192 data: 4 4 02/30 Write block 529 len 36864 data: 3 2 3 3 3 3 3 3 3 03/30 Write block 211 len 28672 data: 3 2 2 2 2 2 2 04/30 Write block 110 len 24576 data: 5 6 6 6 6 4 05/30 Read block 326 len 28672 06/30 Write block 265 len 16384 data: 6 6 5 5 07/30 Write block 207 len 28672 data: 4 4 3 3 4 3 3 08/30 Write block 83 len 28672 data: 2 3 3 4 6 6 5 09/30 Read block 326 len 32768 10/30 Read block 340 len 20480 11/30 Read block 86 len 36864 12/30 Read block 183 len 36864 13/30 Read block 171 len 36864 14/30 Write block 460 len 24576 data: 3 3 3 3 3 3 15/30 Flush 16/30 Flush 17/30 Write block 202 len 20480 data: 3 3 4 3 4 18/30 Flush 19/30 Write block 449 len 12288 data: 3 3 3 20/30 Flush 21/30 Write block 356 len 12288 data: 4 5 3 22/30 Read block 227 len 12288 23/30 Flush 24/30 Write block 395 len 20480 data: 4 3 3 3 3 25/30 Read block 561 len 8192 26/30 Read block 121 len 24576 27/30 Read block 440 len 28672 28/30 Write block 562 len 8192 data: 4 4 29/30 Write block 211 len 8192 data: 5 4 30/30 Write block 74 len 20480 data: 3 4 3 3 3 ---------------------------------------------------------------- Crucible gen:30 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 5 Done Done Done false 1024 Acked Read 2 Done Done Done false 1025 Acked Read 6 Done Done Done false 1026 Acked Read 7 Done Done Done false 1027 Acked Write 2 Done Done Done false 1028 Acked Write 2 Sent Sent Sent false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:34:17.57311771Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1302} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 24 24 24 43 43 43 F F F 1 020-039 25 25 25 45 45 45 F F F 2 040-059 26 26 26 47 47 47 F F F 3 060-079 27 27 27 48 48 48 T T T 4 080-099 30 30 30 52 52 52 F F F 5 100-119 30 30 30 52 52 52 F F F 6 120-139 29 29 29 51 51 51 F F F 7 140-159 29 29 29 51 51 51 F F F 8 160-179 29 29 29 51 51 51 F F F 9 180-199 29 29 29 51 51 51 F F F 10 200-219 30 30 30 54 54 54 T T T 11 220-239 19 19 19 35 35 35 F F F 12 240-259 25 25 25 45 45 45 F F F 13 260-279 30 30 30 52 52 52 F F F 14 280-299 28 28 28 49 49 49 F F F 15 300-319 24 24 24 43 43 43 F F F 16 320-339 29 29 29 51 51 51 F F F 17 340-359 30 30 30 56 56 56 F F F 18 360-379 24 24 24 43 43 43 F F F 19 380-399 26 26 26 46 46 46 T T T 20 400-419 30 30 30 52 52 52 F F F 21 420-439 25 25 25 45 45 45 F F F 22 440-459 30 30 30 55 55 55 F F F 23 460-479 30 30 30 52 52 52 F F F 24 480-499 12 12 12 20 20 20 F F F 25 500-519 26 26 26 46 46 46 F F F 26 520-539 30 30 30 52 52 52 F F F 27 540-559 26 26 26 46 46 46 F F F 28 560-579 28 28 28 50 50 50 T T T 29 580-599 28 28 28 50 50 50 F F F Max gen: 30, Max flush: 56 A Difference in extent metadata is expected here On loop 15 Verifying data now No change after verify Loop: 15 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 24 24 24 43 43 43 F F F 1 020-039 25 25 25 45 45 45 F F F 2 040-059 26 26 26 47 47 47 F F F 3 060-079 31 31 31 57 57 57 F F F 4 080-099 30 30 30 52 52 52 F F F 5 100-119 30 30 30 52 52 52 F F F 6 120-139 29 29 29 51 51 51 F F F 7 140-159 29 29 29 51 51 51 F F F 8 160-179 29 29 29 51 51 51 F F F 9 180-199 29 29 29 51 51 51 F F F 10 200-219 31 31 31 57 57 57 F F F 11 220-239 19 19 19 35 35 35 F F F 12 240-259 25 25 25 45 45 45 F F F 13 260-279 30 30 30 52 52 52 F F F 14 280-299 28 28 28 49 49 49 F F F 15 300-319 24 24 24 43 43 43 F F F 16 320-339 29 29 29 51 51 51 F F F 17 340-359 30 30 30 56 56 56 F F F 18 360-379 24 24 24 43 43 43 F F F 19 380-399 31 31 31 57 57 57 F F F 20 400-419 30 30 30 52 52 52 F F F 21 420-439 25 25 25 45 45 45 F F F 22 440-459 30 30 30 55 55 55 F F F 23 460-479 30 30 30 52 52 52 F F F 24 480-499 12 12 12 20 20 20 F F F 25 500-519 26 26 26 46 46 46 F F F 26 520-539 30 30 30 52 52 52 F F F 27 540-559 26 26 26 46 46 46 F F F 28 560-579 31 31 31 57 57 57 F F F 29 580-599 28 28 28 50 50 50 F F F Max gen: 31, Max flush: 57 Begin loop 16 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:21.868057188Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1310,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:21.912669012Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1310,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:21.91342037Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1310} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"26e7d501b33f47379e53601720353c2983e7654b\",\n git_commit_timestamp: \"2024-12-12T20:16:40.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:21.913457278Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1310} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:21.913471457Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1310} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:21.913485506Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1310} {"msg":"Crucible a7ed17b8-be81-4ecb-97a5-b0d4bdc35dbf has session id: 9ce5742c-77f4-42df-8a9c-a87004cc248f","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:21.913982571Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1310,"session_id":"9ce5742c-77f4-42df-8a9c-a87004cc248f"} {"msg":"Upstairs opts: Upstairs UUID: a7ed17b8-be81-4ecb-97a5-b0d4bdc35dbf, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:21.914015269Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1310,"session_id":"9ce5742c-77f4-42df-8a9c-a87004cc248f"} {"msg":"Crucible stats registered with UUID: a7ed17b8-be81-4ecb-97a5-b0d4bdc35dbf","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:21.914029998Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1310,"session_id":"9ce5742c-77f4-42df-8a9c-a87004cc248f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:21.944915689Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1310,"task":"crutest"} {"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:21.944949716Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1310} {"msg":"a7ed17b8-be81-4ecb-97a5-b0d4bdc35dbf active request set","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:21.945016022Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1310,"session_id":"9ce5742c-77f4-42df-8a9c-a87004cc248f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:21.94503734Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1310,"client":"0","":"downstairs","session_id":"9ce5742c-77f4-42df-8a9c-a87004cc248f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:21.945071078Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1310,"client":"0","":"downstairs","session_id":"9ce5742c-77f4-42df-8a9c-a87004cc248f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:21.945103496Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1310,"client":"1","":"downstairs","session_id":"9ce5742c-77f4-42df-8a9c-a87004cc248f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:21.945138663Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1310,"client":"1","":"downstairs","session_id":"9ce5742c-77f4-42df-8a9c-a87004cc248f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:21.945173391Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1310,"client":"2","":"downstairs","session_id":"9ce5742c-77f4-42df-8a9c-a87004cc248f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:21.945208798Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1310,"client":"2","":"downstairs","session_id":"9ce5742c-77f4-42df-8a9c-a87004cc248f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:21.945247016Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1310,"":"io task","client":"2","":"downstairs","session_id":"9ce5742c-77f4-42df-8a9c-a87004cc248f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:21.945277404Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1310,"":"io task","client":"0","":"downstairs","session_id":"9ce5742c-77f4-42df-8a9c-a87004cc248f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:21.945377997Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1310,"":"io task","client":"1","":"downstairs","session_id":"9ce5742c-77f4-42df-8a9c-a87004cc248f"} {"msg":"ds_connection connected from Ok(127.0.0.1:58502)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:21.945421464Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1310,"":"io task","client":"0","":"downstairs","session_id":"9ce5742c-77f4-42df-8a9c-a87004cc248f"} {"msg":"ds_connection connected from Ok(127.0.0.1:60259)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:21.945452332Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1310,"":"io task","client":"2","":"downstairs","session_id":"9ce5742c-77f4-42df-8a9c-a87004cc248f"} {"msg":"ds_connection connected from Ok(127.0.0.1:43353)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:21.94547828Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1310,"":"io task","client":"1","":"downstairs","session_id":"9ce5742c-77f4-42df-8a9c-a87004cc248f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:21.946104376Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1310,"client":"0","":"downstairs","session_id":"9ce5742c-77f4-42df-8a9c-a87004cc248f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:21.946133185Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1310,"client":"1","":"downstairs","session_id":"9ce5742c-77f4-42df-8a9c-a87004cc248f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:21.946152503Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1310,"client":"2","":"downstairs","session_id":"9ce5742c-77f4-42df-8a9c-a87004cc248f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:21.946865214Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1310,"client":"1","":"downstairs","session_id":"9ce5742c-77f4-42df-8a9c-a87004cc248f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:21.946894982Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1310,"client":"2","":"downstairs","session_id":"9ce5742c-77f4-42df-8a9c-a87004cc248f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:21.946912881Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1310,"client":"0","":"downstairs","session_id":"9ce5742c-77f4-42df-8a9c-a87004cc248f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:21.947414046Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1310,"session_id":"9ce5742c-77f4-42df-8a9c-a87004cc248f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:21.947510579Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1310,"session_id":"9ce5742c-77f4-42df-8a9c-a87004cc248f"} {"msg":"[0]R flush_numbers[0..12]: [43, 45, 47, 57, 52, 52, 51, 51, 51, 51, 57, 35]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:21.947585385Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1310,"":"downstairs","":"downstairs","session_id":"9ce5742c-77f4-42df-8a9c-a87004cc248f"} {"msg":"[0]R generation[0..12]: [24, 25, 26, 31, 30, 30, 29, 29, 29, 29, 31, 19]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:21.947604673Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1310,"":"downstairs","":"downstairs","session_id":"9ce5742c-77f4-42df-8a9c-a87004cc248f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:21.947619242Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1310,"":"downstairs","":"downstairs","session_id":"9ce5742c-77f4-42df-8a9c-a87004cc248f"} {"msg":"[1]R flush_numbers[0..12]: [43, 45, 47, 57, 52, 52, 51, 51, 51, 51, 57, 35]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:21.947632971Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1310,"":"downstairs","":"downstairs","session_id":"9ce5742c-77f4-42df-8a9c-a87004cc248f"} {"msg":"[1]R generation[0..12]: [24, 25, 26, 31, 30, 30, 29, 29, 29, 29, 31, 19]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:21.94764639Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1310,"":"downstairs","":"downstairs","session_id":"9ce5742c-77f4-42df-8a9c-a87004cc248f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:21.947660239Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1310,"":"downstairs","":"downstairs","session_id":"9ce5742c-77f4-42df-8a9c-a87004cc248f"} {"msg":"[2]R flush_numbers[0..12]: [43, 45, 47, 57, 52, 52, 51, 51, 51, 51, 57, 35]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:21.947674198Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1310,"":"downstairs","":"downstairs","session_id":"9ce5742c-77f4-42df-8a9c-a87004cc248f"} {"msg":"[2]R generation[0..12]: [24, 25, 26, 31, 30, 30, 29, 29, 29, 29, 31, 19]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:21.947687877Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1310,"":"downstairs","":"downstairs","session_id":"9ce5742c-77f4-42df-8a9c-a87004cc248f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:21.947704756Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1310,"":"downstairs","":"downstairs","session_id":"9ce5742c-77f4-42df-8a9c-a87004cc248f"} {"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:21.947718045Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1310,"":"downstairs","":"downstairs","session_id":"9ce5742c-77f4-42df-8a9c-a87004cc248f"} {"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:21.947732064Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1310,"":"downstairs","":"downstairs","session_id":"9ce5742c-77f4-42df-8a9c-a87004cc248f"} {"msg":"Next flush: 58","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:21.947744943Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1310,"":"downstairs","":"downstairs","session_id":"9ce5742c-77f4-42df-8a9c-a87004cc248f"} {"msg":"Wait for a query_work_queue command to finish before sending IO All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:21.947762222Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1310,"":"downstairs","":"downstairs","session_id":"9ce5742c-77f4-42df-8a9c-a87004cc248f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:21.947775421Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1310,"session_id":"9ce5742c-77f4-42df-8a9c-a87004cc248f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:21.947787271Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1310,"client":"0","":"downstairs","session_id":"9ce5742c-77f4-42df-8a9c-a87004cc248f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:21.947800479Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1310,"client":"1","":"downstairs","session_id":"9ce5742c-77f4-42df-8a9c-a87004cc248f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:21.947813459Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1310,"client":"2","":"downstairs","session_id":"9ce5742c-77f4-42df-8a9c-a87004cc248f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:21.947826448Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1310,"session_id":"9ce5742c-77f4-42df-8a9c-a87004cc248f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:21.947838447Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1310,"session_id":"9ce5742c-77f4-42df-8a9c-a87004cc248f"} {"msg":"a7ed17b8-be81-4ecb-97a5-b0d4bdc35dbf is now active with session: 9ce5742c-77f4-42df-8a9c-a87004cc248f","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:21.947851136Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1310,"session_id":"9ce5742c-77f4-42df-8a9c-a87004cc248f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:21.947862905Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1310,"session_id":"9ce5742c-77f4-42df-8a9c-a87004cc248f"} {"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:21.947874974Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1310} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 390 len 8192 data: 5 5 02/30 Flush 03/30 Write block 456 len 16384 data: 3 4 3 3 04/30 Read block 337 len 28672 05/30 Read block 140 len 32768 06/30 Read block 289 len 32768 07/30 Read block 165 len 32768 08/30 Read block 374 len 8192 09/30 Write block 266 len 12288 data: 7 6 6 10/30 Write block 398 len 8192 data: 4 4 11/30 Read block 323 len 4096 12/30 Write block 442 len 24576 data: 4 4 4 4 6 6 13/30 Write block 307 len 20480 data: 2 3 3 3 6 14/30 Write block 333 len 8192 data: 4 4 15/30 Flush 16/30 Write block 519 len 28672 data: 7 6 6 6 7 6 4 17/30 Write block 269 len 40960 data: 4 4 4 6 4 4 3 3 3 3 18/30 Read block 401 len 36864 19/30 Read block 537 len 4096 20/30 Write block 85 len 28672 data: 4 5 7 7 6 5 5 21/30 Read block 79 len 8192 22/30 Write block 528 len 20480 data: 3 4 3 4 4 23/30 Flush 24/30 Write block 210 len 32768 data: 4 6 5 4 3 3 3 3 25/30 Flush 26/30 Write block 32 len 16384 data: 5 5 5 6 27/30 Write block 72 len 12288 data: 4 4 4 28/30 Read block 583 len 16384 29/30 Write block 62 len 20480 data: 2 3 3 3 3 30/30 Write block 253 len 4096 data: 6 ---------------------------------------------------------------- Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 4 Done Done Done false 1026 Acked Write 3 Done Done Done false 1027 Acked Read 4 Done Done Done false 1028 Acked Write 5 Sent Sent Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:34:21.965697205Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1310} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 24 24 24 43 43 43 F F F 1 020-039 25 25 25 45 45 45 T T T 2 040-059 26 26 26 47 47 47 F F F 3 060-079 31 31 31 57 57 57 T T T 4 080-099 32 32 32 60 60 60 F F F 5 100-119 30 30 30 52 52 52 F F F 6 120-139 29 29 29 51 51 51 F F F 7 140-159 29 29 29 51 51 51 F F F 8 160-179 29 29 29 51 51 51 F F F 9 180-199 29 29 29 51 51 51 F F F 10 200-219 32 32 32 61 61 61 F F F 11 220-239 19 19 19 35 35 35 F F F 12 240-259 25 25 25 45 45 45 T T T 13 260-279 32 32 32 60 60 60 F F F 14 280-299 28 28 28 49 49 49 F F F 15 300-319 32 32 32 59 59 59 F F F 16 320-339 32 32 32 59 59 59 F F F 17 340-359 30 30 30 56 56 56 F F F 18 360-379 24 24 24 43 43 43 F F F 19 380-399 32 32 32 59 59 59 F F F 20 400-419 30 30 30 52 52 52 F F F 21 420-439 25 25 25 45 45 45 F F F 22 440-459 32 32 32 59 59 59 F F F 23 460-479 30 30 30 52 52 52 F F F 24 480-499 12 12 12 20 20 20 F F F 25 500-519 32 32 32 60 60 60 F F F 26 520-539 32 32 32 60 60 60 F F F 27 540-559 26 26 26 46 46 46 F F F 28 560-579 31 31 31 57 57 57 F F F 29 580-599 28 28 28 50 50 50 F F F Max gen: 32, Max flush: 61 A Difference in extent metadata is expected here On loop 16 Verifying data now No change after verify Loop: 16 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 24 24 24 43 43 43 F F F 1 020-039 33 33 33 62 62 62 F F F 2 040-059 26 26 26 47 47 47 F F F 3 060-079 33 33 33 62 62 62 F F F 4 080-099 32 32 32 60 60 60 F F F 5 100-119 30 30 30 52 52 52 F F F 6 120-139 29 29 29 51 51 51 F F F 7 140-159 29 29 29 51 51 51 F F F 8 160-179 29 29 29 51 51 51 F F F 9 180-199 29 29 29 51 51 51 F F F 10 200-219 32 32 32 61 61 61 F F F 11 220-239 19 19 19 35 35 35 F F F 12 240-259 33 33 33 62 62 62 F F F 13 260-279 32 32 32 60 60 60 F F F 14 280-299 28 28 28 49 49 49 F F F 15 300-319 32 32 32 59 59 59 F F F 16 320-339 32 32 32 59 59 59 F F F 17 340-359 30 30 30 56 56 56 F F F 18 360-379 24 24 24 43 43 43 F F F 19 380-399 32 32 32 59 59 59 F F F 20 400-419 30 30 30 52 52 52 F F F 21 420-439 25 25 25 45 45 45 F F F 22 440-459 32 32 32 59 59 59 F F F 23 460-479 30 30 30 52 52 52 F F F 24 480-499 12 12 12 20 20 20 F F F 25 500-519 32 32 32 60 60 60 F F F 26 520-539 32 32 32 60 60 60 F F F 27 540-559 26 26 26 46 46 46 F F F 28 560-579 31 31 31 57 57 57 F F F 29 580-599 28 28 28 50 50 50 F F F Max gen: 33, Max flush: 62 Begin loop 17 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:26.227948557Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1318,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:26.273781537Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1318,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:26.274484449Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1318} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"26e7d501b33f47379e53601720353c2983e7654b\",\n git_commit_timestamp: \"2024-12-12T20:16:40.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:26.274520006Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1318} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:26.274534165Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1318} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:26.274548074Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1318} {"msg":"Crucible f33bd3a4-d4cb-4334-b9a3-7f01bd7fd48f has session id: 2c88277f-72cb-4761-b852-fd952eb3bdac","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:26.274926968Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1318,"session_id":"2c88277f-72cb-4761-b852-fd952eb3bdac"} {"msg":"Upstairs opts: Upstairs UUID: f33bd3a4-d4cb-4334-b9a3-7f01bd7fd48f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:26.274972225Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1318,"session_id":"2c88277f-72cb-4761-b852-fd952eb3bdac"} {"msg":"Crucible stats registered with UUID: f33bd3a4-d4cb-4334-b9a3-7f01bd7fd48f","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:26.275002943Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1318,"session_id":"2c88277f-72cb-4761-b852-fd952eb3bdac"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:26.306168734Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1318,"task":"crutest"} {"msg":"The guest has requested activation with gen:34","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:26.306198692Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1318} {"msg":"f33bd3a4-d4cb-4334-b9a3-7f01bd7fd48f active request set","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:26.306262827Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1318,"session_id":"2c88277f-72cb-4761-b852-fd952eb3bdac"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:26.306291085Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1318,"client":"0","":"downstairs","session_id":"2c88277f-72cb-4761-b852-fd952eb3bdac"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:26.306315184Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1318,"client":"0","":"downstairs","session_id":"2c88277f-72cb-4761-b852-fd952eb3bdac"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:26.306330063Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1318,"client":"1","":"downstairs","session_id":"2c88277f-72cb-4761-b852-fd952eb3bdac"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:26.306344352Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1318,"client":"1","":"downstairs","session_id":"2c88277f-72cb-4761-b852-fd952eb3bdac"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:26.306361401Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1318,"client":"2","":"downstairs","session_id":"2c88277f-72cb-4761-b852-fd952eb3bdac"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:26.306386549Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1318,"client":"2","":"downstairs","session_id":"2c88277f-72cb-4761-b852-fd952eb3bdac"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:26.306411747Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1318,"":"io task","client":"2","":"downstairs","session_id":"2c88277f-72cb-4761-b852-fd952eb3bdac"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:26.306436835Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1318,"":"io task","client":"0","":"downstairs","session_id":"2c88277f-72cb-4761-b852-fd952eb3bdac"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:26.306462843Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1318,"":"io task","client":"1","":"downstairs","session_id":"2c88277f-72cb-4761-b852-fd952eb3bdac"} {"msg":"ds_connection connected from Ok(127.0.0.1:48983)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:26.306665409Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1318,"":"io task","client":"2","":"downstairs","session_id":"2c88277f-72cb-4761-b852-fd952eb3bdac"} {"msg":"ds_connection connected from Ok(127.0.0.1:46451)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:26.306684428Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1318,"":"io task","client":"0","":"downstairs","session_id":"2c88277f-72cb-4761-b852-fd952eb3bdac"} {"msg":"ds_connection connected from Ok(127.0.0.1:65071)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:26.306702387Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1318,"":"io task","client":"1","":"downstairs","session_id":"2c88277f-72cb-4761-b852-fd952eb3bdac"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:26.307286097Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1318,"client":"0","":"downstairs","session_id":"2c88277f-72cb-4761-b852-fd952eb3bdac"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:26.307324374Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1318,"client":"2","":"downstairs","session_id":"2c88277f-72cb-4761-b852-fd952eb3bdac"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:26.307463494Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1318,"client":"1","":"downstairs","session_id":"2c88277f-72cb-4761-b852-fd952eb3bdac"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:26.308024956Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1318,"client":"0","":"downstairs","session_id":"2c88277f-72cb-4761-b852-fd952eb3bdac"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:26.308066533Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1318,"client":"1","":"downstairs","session_id":"2c88277f-72cb-4761-b852-fd952eb3bdac"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:26.308099541Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1318,"client":"2","":"downstairs","session_id":"2c88277f-72cb-4761-b852-fd952eb3bdac"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:26.308575728Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1318,"session_id":"2c88277f-72cb-4761-b852-fd952eb3bdac"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:26.308681541Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1318,"session_id":"2c88277f-72cb-4761-b852-fd952eb3bdac"} {"msg":"[0]R flush_numbers[0..12]: [43, 62, 47, 62, 60, 52, 51, 51, 51, 51, 61, 35]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:26.308783874Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1318,"":"downstairs","":"downstairs","session_id":"2c88277f-72cb-4761-b852-fd952eb3bdac"} {"msg":"[0]R generation[0..12]: [24, 33, 26, 33, 32, 30, 29, 29, 29, 29, 32, 19]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:26.308813332Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1318,"":"downstairs","":"downstairs","session_id":"2c88277f-72cb-4761-b852-fd952eb3bdac"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:26.30882992Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1318,"":"downstairs","":"downstairs","session_id":"2c88277f-72cb-4761-b852-fd952eb3bdac"} {"msg":"[1]R flush_numbers[0..12]: [43, 62, 47, 62, 60, 52, 51, 51, 51, 51, 61, 35]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:26.308844279Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1318,"":"downstairs","":"downstairs","session_id":"2c88277f-72cb-4761-b852-fd952eb3bdac"} {"msg":"[1]R generation[0..12]: [24, 33, 26, 33, 32, 30, 29, 29, 29, 29, 32, 19]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:26.308857958Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1318,"":"downstairs","":"downstairs","session_id":"2c88277f-72cb-4761-b852-fd952eb3bdac"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:26.308875927Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1318,"":"downstairs","":"downstairs","session_id":"2c88277f-72cb-4761-b852-fd952eb3bdac"} {"msg":"[2]R flush_numbers[0..12]: [43, 62, 47, 62, 60, 52, 51, 51, 51, 51, 61, 35]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:26.308890146Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1318,"":"downstairs","":"downstairs","session_id":"2c88277f-72cb-4761-b852-fd952eb3bdac"} {"msg":"[2]R generation[0..12]: [24, 33, 26, 33, 32, 30, 29, 29, 29, 29, 32, 19]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:26.308903825Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1318,"":"downstairs","":"downstairs","session_id":"2c88277f-72cb-4761-b852-fd952eb3bdac"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:26.308917754Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1318,"":"downstairs","":"downstairs","session_id":"2c88277f-72cb-4761-b852-fd952eb3bdac"} {"msg":"Max found gen is 34","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:26.308930793Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1318,"":"downstairs","":"downstairs","session_id":"2c88277f-72cb-4761-b852-fd952eb3bdac"} {"msg":"Generation requested: 34 >= found:34","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-12-12T20:34:26.308943832Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1318,"":"downstairs","":"downstairs","session_id":"2c88277f-72cb-4761-b852-fd952eb3bdac"} {"msg":"Next flush: 63","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:26.308966181Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1318,"":"downstairs","":"downstairs","session_id":"2c88277f-72cb-4761-b852-fd952eb3bdac"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:26.30898064Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1318,"":"downstairs","":"downstairs","session_id":"2c88277f-72cb-4761-b852-fd952eb3bdac"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:26.308993849Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1318,"session_id":"2c88277f-72cb-4761-b852-fd952eb3bdac"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:26.309005878Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1318,"client":"0","":"downstairs","session_id":"2c88277f-72cb-4761-b852-fd952eb3bdac"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:26.309019147Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1318,"client":"1","":"downstairs","session_id":"2c88277f-72cb-4761-b852-fd952eb3bdac"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:26.309032126Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1318,"client":"2","":"downstairs","session_id":"2c88277f-72cb-4761-b852-fd952eb3bdac"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:26.309045345Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1318,"session_id":"2c88277f-72cb-4761-b852-fd952eb3bdac"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:26.309057475Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1318,"session_id":"2c88277f-72cb-4761-b852-fd952eb3bdac"} {"msg":"f33bd3a4-d4cb-4334-b9a3-7f01bd7fd48f is now active with session: 2c88277f-72cb-4761-b852-fd952eb3bdac","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:26.309070074Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1318,"session_id":"2c88277f-72cb-4761-b852-fd952eb3bdac"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:26.309083973Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1318,"session_id":"2c88277f-72cb-4761-b852-fd952eb3bdac"} {"msg":"The guest has finished waiting for activation with:34","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:26.309096292Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1318} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 93 len 4096 data: 7 02/30 Read block 412 len 16384 03/30 Flush 04/30 Flush 05/30 Flush 06/30 Write block 332 len 20480 data: 3 5 5 4 4 07/30 Write block 238 len 36864 data: 2 3 3 3 3 3 2 2 2 08/30 Write block 501 len 20480 data: 3 4 3 3 3 09/30 Write block 36 len 36864 data: 5 6 7 8 6 6 5 5 4 10/30 Read block 282 len 20480 11/30 Flush 12/30 Write block 313 len 24576 data: 7 8 9 9 8 7 13/30 Write block 402 len 28672 data: 5 5 4 4 4 4 4 14/30 Flush 15/30 Write block 378 len 36864 data: 4 5 5 5 6 5 4 5 6 16/30 Read block 562 len 32768 17/30 Read block 255 len 24576 18/30 Read block 220 len 8192 19/30 Read block 497 len 24576 20/30 Write block 80 len 4096 data: 3 21/30 Write block 128 len 12288 data: 3 3 3 22/30 Read block 223 len 8192 23/30 Flush 24/30 Write block 136 len 16384 data: 3 3 3 3 25/30 Flush 26/30 Write block 516 len 4096 data: 3 27/30 Write block 142 len 4096 data: 2 28/30 Write block 56 len 24576 data: 6 6 7 6 5 2 29/30 Read block 94 len 16384 30/30 Read block 4 len 8192 ---------------------------------------------------------------- Crucible gen:34 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 1 Done Done Done false 1026 Acked Write 1 Done Done Done false 1027 Acked Write 6 Done Done Done false 1028 Acked Read 4 Done Done Done false 1029 Acked Read 2 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:34:26.327006667Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1318} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 24 24 24 43 43 43 F F F 1 020-039 34 34 34 66 66 66 F F F 2 040-059 34 34 34 66 66 66 T T T 3 060-079 33 33 33 62 62 62 T T T 4 080-099 34 34 34 68 68 68 F F F 5 100-119 30 30 30 52 52 52 F F F 6 120-139 34 34 34 69 69 69 F F F 7 140-159 29 29 29 51 51 51 T T T 8 160-179 29 29 29 51 51 51 F F F 9 180-199 29 29 29 51 51 51 F F F 10 200-219 32 32 32 61 61 61 F F F 11 220-239 34 34 34 66 66 66 F F F 12 240-259 34 34 34 66 66 66 F F F 13 260-279 32 32 32 60 60 60 F F F 14 280-299 28 28 28 49 49 49 F F F 15 300-319 34 34 34 67 67 67 F F F 16 320-339 34 34 34 66 66 66 F F F 17 340-359 30 30 30 56 56 56 F F F 18 360-379 34 34 34 68 68 68 F F F 19 380-399 34 34 34 68 68 68 F F F 20 400-419 34 34 34 67 67 67 F F F 21 420-439 25 25 25 45 45 45 F F F 22 440-459 32 32 32 59 59 59 F F F 23 460-479 30 30 30 52 52 52 F F F 24 480-499 12 12 12 20 20 20 F F F 25 500-519 34 34 34 66 66 66 T T T 26 520-539 32 32 32 60 60 60 F F F 27 540-559 26 26 26 46 46 46 F F F 28 560-579 31 31 31 57 57 57 F F F 29 580-599 28 28 28 50 50 50 F F F Max gen: 34, Max flush: 69 A Difference in extent metadata is expected here On loop 17 Verifying data now No change after verify Loop: 17 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 24 24 24 43 43 43 F F F 1 020-039 34 34 34 66 66 66 F F F 2 040-059 35 35 35 70 70 70 F F F 3 060-079 35 35 35 70 70 70 F F F 4 080-099 34 34 34 68 68 68 F F F 5 100-119 30 30 30 52 52 52 F F F 6 120-139 34 34 34 69 69 69 F F F 7 140-159 35 35 35 70 70 70 F F F 8 160-179 29 29 29 51 51 51 F F F 9 180-199 29 29 29 51 51 51 F F F 10 200-219 32 32 32 61 61 61 F F F 11 220-239 34 34 34 66 66 66 F F F 12 240-259 34 34 34 66 66 66 F F F 13 260-279 32 32 32 60 60 60 F F F 14 280-299 28 28 28 49 49 49 F F F 15 300-319 34 34 34 67 67 67 F F F 16 320-339 34 34 34 66 66 66 F F F 17 340-359 30 30 30 56 56 56 F F F 18 360-379 34 34 34 68 68 68 F F F 19 380-399 34 34 34 68 68 68 F F F 20 400-419 34 34 34 67 67 67 F F F 21 420-439 25 25 25 45 45 45 F F F 22 440-459 32 32 32 59 59 59 F F F 23 460-479 30 30 30 52 52 52 F F F 24 480-499 12 12 12 20 20 20 F F F 25 500-519 35 35 35 70 70 70 F F F 26 520-539 32 32 32 60 60 60 F F F 27 540-559 26 26 26 46 46 46 F F F 28 560-579 31 31 31 57 57 57 F F F 29 580-599 28 28 28 50 50 50 F F F Max gen: 35, Max flush: 70 Begin loop 18 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:30.591266751Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1326,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:30.635896063Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1326,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:30.636564117Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1326} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"26e7d501b33f47379e53601720353c2983e7654b\",\n git_commit_timestamp: \"2024-12-12T20:16:40.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:30.636601234Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1326} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:30.636615413Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1326} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:30.636629883Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1326} {"msg":"Crucible 34c22854-acfb-4d31-b32c-0cf13bbcaad7 has session id: 3d05894a-2589-4869-b821-6d7360e3cc62","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:30.63710299Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1326,"session_id":"3d05894a-2589-4869-b821-6d7360e3cc62"} {"msg":"Upstairs opts: Upstairs UUID: 34c22854-acfb-4d31-b32c-0cf13bbcaad7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:30.637148987Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1326,"session_id":"3d05894a-2589-4869-b821-6d7360e3cc62"} {"msg":"Crucible stats registered with UUID: 34c22854-acfb-4d31-b32c-0cf13bbcaad7","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:30.637167946Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1326,"session_id":"3d05894a-2589-4869-b821-6d7360e3cc62"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:30.668269141Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1326,"task":"crutest"} {"msg":"The guest has requested activation with gen:36","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:30.668312328Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1326} {"msg":"34c22854-acfb-4d31-b32c-0cf13bbcaad7 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:30.668377293Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1326,"session_id":"3d05894a-2589-4869-b821-6d7360e3cc62"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:30.668411631Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1326,"client":"0","":"downstairs","session_id":"3d05894a-2589-4869-b821-6d7360e3cc62"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:30.668441999Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1326,"client":"0","":"downstairs","session_id":"3d05894a-2589-4869-b821-6d7360e3cc62"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:30.668460218Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1326,"client":"1","":"downstairs","session_id":"3d05894a-2589-4869-b821-6d7360e3cc62"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:30.668478127Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1326,"client":"1","":"downstairs","session_id":"3d05894a-2589-4869-b821-6d7360e3cc62"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:30.668494795Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1326,"client":"2","":"downstairs","session_id":"3d05894a-2589-4869-b821-6d7360e3cc62"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:30.668512204Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1326,"client":"2","":"downstairs","session_id":"3d05894a-2589-4869-b821-6d7360e3cc62"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:30.668528443Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1326,"":"io task","client":"2","":"downstairs","session_id":"3d05894a-2589-4869-b821-6d7360e3cc62"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:30.668545452Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1326,"":"io task","client":"0","":"downstairs","session_id":"3d05894a-2589-4869-b821-6d7360e3cc62"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:30.668562141Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1326,"":"io task","client":"1","":"downstairs","session_id":"3d05894a-2589-4869-b821-6d7360e3cc62"} {"msg":"ds_connection connected from Ok(127.0.0.1:36636)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:30.668687812Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1326,"":"io task","client":"0","":"downstairs","session_id":"3d05894a-2589-4869-b821-6d7360e3cc62"} {"msg":"ds_connection connected from Ok(127.0.0.1:61596)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:30.668730809Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1326,"":"io task","client":"2","":"downstairs","session_id":"3d05894a-2589-4869-b821-6d7360e3cc62"} {"msg":"ds_connection connected from Ok(127.0.0.1:34963)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:30.668763067Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1326,"":"io task","client":"1","":"downstairs","session_id":"3d05894a-2589-4869-b821-6d7360e3cc62"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:30.669494246Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1326,"client":"0","":"downstairs","session_id":"3d05894a-2589-4869-b821-6d7360e3cc62"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:30.669533234Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1326,"client":"2","":"downstairs","session_id":"3d05894a-2589-4869-b821-6d7360e3cc62"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:30.669564141Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1326,"client":"1","":"downstairs","session_id":"3d05894a-2589-4869-b821-6d7360e3cc62"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:30.670154371Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1326,"client":"2","":"downstairs","session_id":"3d05894a-2589-4869-b821-6d7360e3cc62"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:30.670257824Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1326,"client":"1","":"downstairs","session_id":"3d05894a-2589-4869-b821-6d7360e3cc62"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:30.670339858Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1326,"client":"0","":"downstairs","session_id":"3d05894a-2589-4869-b821-6d7360e3cc62"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:30.670679025Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1326,"session_id":"3d05894a-2589-4869-b821-6d7360e3cc62"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:30.670772538Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1326,"session_id":"3d05894a-2589-4869-b821-6d7360e3cc62"} {"msg":"[0]R flush_numbers[0..12]: [43, 66, 70, 70, 68, 52, 69, 70, 51, 51, 61, 66]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:30.670897889Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1326,"":"downstairs","":"downstairs","session_id":"3d05894a-2589-4869-b821-6d7360e3cc62"} {"msg":"[0]R generation[0..12]: [24, 34, 35, 35, 34, 30, 34, 35, 29, 29, 32, 34]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:30.670918848Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1326,"":"downstairs","":"downstairs","session_id":"3d05894a-2589-4869-b821-6d7360e3cc62"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:30.670933837Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1326,"":"downstairs","":"downstairs","session_id":"3d05894a-2589-4869-b821-6d7360e3cc62"} {"msg":"[1]R flush_numbers[0..12]: [43, 66, 70, 70, 68, 52, 69, 70, 51, 51, 61, 66]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:30.670950796Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1326,"":"downstairs","":"downstairs","session_id":"3d05894a-2589-4869-b821-6d7360e3cc62"} {"msg":"[1]R generation[0..12]: [24, 34, 35, 35, 34, 30, 34, 35, 29, 29, 32, 34]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:30.670964775Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1326,"":"downstairs","":"downstairs","session_id":"3d05894a-2589-4869-b821-6d7360e3cc62"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:30.670978814Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1326,"":"downstairs","":"downstairs","session_id":"3d05894a-2589-4869-b821-6d7360e3cc62"} {"msg":"[2]R flush_numbers[0..12]: [43, 66, 70, 70, 68, 52, 69, 70, 51, 51, 61, 66]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:30.671005442Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1326,"":"downstairs","":"downstairs","session_id":"3d05894a-2589-4869-b821-6d7360e3cc62"} {"msg":"[2]R generation[0..12]: [24, 34, 35, 35, 34, 30, 34, 35, 29, 29, 32, 34]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:30.671020211Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1326,"":"downstairs","":"downstairs","session_id":"3d05894a-2589-4869-b821-6d7360e3cc62"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:30.67103415Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1326,"":"downstairs","":"downstairs","session_id":"3d05894a-2589-4869-b821-6d7360e3cc62"} {"msg":"Max found gen is 36","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:30.671047219Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1326,"":"downstairs","":"downstairs","session_id":"3d05894a-2589-4869-b821-6d7360e3cc62"} {"msg":"Generation requested: 36 >= found:36","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:30.671060318Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1326,"":"downstairs","":"downstairs","session_id":"3d05894a-2589-4869-b821-6d7360e3cc62"} {"msg":"Next flush: 71","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-12-12T20:34:30.671073068Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1326,"":"downstairs","":"downstairs","session_id":"3d05894a-2589-4869-b821-6d7360e3cc62"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:30.671090576Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1326,"":"downstairs","":"downstairs","session_id":"3d05894a-2589-4869-b821-6d7360e3cc62"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:30.671103835Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1326,"session_id":"3d05894a-2589-4869-b821-6d7360e3cc62"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:30.671115805Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1326,"client":"0","":"downstairs","session_id":"3d05894a-2589-4869-b821-6d7360e3cc62"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:30.671128974Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1326,"client":"1","":"downstairs","session_id":"3d05894a-2589-4869-b821-6d7360e3cc62"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:30.671141953Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1326,"client":"2","":"downstairs","session_id":"3d05894a-2589-4869-b821-6d7360e3cc62"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:30.671154902Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1326,"session_id":"3d05894a-2589-4869-b821-6d7360e3cc62"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:30.671169011Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1326,"session_id":"3d05894a-2589-4869-b821-6d7360e3cc62"} {"msg":"34c22854-acfb-4d31-b32c-0cf13bbcaad7 is now active with session: 3d05894a-2589-4869-b821-6d7360e3cc62","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:30.67118187Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1326,"session_id":"3d05894a-2589-4869-b821-6d7360e3cc62"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:30.671193899Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1326,"session_id":"3d05894a-2589-4869-b821-6d7360e3cc62"} {"msg":"The guest has finished waiting for activation with:36","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:30.671205948Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1326} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 597 len 12288 data: 2 2 2 02/30 Write block 233 len 36864 data: 4 4 4 3 2 3 4 4 4 03/30 Write block 376 len 20480 data: 4 3 5 6 6 04/30 Flush 05/30 Write block 7 len 20480 data: 2 3 3 3 3 06/30 Write block 405 len 16384 data: 5 5 5 5 07/30 Write block 588 len 28672 data: 2 2 3 2 2 2 2 08/30 Flush 09/30 Write block 142 len 36864 data: 3 2 3 4 4 4 5 3 3 10/30 Write block 81 len 20480 data: 3 2 3 4 5 11/30 Write block 30 len 28672 data: 4 5 6 6 6 7 6 12/30 Read block 334 len 40960 13/30 Read block 100 len 40960 14/30 Flush 15/30 Write block 350 len 4096 data: 3 16/30 Flush 17/30 Write block 20 len 20480 data: 3 3 3 2 3 18/30 Read block 170 len 36864 19/30 Write block 450 len 20480 data: 4 4 4 4 4 20/30 Write block 460 len 16384 data: 4 4 4 4 21/30 Read block 430 len 8192 22/30 Read block 165 len 32768 23/30 Write block 39 len 8192 data: 9 7 24/30 Read block 87 len 40960 25/30 Read block 558 len 24576 26/30 Read block 446 len 28672 27/30 Read block 481 len 20480 28/30 Read block 239 len 20480 29/30 Read block 144 len 24576 30/30 Write block 8 len 32768 data: 4 4 4 4 3 3 4 6 ---------------------------------------------------------------- Crucible gen:36 GIO:true work queues: Upstairs:0 downstairs:14 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1016 Acked Write 5 Done Done Done false 1017 Acked Read 9 Done Done Done false 1018 Acked Write 5 Done Done Done false 1019 Acked Write 4 Done Done Done false 1020 Acked Read 2 Done Done Done false 1021 Acked Read 8 Done Done Done false 1022 Acked Write 2 Done Done Done false 1023 Acked Read 10 Done Done Done false 1024 Acked Read 6 Done Done Done false 1025 Acked Read 7 Done Done Done false 1026 Acked Read 5 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Read 6 Done Done Done false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 13 13 13 39 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1015 1015 1015 Downstairs last five completed: 1015 1014 1013 1012 1011 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:34:30.689989373Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1326} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 36 36 36 72 72 72 T T T 1 020-039 36 36 36 73 73 73 T T T 2 040-059 35 35 35 70 70 70 T T T 3 060-079 35 35 35 70 70 70 F F F 4 080-099 36 36 36 73 73 73 F F F 5 100-119 30 30 30 52 52 52 F F F 6 120-139 34 34 34 69 69 69 F F F 7 140-159 36 36 36 73 73 73 F F F 8 160-179 29 29 29 51 51 51 F F F 9 180-199 29 29 29 51 51 51 F F F 10 200-219 32 32 32 61 61 61 F F F 11 220-239 36 36 36 71 71 71 F F F 12 240-259 36 36 36 71 71 71 F F F 13 260-279 32 32 32 60 60 60 F F F 14 280-299 28 28 28 49 49 49 F F F 15 300-319 34 34 34 67 67 67 F F F 16 320-339 34 34 34 66 66 66 F F F 17 340-359 36 36 36 74 74 74 F F F 18 360-379 36 36 36 71 71 71 F F F 19 380-399 36 36 36 71 71 71 F F F 20 400-419 36 36 36 72 72 72 F F F 21 420-439 25 25 25 45 45 45 F F F 22 440-459 32 32 32 59 59 59 T T T 23 460-479 30 30 30 52 52 52 T T T 24 480-499 12 12 12 20 20 20 F F F 25 500-519 35 35 35 70 70 70 F F F 26 520-539 32 32 32 60 60 60 F F F 27 540-559 26 26 26 46 46 46 F F F 28 560-579 31 31 31 57 57 57 F F F 29 580-599 36 36 36 72 72 72 F F F Max gen: 36, Max flush: 74 A Difference in extent metadata is expected here On loop 18 Verifying data now No change after verify Loop: 18 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 37 37 37 75 75 75 F F F 1 020-039 37 37 37 75 75 75 F F F 2 040-059 37 37 37 75 75 75 F F F 3 060-079 35 35 35 70 70 70 F F F 4 080-099 36 36 36 73 73 73 F F F 5 100-119 30 30 30 52 52 52 F F F 6 120-139 34 34 34 69 69 69 F F F 7 140-159 36 36 36 73 73 73 F F F 8 160-179 29 29 29 51 51 51 F F F 9 180-199 29 29 29 51 51 51 F F F 10 200-219 32 32 32 61 61 61 F F F 11 220-239 36 36 36 71 71 71 F F F 12 240-259 36 36 36 71 71 71 F F F 13 260-279 32 32 32 60 60 60 F F F 14 280-299 28 28 28 49 49 49 F F F 15 300-319 34 34 34 67 67 67 F F F 16 320-339 34 34 34 66 66 66 F F F 17 340-359 36 36 36 74 74 74 F F F 18 360-379 36 36 36 71 71 71 F F F 19 380-399 36 36 36 71 71 71 F F F 20 400-419 36 36 36 72 72 72 F F F 21 420-439 25 25 25 45 45 45 F F F 22 440-459 37 37 37 75 75 75 F F F 23 460-479 37 37 37 75 75 75 F F F 24 480-499 12 12 12 20 20 20 F F F 25 500-519 35 35 35 70 70 70 F F F 26 520-539 32 32 32 60 60 60 F F F 27 540-559 26 26 26 46 46 46 F F F 28 560-579 31 31 31 57 57 57 F F F 29 580-599 36 36 36 72 72 72 F F F Max gen: 37, Max flush: 75 Begin loop 19 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:34.960734269Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1334,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:34:35.003892253Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1334,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:35.004301664Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1334,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:35.036080323Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1334,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:35.036651163Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1334} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"26e7d501b33f47379e53601720353c2983e7654b\",\n git_commit_timestamp: \"2024-12-12T20:16:40.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:35.036686641Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1334} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:35.03670051Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1334} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:35.036714889Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1334} {"msg":"Crucible eef9a1d9-fef0-408b-b955-d41c0f733d8a has session id: 173f483c-deee-41f2-9a34-e2b6aed083b0","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:35.037125741Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1334,"session_id":"173f483c-deee-41f2-9a34-e2b6aed083b0"} {"msg":"Upstairs opts: Upstairs UUID: eef9a1d9-fef0-408b-b955-d41c0f733d8a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:35.037174658Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1334,"session_id":"173f483c-deee-41f2-9a34-e2b6aed083b0"} {"msg":"Crucible stats registered with UUID: eef9a1d9-fef0-408b-b955-d41c0f733d8a","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:35.037201545Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1334,"session_id":"173f483c-deee-41f2-9a34-e2b6aed083b0"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:35.067964274Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1334,"task":"crutest"} {"msg":"The guest has requested activation with gen:38","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:35.067995812Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1334} {"msg":"eef9a1d9-fef0-408b-b955-d41c0f733d8a active request set","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:35.0680262Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1334,"session_id":"173f483c-deee-41f2-9a34-e2b6aed083b0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:35.068044389Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1334,"client":"0","":"downstairs","session_id":"173f483c-deee-41f2-9a34-e2b6aed083b0"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:35.068072027Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1334,"client":"0","":"downstairs","session_id":"173f483c-deee-41f2-9a34-e2b6aed083b0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:35.068093535Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1334,"client":"1","":"downstairs","session_id":"173f483c-deee-41f2-9a34-e2b6aed083b0"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:35.068114604Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1334,"client":"1","":"downstairs","session_id":"173f483c-deee-41f2-9a34-e2b6aed083b0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:35.068145232Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1334,"client":"2","":"downstairs","session_id":"173f483c-deee-41f2-9a34-e2b6aed083b0"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:35.06817926Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1334,"client":"2","":"downstairs","session_id":"173f483c-deee-41f2-9a34-e2b6aed083b0"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:35.068210717Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1334,"":"io task","client":"2","":"downstairs","session_id":"173f483c-deee-41f2-9a34-e2b6aed083b0"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:35.068243805Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1334,"":"io task","client":"0","":"downstairs","session_id":"173f483c-deee-41f2-9a34-e2b6aed083b0"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:35.068278783Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1334,"":"io task","client":"1","":"downstairs","session_id":"173f483c-deee-41f2-9a34-e2b6aed083b0"} {"msg":"ds_connection connected from Ok(127.0.0.1:36444)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:35.068444551Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1334,"":"io task","client":"2","":"downstairs","session_id":"173f483c-deee-41f2-9a34-e2b6aed083b0"} {"msg":"ds_connection connected from Ok(127.0.0.1:35815)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:35.068489508Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1334,"":"io task","client":"0","":"downstairs","session_id":"173f483c-deee-41f2-9a34-e2b6aed083b0"} {"msg":"ds_connection connected from Ok(127.0.0.1:54407)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:35.068512166Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1334,"":"io task","client":"1","":"downstairs","session_id":"173f483c-deee-41f2-9a34-e2b6aed083b0"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:35.069060459Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1334,"client":"2","":"downstairs","session_id":"173f483c-deee-41f2-9a34-e2b6aed083b0"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:35.069088377Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1334,"client":"1","":"downstairs","session_id":"173f483c-deee-41f2-9a34-e2b6aed083b0"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:35.069103086Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1334,"client":"0","":"downstairs","session_id":"173f483c-deee-41f2-9a34-e2b6aed083b0"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:35.069730982Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1334,"client":"1","":"downstairs","session_id":"173f483c-deee-41f2-9a34-e2b6aed083b0"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:35.0697594Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1334,"client":"2","":"downstairs","session_id":"173f483c-deee-41f2-9a34-e2b6aed083b0"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:35.069780529Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1334,"client":"0","":"downstairs","session_id":"173f483c-deee-41f2-9a34-e2b6aed083b0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:35.070289734Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1334,"session_id":"173f483c-deee-41f2-9a34-e2b6aed083b0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:35.070411055Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1334,"session_id":"173f483c-deee-41f2-9a34-e2b6aed083b0"} {"msg":"[0]R flush_numbers[0..12]: [75, 75, 75, 70, 73, 52, 69, 73, 51, 51, 61, 71]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:35.070485121Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1334,"":"downstairs","":"downstairs","session_id":"173f483c-deee-41f2-9a34-e2b6aed083b0"} {"msg":"[0]R generation[0..12]: [37, 37, 37, 35, 36, 30, 34, 36, 29, 29, 32, 36]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:35.070506319Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1334,"":"downstairs","":"downstairs","session_id":"173f483c-deee-41f2-9a34-e2b6aed083b0"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:35.070521578Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1334,"":"downstairs","":"downstairs","session_id":"173f483c-deee-41f2-9a34-e2b6aed083b0"} {"msg":"[1]R flush_numbers[0..12]: [75, 75, 75, 70, 73, 52, 69, 73, 51, 51, 61, 71]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:35.070535797Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1334,"":"downstairs","":"downstairs","session_id":"173f483c-deee-41f2-9a34-e2b6aed083b0"} {"msg":"[1]R generation[0..12]: [37, 37, 37, 35, 36, 30, 34, 36, 29, 29, 32, 36]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:35.070549936Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1334,"":"downstairs","":"downstairs","session_id":"173f483c-deee-41f2-9a34-e2b6aed083b0"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:35.070563965Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1334,"":"downstairs","":"downstairs","session_id":"173f483c-deee-41f2-9a34-e2b6aed083b0"} {"msg":"[2]R flush_numbers[0..12]: [75, 75, 75, 70, 73, 52, 69, 73, 51, 51, 61, 71]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:35.070577384Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1334,"":"downstairs","":"downstairs","session_id":"173f483c-deee-41f2-9a34-e2b6aed083b0"} {"msg":"[2]R generation[0..12]: [37, 37, 37, 35, 36, 30, 34, 36, 29, 29, 32, 36]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:35.070591053Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1334,"":"downstairs","":"downstairs","session_id":"173f483c-deee-41f2-9a34-e2b6aed083b0"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:35.070604972Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1334,"":"downstairs","":"downstairs","session_id":"173f483c-deee-41f2-9a34-e2b6aed083b0"} {"msg":"Max found gen is 38","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:35.070618071Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1334,"":"downstairs","":"downstairs","session_id":"173f483c-deee-41f2-9a34-e2b6aed083b0"} {"msg":"Generation requested: 38 >= found:38","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:35.07063123Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1334,"":"downstairs","":"downstairs","session_id":"173f483c-deee-41f2-9a34-e2b6aed083b0"} {"msg":"Next flush: 76","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:35.070647339Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1334,"":"downstairs","":"downstairs","session_id":"173f483c-deee-41f2-9a34-e2b6aed083b0"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:35.070660738Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1334,"":"downstairs","":"downstairs","session_id":"173f483c-deee-41f2-9a34-e2b6aed083b0"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-12-12T20:34:35.070683217Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1334,"session_id":"173f483c-deee-41f2-9a34-e2b6aed083b0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:35.070708105Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1334,"client":"0","":"downstairs","session_id":"173f483c-deee-41f2-9a34-e2b6aed083b0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:35.070732304Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1334,"client":"1","":"downstairs","session_id":"173f483c-deee-41f2-9a34-e2b6aed083b0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:35.070759911Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1334,"client":"2","":"downstairs","session_id":"173f483c-deee-41f2-9a34-e2b6aed083b0"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:35.07078782Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1334,"session_id":"173f483c-deee-41f2-9a34-e2b6aed083b0"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:35.070813868Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1334,"session_id":"173f483c-deee-41f2-9a34-e2b6aed083b0"} {"msg":"eef9a1d9-fef0-408b-b955-d41c0f733d8a is now active with session: 173f483c-deee-41f2-9a34-e2b6aed083b0","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:35.070841676Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1334,"session_id":"173f483c-deee-41f2-9a34-e2b6aed083b0"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:35.070867944Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1334,"session_id":"173f483c-deee-41f2-9a34-e2b6aed083b0"} {"msg":"The guest has finished waiting for activation with:38","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:35.070893892Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1334} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 579 len 36864 data: 4 3 3 4 4 4 3 3 2 02/30 Read block 442 len 24576 03/30 Read block 335 len 12288 04/30 Read block 80 len 16384 05/30 Read block 36 len 20480 06/30 Read block 549 len 28672 07/30 Write block 376 len 40960 data: 5 4 6 7 7 6 7 6 5 6 08/30 Flush 09/30 Write block 366 len 4096 data: 4 10/30 Read block 343 len 36864 11/30 Read block 539 len 16384 12/30 Read block 65 len 24576 13/30 Write block 547 len 32768 data: 4 4 4 4 6 6 7 8 14/30 Read block 509 len 32768 15/30 Write block 393 len 40960 data: 5 6 5 4 4 5 5 3 4 6 16/30 Write block 296 len 24576 data: 5 5 4 3 3 3 17/30 Write block 170 len 4096 data: 2 18/30 Flush 19/30 Write block 222 len 24576 data: 2 2 3 4 5 6 20/30 Flush 21/30 Write block 120 len 16384 data: 3 3 3 3 22/30 Write block 44 len 28672 data: 5 5 4 4 5 6 6 23/30 Read block 251 len 28672 24/30 Write block 179 len 4096 data: 5 25/30 Write block 313 len 32768 data: 8 9 10 10 9 8 4 4 26/30 Write block 262 len 28672 data: 3 4 4 7 8 7 7 27/30 Write block 59 len 4096 data: 7 28/30 Write block 229 len 36864 data: 6 7 6 5 5 5 5 4 3 29/30 Read block 49 len 12288 30/30 Write block 317 len 32768 data: 10 9 5 5 4 5 4 2 ---------------------------------------------------------------- Crucible gen:38 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1020 Acked Write 4 Done Done Done false 1021 Acked Write 7 Done Done Done false 1022 Acked Read 7 Done Done Done false 1023 Acked Write 1 Done Done Done false 1024 Acked Write 8 Done Done Done false 1025 Acked Write 7 Done Done Done false 1026 Acked Write 1 Done Done Done false 1027 Acked Write 9 Done Done Done false 1028 Acked Read 3 Done Done Done false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 9 9 9 27 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1019 1019 1019 Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:34:35.091301785Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1334} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 37 37 37 75 75 75 F F F 1 020-039 37 37 37 75 75 75 F F F 2 040-059 37 37 37 75 75 75 T T T 3 060-079 35 35 35 70 70 70 F F F 4 080-099 36 36 36 73 73 73 F F F 5 100-119 30 30 30 52 52 52 F F F 6 120-139 34 34 34 69 69 69 T T T 7 140-159 36 36 36 73 73 73 F F F 8 160-179 38 38 38 77 77 77 T T T 9 180-199 29 29 29 51 51 51 F F F 10 200-219 32 32 32 61 61 61 F F F 11 220-239 38 38 38 78 78 78 T T T 12 240-259 36 36 36 71 71 71 F F F 13 260-279 32 32 32 60 60 60 T T T 14 280-299 38 38 38 77 77 77 F F F 15 300-319 38 38 38 77 77 77 T T T 16 320-339 34 34 34 66 66 66 T T T 17 340-359 36 36 36 74 74 74 F F F 18 360-379 38 38 38 77 77 77 F F F 19 380-399 38 38 38 77 77 77 F F F 20 400-419 38 38 38 77 77 77 F F F 21 420-439 25 25 25 45 45 45 F F F 22 440-459 37 37 37 75 75 75 F F F 23 460-479 37 37 37 75 75 75 F F F 24 480-499 12 12 12 20 20 20 F F F 25 500-519 35 35 35 70 70 70 F F F 26 520-539 32 32 32 60 60 60 F F F 27 540-559 38 38 38 77 77 77 F F F 28 560-579 38 38 38 76 76 76 F F F 29 580-599 38 38 38 76 76 76 F F F Max gen: 38, Max flush: 78 A Difference in extent metadata is expected here On loop 19 Verifying data now No change after verify Loop: 19 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 37 37 37 75 75 75 F F F 1 020-039 37 37 37 75 75 75 F F F 2 040-059 39 39 39 79 79 79 F F F 3 060-079 35 35 35 70 70 70 F F F 4 080-099 36 36 36 73 73 73 F F F 5 100-119 30 30 30 52 52 52 F F F 6 120-139 39 39 39 79 79 79 F F F 7 140-159 36 36 36 73 73 73 F F F 8 160-179 39 39 39 79 79 79 F F F 9 180-199 29 29 29 51 51 51 F F F 10 200-219 32 32 32 61 61 61 F F F 11 220-239 39 39 39 79 79 79 F F F 12 240-259 36 36 36 71 71 71 F F F 13 260-279 39 39 39 79 79 79 F F F 14 280-299 38 38 38 77 77 77 F F F 15 300-319 39 39 39 79 79 79 F F F 16 320-339 39 39 39 79 79 79 F F F 17 340-359 36 36 36 74 74 74 F F F 18 360-379 38 38 38 77 77 77 F F F 19 380-399 38 38 38 77 77 77 F F F 20 400-419 38 38 38 77 77 77 F F F 21 420-439 25 25 25 45 45 45 F F F 22 440-459 37 37 37 75 75 75 F F F 23 460-479 37 37 37 75 75 75 F F F 24 480-499 12 12 12 20 20 20 F F F 25 500-519 35 35 35 70 70 70 F F F 26 520-539 32 32 32 60 60 60 F F F 27 540-559 38 38 38 77 77 77 F F F 28 560-579 38 38 38 76 76 76 F F F 29 580-599 38 38 38 76 76 76 F F F Max gen: 39, Max flush: 79 Begin loop 20 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:39.37656426Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1342,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:34:39.419763931Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1342,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:39.420174703Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1342,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:39.451892025Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1342,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:39.452473005Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1342} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"26e7d501b33f47379e53601720353c2983e7654b\",\n git_commit_timestamp: \"2024-12-12T20:16:40.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:39.452507433Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1342} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:39.452520942Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1342} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:39.452535341Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1342} {"msg":"Crucible 2aa23e8a-c765-49a4-b9a9-6be962b3d966 has session id: c12ec38d-3d35-4052-9ca2-4c1746451064","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:39.452921165Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1342,"session_id":"c12ec38d-3d35-4052-9ca2-4c1746451064"} {"msg":"Upstairs opts: Upstairs UUID: 2aa23e8a-c765-49a4-b9a9-6be962b3d966, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:39.452963882Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1342,"session_id":"c12ec38d-3d35-4052-9ca2-4c1746451064"} {"msg":"Crucible stats registered with UUID: 2aa23e8a-c765-49a4-b9a9-6be962b3d966","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:39.452993469Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1342,"session_id":"c12ec38d-3d35-4052-9ca2-4c1746451064"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:39.484544614Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1342,"task":"crutest"} {"msg":"The guest has requested activation with gen:40","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:39.484576582Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1342} {"msg":"2aa23e8a-c765-49a4-b9a9-6be962b3d966 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:39.484652357Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1342,"session_id":"c12ec38d-3d35-4052-9ca2-4c1746451064"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:39.484673055Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1342,"client":"0","":"downstairs","session_id":"c12ec38d-3d35-4052-9ca2-4c1746451064"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:39.484700953Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1342,"client":"0","":"downstairs","session_id":"c12ec38d-3d35-4052-9ca2-4c1746451064"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:39.484718292Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1342,"client":"1","":"downstairs","session_id":"c12ec38d-3d35-4052-9ca2-4c1746451064"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:39.484735901Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1342,"client":"1","":"downstairs","session_id":"c12ec38d-3d35-4052-9ca2-4c1746451064"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:39.484752199Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1342,"client":"2","":"downstairs","session_id":"c12ec38d-3d35-4052-9ca2-4c1746451064"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:39.484769848Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1342,"client":"2","":"downstairs","session_id":"c12ec38d-3d35-4052-9ca2-4c1746451064"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:39.484786017Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1342,"":"io task","client":"2","":"downstairs","session_id":"c12ec38d-3d35-4052-9ca2-4c1746451064"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:39.484809446Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1342,"":"io task","client":"0","":"downstairs","session_id":"c12ec38d-3d35-4052-9ca2-4c1746451064"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:39.484841424Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1342,"":"io task","client":"1","":"downstairs","session_id":"c12ec38d-3d35-4052-9ca2-4c1746451064"} {"msg":"ds_connection connected from Ok(127.0.0.1:37703)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:39.484877051Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1342,"":"io task","client":"0","":"downstairs","session_id":"c12ec38d-3d35-4052-9ca2-4c1746451064"} {"msg":"ds_connection connected from Ok(127.0.0.1:53960)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:39.484911979Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1342,"":"io task","client":"2","":"downstairs","session_id":"c12ec38d-3d35-4052-9ca2-4c1746451064"} {"msg":"ds_connection connected from Ok(127.0.0.1:36744)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:39.484953156Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1342,"":"io task","client":"1","":"downstairs","session_id":"c12ec38d-3d35-4052-9ca2-4c1746451064"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:39.485685525Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1342,"client":"1","":"downstairs","session_id":"c12ec38d-3d35-4052-9ca2-4c1746451064"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:39.48576647Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1342,"client":"2","":"downstairs","session_id":"c12ec38d-3d35-4052-9ca2-4c1746451064"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:39.485787488Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1342,"client":"0","":"downstairs","session_id":"c12ec38d-3d35-4052-9ca2-4c1746451064"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:39.48633669Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1342,"client":"1","":"downstairs","session_id":"c12ec38d-3d35-4052-9ca2-4c1746451064"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:39.486416715Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1342,"client":"2","":"downstairs","session_id":"c12ec38d-3d35-4052-9ca2-4c1746451064"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:39.486620051Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1342,"client":"0","":"downstairs","session_id":"c12ec38d-3d35-4052-9ca2-4c1746451064"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:39.486875603Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1342,"session_id":"c12ec38d-3d35-4052-9ca2-4c1746451064"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:39.486943499Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1342,"session_id":"c12ec38d-3d35-4052-9ca2-4c1746451064"} {"msg":"[0]R flush_numbers[0..12]: [75, 75, 79, 70, 73, 52, 79, 73, 79, 51, 61, 79]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:39.487152414Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1342,"":"downstairs","":"downstairs","session_id":"c12ec38d-3d35-4052-9ca2-4c1746451064"} {"msg":"[0]R generation[0..12]: [37, 37, 39, 35, 36, 30, 39, 36, 39, 29, 32, 39]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:39.487181132Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1342,"":"downstairs","":"downstairs","session_id":"c12ec38d-3d35-4052-9ca2-4c1746451064"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:39.487197401Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1342,"":"downstairs","":"downstairs","session_id":"c12ec38d-3d35-4052-9ca2-4c1746451064"} {"msg":"[1]R flush_numbers[0..12]: [75, 75, 79, 70, 73, 52, 79, 73, 79, 51, 61, 79]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:39.4872117Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1342,"":"downstairs","":"downstairs","session_id":"c12ec38d-3d35-4052-9ca2-4c1746451064"} {"msg":"[1]R generation[0..12]: [37, 37, 39, 35, 36, 30, 39, 36, 39, 29, 32, 39]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:39.487225549Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1342,"":"downstairs","":"downstairs","session_id":"c12ec38d-3d35-4052-9ca2-4c1746451064"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:39.487243298Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1342,"":"downstairs","":"downstairs","session_id":"c12ec38d-3d35-4052-9ca2-4c1746451064"} {"msg":"[2]R flush_numbers[0..12]: [75, 75, 79, 70, 73, 52, 79, 73, 79, 51, 61, 79]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:39.487266906Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1342,"":"downstairs","":"downstairs","session_id":"c12ec38d-3d35-4052-9ca2-4c1746451064"} {"msg":"[2]R generation[0..12]: [37, 37, 39, 35, 36, 30, 39, 36, 39, 29, 32, 39]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:39.487311763Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1342,"":"downstairs","":"downstairs","session_id":"c12ec38d-3d35-4052-9ca2-4c1746451064"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:39.487338441Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1342,"":"downstairs","":"downstairs","session_id":"c12ec38d-3d35-4052-9ca2-4c1746451064"} {"msg":"Max found gen is 40","v":0,"name":"crucible","level":30,Wait for a query_work_queue command to finish before sending IO "time":"2024-12-12T20:34:39.487362579Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1342,"":"downstairs","":"downstairs","session_id":"c12ec38d-3d35-4052-9ca2-4c1746451064"} {"msg":"Generation requested: 40 >= found:40","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:39.487392717Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1342,"":"downstairs","":"downstairs","session_id":"c12ec38d-3d35-4052-9ca2-4c1746451064"} {"msg":"Next flush: 80","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:39.487419536Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1342,"":"downstairs","":"downstairs","session_id":"c12ec38d-3d35-4052-9ca2-4c1746451064"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:39.487435085Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1342,"":"downstairs","":"downstairs","session_id":"c12ec38d-3d35-4052-9ca2-4c1746451064"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:39.487448674Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1342,"session_id":"c12ec38d-3d35-4052-9ca2-4c1746451064"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:39.487469092Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1342,"client":"0","":"downstairs","session_id":"c12ec38d-3d35-4052-9ca2-4c1746451064"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:39.487494621Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1342,"client":"1","":"downstairs","session_id":"c12ec38d-3d35-4052-9ca2-4c1746451064"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:39.487520649Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1342,"client":"2","":"downstairs","session_id":"c12ec38d-3d35-4052-9ca2-4c1746451064"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:39.487537627Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1342,"session_id":"c12ec38d-3d35-4052-9ca2-4c1746451064"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:39.487552757Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1342,"session_id":"c12ec38d-3d35-4052-9ca2-4c1746451064"} {"msg":"2aa23e8a-c765-49a4-b9a9-6be962b3d966 is now active with session: c12ec38d-3d35-4052-9ca2-4c1746451064","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:39.487576255Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1342,"session_id":"c12ec38d-3d35-4052-9ca2-4c1746451064"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:39.487603503Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1342,"session_id":"c12ec38d-3d35-4052-9ca2-4c1746451064"} {"msg":"The guest has finished waiting for activation with:40","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:39.487626451Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1342} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 501 len 8192 data: 4 5 02/30 Write block 555 len 8192 data: 8 6 03/30 Read block 330 len 32768 04/30 Read block 424 len 36864 05/30 Read block 520 len 36864 06/30 Read block 255 len 8192 07/30 Write block 217 len 16384 data: 4 2 3 3 08/30 Write block 523 len 20480 data: 8 7 5 3 3 09/30 Read block 10 len 16384 10/30 Read block 546 len 24576 11/30 Write block 187 len 28672 data: 6 7 6 6 6 6 6 12/30 Read block 130 len 28672 13/30 Write block 69 len 12288 data: 3 3 3 14/30 Read block 379 len 16384 15/30 Read block 120 len 24576 16/30 Write block 584 len 40960 data: 5 4 4 3 3 3 4 3 3 3 17/30 Write block 77 len 4096 data: 4 18/30 Read block 222 len 16384 19/30 Write block 284 len 24576 data: 3 4 4 3 3 3 20/30 Read block 312 len 8192 21/30 Read block 78 len 12288 22/30 Read block 532 len 32768 23/30 Write block 494 len 32768 data: 2 2 2 2 2 2 3 5 24/30 Read block 113 len 8192 25/30 Read block 457 len 40960 26/30 Flush 27/30 Write block 346 len 16384 data: 4 4 4 3 28/30 Write block 33 len 40960 data: 7 7 8 7 7 8 10 8 7 6 29/30 Read block 179 len 16384 30/30 Read block 390 len 40960 ---------------------------------------------------------------- Crucible gen:40 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 4 Done Done Done false 1027 Acked Write 10 Done Done Done false 1028 Acked Read 4 Done Done Done false 1029 Acked Read 10 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:34:39.507505491Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1342} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 37 37 37 75 75 75 F F F 1 020-039 37 37 37 75 75 75 T T T 2 040-059 39 39 39 79 79 79 T T T 3 060-079 40 40 40 80 80 80 F F F 4 080-099 36 36 36 73 73 73 F F F 5 100-119 30 30 30 52 52 52 F F F 6 120-139 39 39 39 79 79 79 F F F 7 140-159 36 36 36 73 73 73 F F F 8 160-179 39 39 39 79 79 79 F F F 9 180-199 40 40 40 80 80 80 F F F 10 200-219 40 40 40 80 80 80 F F F 11 220-239 40 40 40 80 80 80 F F F 12 240-259 36 36 36 71 71 71 F F F 13 260-279 39 39 39 79 79 79 F F F 14 280-299 40 40 40 80 80 80 F F F 15 300-319 39 39 39 79 79 79 F F F 16 320-339 39 39 39 79 79 79 F F F 17 340-359 36 36 36 74 74 74 T T T 18 360-379 38 38 38 77 77 77 F F F 19 380-399 38 38 38 77 77 77 F F F 20 400-419 38 38 38 77 77 77 F F F 21 420-439 25 25 25 45 45 45 F F F 22 440-459 37 37 37 75 75 75 F F F 23 460-479 37 37 37 75 75 75 F F F 24 480-499 40 40 40 80 80 80 F F F 25 500-519 40 40 40 80 80 80 F F F 26 520-539 40 40 40 80 80 80 F F F 27 540-559 40 40 40 80 80 80 F F F 28 560-579 38 38 38 76 76 76 F F F 29 580-599 40 40 40 80 80 80 F F F Max gen: 40, Max flush: 80 A Difference in extent metadata is expected here On loop 20 Verifying data now No change after verify Loop: 20 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 37 37 37 75 75 75 F F F 1 020-039 41 41 41 81 81 81 F F F 2 040-059 41 41 41 81 81 81 F F F 3 060-079 40 40 40 80 80 80 F F F 4 080-099 36 36 36 73 73 73 F F F 5 100-119 30 30 30 52 52 52 F F F 6 120-139 39 39 39 79 79 79 F F F 7 140-159 36 36 36 73 73 73 F F F 8 160-179 39 39 39 79 79 79 F F F 9 180-199 40 40 40 80 80 80 F F F 10 200-219 40 40 40 80 80 80 F F F 11 220-239 40 40 40 80 80 80 F F F 12 240-259 36 36 36 71 71 71 F F F 13 260-279 39 39 39 79 79 79 F F F 14 280-299 40 40 40 80 80 80 F F F 15 300-319 39 39 39 79 79 79 F F F 16 320-339 39 39 39 79 79 79 F F F 17 340-359 41 41 41 81 81 81 F F F 18 360-379 38 38 38 77 77 77 F F F 19 380-399 38 38 38 77 77 77 F F F 20 400-419 38 38 38 77 77 77 F F F 21 420-439 25 25 25 45 45 45 F F F 22 440-459 37 37 37 75 75 75 F F F 23 460-479 37 37 37 75 75 75 F F F 24 480-499 40 40 40 80 80 80 F F F 25 500-519 40 40 40 80 80 80 F F F 26 520-539 40 40 40 80 80 80 F F F 27 540-559 40 40 40 80 80 80 F F F 28 560-579 38 38 38 76 76 76 F F F 29 580-599 40 40 40 80 80 80 F F F Max gen: 41, Max flush: 81 Begin loop 21 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:43.803945725Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1350,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:43.848429678Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1350,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:43.849139479Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1350} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"26e7d501b33f47379e53601720353c2983e7654b\",\n git_commit_timestamp: \"2024-12-12T20:16:40.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:43.849175626Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1350} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:43.849189785Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1350} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:43.849204184Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1350} {"msg":"Crucible fc9766f5-5313-4f29-8226-f08631dbfbad has session id: d24a3a85-4c0a-41fa-89cb-5cb243066ec9","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:43.849639754Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1350,"session_id":"d24a3a85-4c0a-41fa-89cb-5cb243066ec9"} {"msg":"Upstairs opts: Upstairs UUID: fc9766f5-5313-4f29-8226-f08631dbfbad, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:43.849682871Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1350,"session_id":"d24a3a85-4c0a-41fa-89cb-5cb243066ec9"} {"msg":"Crucible stats registered with UUID: fc9766f5-5313-4f29-8226-f08631dbfbad","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:43.84970748Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1350,"session_id":"d24a3a85-4c0a-41fa-89cb-5cb243066ec9"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:43.880357206Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1350,"task":"crutest"} {"msg":"The guest has requested activation with gen:42","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:43.880393364Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1350} {"msg":"fc9766f5-5313-4f29-8226-f08631dbfbad active request set","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:43.880458619Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1350,"session_id":"d24a3a85-4c0a-41fa-89cb-5cb243066ec9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:43.880496906Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1350,"client":"0","":"downstairs","session_id":"d24a3a85-4c0a-41fa-89cb-5cb243066ec9"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:43.880546203Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1350,"client":"0","":"downstairs","session_id":"d24a3a85-4c0a-41fa-89cb-5cb243066ec9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:43.880582991Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1350,"client":"1","":"downstairs","session_id":"d24a3a85-4c0a-41fa-89cb-5cb243066ec9"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:43.880620818Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1350,"client":"1","":"downstairs","session_id":"d24a3a85-4c0a-41fa-89cb-5cb243066ec9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:43.880655935Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1350,"client":"2","":"downstairs","session_id":"d24a3a85-4c0a-41fa-89cb-5cb243066ec9"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:43.880695883Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1350,"client":"2","":"downstairs","session_id":"d24a3a85-4c0a-41fa-89cb-5cb243066ec9"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:43.88073147Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1350,"":"io task","client":"2","":"downstairs","session_id":"d24a3a85-4c0a-41fa-89cb-5cb243066ec9"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:43.880766298Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1350,"":"io task","client":"0","":"downstairs","session_id":"d24a3a85-4c0a-41fa-89cb-5cb243066ec9"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:43.880801205Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1350,"":"io task","client":"1","":"downstairs","session_id":"d24a3a85-4c0a-41fa-89cb-5cb243066ec9"} {"msg":"ds_connection connected from Ok(127.0.0.1:62747)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:43.880923087Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1350,"":"io task","client":"0","":"downstairs","session_id":"d24a3a85-4c0a-41fa-89cb-5cb243066ec9"} {"msg":"ds_connection connected from Ok(127.0.0.1:33109)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:43.880962654Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1350,"":"io task","client":"2","":"downstairs","session_id":"d24a3a85-4c0a-41fa-89cb-5cb243066ec9"} {"msg":"ds_connection connected from Ok(127.0.0.1:55588)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:43.880986223Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1350,"":"io task","client":"1","":"downstairs","session_id":"d24a3a85-4c0a-41fa-89cb-5cb243066ec9"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:43.881455191Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1350,"client":"0","":"downstairs","session_id":"d24a3a85-4c0a-41fa-89cb-5cb243066ec9"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:43.881485128Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1350,"client":"1","":"downstairs","session_id":"d24a3a85-4c0a-41fa-89cb-5cb243066ec9"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:43.881519646Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1350,"client":"2","":"downstairs","session_id":"d24a3a85-4c0a-41fa-89cb-5cb243066ec9"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:43.882050789Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1350,"client":"0","":"downstairs","session_id":"d24a3a85-4c0a-41fa-89cb-5cb243066ec9"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:43.882100106Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1350,"client":"1","":"downstairs","session_id":"d24a3a85-4c0a-41fa-89cb-5cb243066ec9"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:43.882376337Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1350,"client":"2","":"downstairs","session_id":"d24a3a85-4c0a-41fa-89cb-5cb243066ec9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:43.882641419Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1350,"session_id":"d24a3a85-4c0a-41fa-89cb-5cb243066ec9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:43.882724943Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1350,"session_id":"d24a3a85-4c0a-41fa-89cb-5cb243066ec9"} {"msg":"[0]R flush_numbers[0..12]: [75, 81, 81, 80, 73, 52, 79, 73, 79, 80, 80, 80]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:43.882958307Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1350,"":"downstairs","":"downstairs","session_id":"d24a3a85-4c0a-41fa-89cb-5cb243066ec9"} {"msg":"[0]R generation[0..12]: [37, 41, 41, 40, 36, 30, 39, 36, 39, 40, 40, 40]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:43.882978645Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1350,"":"downstairs","":"downstairs","session_id":"d24a3a85-4c0a-41fa-89cb-5cb243066ec9"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:43.882993564Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1350,"":"downstairs","":"downstairs","session_id":"d24a3a85-4c0a-41fa-89cb-5cb243066ec9"} {"msg":"[1]R flush_numbers[0..12]: [75, 81, 81, 80, 73, 52, 79, 73, 79, 80, 80, 80]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:43.883011183Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1350,"":"downstairs","":"downstairs","session_id":"d24a3a85-4c0a-41fa-89cb-5cb243066ec9"} {"msg":"[1]R generation[0..12]: [37, 41, 41, 40, 36, 30, 39, 36, 39, 40, 40, 40]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:43.883025142Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1350,"":"downstairs","":"downstairs","session_id":"d24a3a85-4c0a-41fa-89cb-5cb243066ec9"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:43.883039031Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1350,"":"downstairs","":"downstairs","session_id":"d24a3a85-4c0a-41fa-89cb-5cb243066ec9"} {"msg":"[2]R flush_numbers[0..12]: [75, 81, 81, 80, 73, 52, 79, 73, 79, 80, 80, 80]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:43.88305245Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1350,"":"downstairs","":"downstairs","session_id":"d24a3a85-4c0a-41fa-89cb-5cb243066ec9"} {"msg":"[2]R generation[0..12]: [37, 41, 41, 40, 36, 30, 39, 36, 39, 40, 40, 40]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:43.883065789Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1350,"":"downstairs","":"downstairs","session_id":"d24a3a85-4c0a-41fa-89cb-5cb243066ec9"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:43.883079608Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1350,"":"downstairs","":"downstairs","session_id":"d24a3a85-4c0a-41fa-89cb-5cb243066ec9"} {"msg":"Max found gen is 42","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:43.883092458Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1350,"":"downstairs","":"downstairs","session_id":"d24a3a85-4c0a-41fa-89cb-5cb243066ec9"} {"msg":"Generation requested: 42 >= found:42","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:43.883105447Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1350,"":"downstairs","":"downstairs","session_id":"d24a3a85-4c0a-41fa-89cb-5cb243066ec9"} {"msg":"Next flush: 82","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:43.883118106Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1350,"":"downstairs","":"downstairs","session_id":"d24a3a85-4c0a-41fa-89cb-5cb243066ec9"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:43.883130775Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1350,"":"downstairs","":"downstairs","session_id":"d24a3a85-4c0a-41fa-89cb-5cb243066ec9"} {Wait for a query_work_queue command to finish before sending IO "msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:43.883148124Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1350,"session_id":"d24a3a85-4c0a-41fa-89cb-5cb243066ec9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:43.883160263Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1350,"client":"0","":"downstairs","session_id":"d24a3a85-4c0a-41fa-89cb-5cb243066ec9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:43.883173282Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1350,"client":"1","":"downstairs","session_id":"d24a3a85-4c0a-41fa-89cb-5cb243066ec9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:43.883186151Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1350,"client":"2","":"downstairs","session_id":"d24a3a85-4c0a-41fa-89cb-5cb243066ec9"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:43.88319923Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1350,"session_id":"d24a3a85-4c0a-41fa-89cb-5cb243066ec9"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:43.883213349Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1350,"session_id":"d24a3a85-4c0a-41fa-89cb-5cb243066ec9"} {"msg":"fc9766f5-5313-4f29-8226-f08631dbfbad is now active with session: d24a3a85-4c0a-41fa-89cb-5cb243066ec9","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:43.883226148Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1350,"session_id":"d24a3a85-4c0a-41fa-89cb-5cb243066ec9"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:43.883237877Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1350,"session_id":"d24a3a85-4c0a-41fa-89cb-5cb243066ec9"} {"msg":"The guest has finished waiting for activation with:42","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:43.883249867Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1350} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 410 len 28672 data: 3 2 2 2 2 2 3 02/30 Write block 429 len 8192 data: 3 2 03/30 Read block 196 len 36864 04/30 Read block 272 len 12288 05/30 Write block 69 len 20480 data: 4 4 4 5 5 06/30 Read block 19 len 12288 07/30 Write block 558 len 8192 data: 5 6 08/30 Read block 222 len 12288 09/30 Write block 133 len 24576 data: 2 2 2 4 4 4 10/30 Read block 53 len 12288 11/30 Read block 505 len 4096 12/30 Read block 135 len 28672 13/30 Read block 261 len 16384 14/30 Read block 390 len 40960 15/30 Read block 113 len 8192 16/30 Flush 17/30 Write block 409 len 32768 data: 4 4 3 3 3 3 3 4 18/30 Write block 484 len 12288 data: 4 5 5 19/30 Read block 262 len 32768 20/30 Read block 102 len 28672 21/30 Flush 22/30 Write block 451 len 20480 data: 5 5 5 5 4 23/30 Flush 24/30 Write block 350 len 24576 data: 4 2 3 4 5 5 25/30 Read block 186 len 36864 26/30 Read block 241 len 8192 27/30 Write block 56 len 20480 data: 7 7 8 8 6 28/30 Read block 423 len 36864 29/30 Write block 536 len 28672 data: 4 4 3 3 5 3 4 30/30 Read block 123 len 28672 ---------------------------------------------------------------- Crucible gen:42 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 6 Done Done Done false 1024 Acked Read 9 Done Done Done false 1025 Acked Read 2 Done Done Done false 1026 Acked Write 5 Done Done Done false 1027 Acked Read 9 Done Done Done false 1028 Acked Write 7 Sent Done Done false 1029 Acked Read 7 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 0 1 3 Done 5 7 6 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:34:43.903284485Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1350} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 37 37 37 75 75 75 F F F 1 020-039 41 41 41 81 81 81 F F F 2 040-059 41 41 41 81 81 81 T T T 3 060-079 42 42 42 82 82 82 T T T 4 080-099 36 36 36 73 73 73 F F F 5 100-119 30 30 30 52 52 52 F F F 6 120-139 42 42 42 82 82 82 F F F 7 140-159 36 36 36 73 73 73 F F F 8 160-179 39 39 39 79 79 79 F F F 9 180-199 40 40 40 80 80 80 F F F 10 200-219 40 40 40 80 80 80 F F F 11 220-239 40 40 40 80 80 80 F F F 12 240-259 36 36 36 71 71 71 F F F 13 260-279 39 39 39 79 79 79 F F F 14 280-299 40 40 40 80 80 80 F F F 15 300-319 39 39 39 79 79 79 F F F 16 320-339 39 39 39 79 79 79 F F F 17 340-359 41 41 41 81 81 81 T T T 18 360-379 38 38 38 77 77 77 F F F 19 380-399 38 38 38 77 77 77 F F F 20 400-419 42 42 42 83 83 83 F F F 21 420-439 42 42 42 82 82 82 F F F 22 440-459 42 42 42 84 84 84 F F F 23 460-479 37 37 37 75 75 75 F F F 24 480-499 42 42 42 83 83 83 F F F 25 500-519 40 40 40 80 80 80 F F F 26 520-539 40 40 40 80 80 80 T T T 27 540-559 42 42 42 82 82 82 T T T 28 560-579 38 38 38 76 76 76 F F F 29 580-599 40 40 40 80 80 80 F F F Max gen: 42, Max flush: 84 A Difference in extent metadata is expected here On loop 21 Verifying data now No change after verify Loop: 21 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 37 37 37 75 75 75 F F F 1 020-039 41 41 41 81 81 81 F F F 2 040-059 43 43 43 85 85 85 F F F 3 060-079 43 43 43 85 85 85 F F F 4 080-099 36 36 36 73 73 73 F F F 5 100-119 30 30 30 52 52 52 F F F 6 120-139 42 42 42 82 82 82 F F F 7 140-159 36 36 36 73 73 73 F F F 8 160-179 39 39 39 79 79 79 F F F 9 180-199 40 40 40 80 80 80 F F F 10 200-219 40 40 40 80 80 80 F F F 11 220-239 40 40 40 80 80 80 F F F 12 240-259 36 36 36 71 71 71 F F F 13 260-279 39 39 39 79 79 79 F F F 14 280-299 40 40 40 80 80 80 F F F 15 300-319 39 39 39 79 79 79 F F F 16 320-339 39 39 39 79 79 79 F F F 17 340-359 43 43 43 85 85 85 F F F 18 360-379 38 38 38 77 77 77 F F F 19 380-399 38 38 38 77 77 77 F F F 20 400-419 42 42 42 83 83 83 F F F 21 420-439 42 42 42 82 82 82 F F F 22 440-459 42 42 42 84 84 84 F F F 23 460-479 37 37 37 75 75 75 F F F 24 480-499 42 42 42 83 83 83 F F F 25 500-519 40 40 40 80 80 80 F F F 26 520-539 43 43 43 85 85 85 F F F 27 540-559 43 43 43 85 85 85 F F F 28 560-579 38 38 38 76 76 76 F F F 29 580-599 40 40 40 80 80 80 F F F Max gen: 43, Max flush: 85 Begin loop 22 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:48.177744005Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1358,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:48.222176481Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1358,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:48.222894152Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1358} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"26e7d501b33f47379e53601720353c2983e7654b\",\n git_commit_timestamp: \"2024-12-12T20:16:40.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:48.22293074Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1358} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:48.222949158Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1358} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:48.222964017Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1358} {"msg":"Crucible a3c66af7-15f2-44b3-bdba-3611c718e890 has session id: 5ccca124-2b36-4b48-96d2-2e2e809a64d8","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:48.223399817Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1358,"session_id":"5ccca124-2b36-4b48-96d2-2e2e809a64d8"} {"msg":"Upstairs opts: Upstairs UUID: a3c66af7-15f2-44b3-bdba-3611c718e890, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:48.223445394Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1358,"session_id":"5ccca124-2b36-4b48-96d2-2e2e809a64d8"} {"msg":"Crucible stats registered with UUID: a3c66af7-15f2-44b3-bdba-3611c718e890","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:48.223470082Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1358,"session_id":"5ccca124-2b36-4b48-96d2-2e2e809a64d8"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:48.254140097Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1358,"task":"crutest"} {"msg":"The guest has requested activation with gen:44","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:48.254174575Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1358} {"msg":"a3c66af7-15f2-44b3-bdba-3611c718e890 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:48.2542435Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1358,"session_id":"5ccca124-2b36-4b48-96d2-2e2e809a64d8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:48.254266739Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1358,"client":"0","":"downstairs","session_id":"5ccca124-2b36-4b48-96d2-2e2e809a64d8"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:48.254294847Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1358,"client":"0","":"downstairs","session_id":"5ccca124-2b36-4b48-96d2-2e2e809a64d8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:48.254312485Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1358,"client":"1","":"downstairs","session_id":"5ccca124-2b36-4b48-96d2-2e2e809a64d8"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:48.254340923Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1358,"client":"1","":"downstairs","session_id":"5ccca124-2b36-4b48-96d2-2e2e809a64d8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:48.254373851Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1358,"client":"2","":"downstairs","session_id":"5ccca124-2b36-4b48-96d2-2e2e809a64d8"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:48.254408779Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1358,"client":"2","":"downstairs","session_id":"5ccca124-2b36-4b48-96d2-2e2e809a64d8"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:48.254448356Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1358,"":"io task","client":"2","":"downstairs","session_id":"5ccca124-2b36-4b48-96d2-2e2e809a64d8"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:48.254482134Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1358,"":"io task","client":"0","":"downstairs","session_id":"5ccca124-2b36-4b48-96d2-2e2e809a64d8"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:48.254513252Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1358,"":"io task","client":"1","":"downstairs","session_id":"5ccca124-2b36-4b48-96d2-2e2e809a64d8"} {"msg":"ds_connection connected from Ok(127.0.0.1:63645)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:48.254651662Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1358,"":"io task","client":"0","":"downstairs","session_id":"5ccca124-2b36-4b48-96d2-2e2e809a64d8"} {"msg":"ds_connection connected from Ok(127.0.0.1:58330)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:48.254702379Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1358,"":"io task","client":"2","":"downstairs","session_id":"5ccca124-2b36-4b48-96d2-2e2e809a64d8"} {"msg":"ds_connection connected from Ok(127.0.0.1:55102)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:48.254726357Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1358,"":"io task","client":"1","":"downstairs","session_id":"5ccca124-2b36-4b48-96d2-2e2e809a64d8"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:48.255273139Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1358,"client":"2","":"downstairs","session_id":"5ccca124-2b36-4b48-96d2-2e2e809a64d8"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:48.255301137Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1358,"client":"0","":"downstairs","session_id":"5ccca124-2b36-4b48-96d2-2e2e809a64d8"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:48.255445558Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1358,"client":"1","":"downstairs","session_id":"5ccca124-2b36-4b48-96d2-2e2e809a64d8"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:48.255908775Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1358,"client":"0","":"downstairs","session_id":"5ccca124-2b36-4b48-96d2-2e2e809a64d8"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:48.25598872Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1358,"client":"1","":"downstairs","session_id":"5ccca124-2b36-4b48-96d2-2e2e809a64d8"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:48.256059695Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1358,"client":"2","":"downstairs","session_id":"5ccca124-2b36-4b48-96d2-2e2e809a64d8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:48.256473976Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1358,"session_id":"5ccca124-2b36-4b48-96d2-2e2e809a64d8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:48.256512114Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1358,"session_id":"5ccca124-2b36-4b48-96d2-2e2e809a64d8"} {"msg":"[0]R flush_numbers[0..12]: [75, 81, 85, 85, 73, 52, 82, 73, 79, 80, 80, 80]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:48.256628606Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1358,"":"downstairs","":"downstairs","session_id":"5ccca124-2b36-4b48-96d2-2e2e809a64d8"} {"msg":"[0]R generation[0..12]: [37, 41, 43, 43, 36, 30, 42, 36, 39, 40, 40, 40]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:48.256651904Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1358,"":"downstairs","":"downstairs","session_id":"5ccca124-2b36-4b48-96d2-2e2e809a64d8"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:48.256667213Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1358,"":"downstairs","":"downstairs","session_id":"5ccca124-2b36-4b48-96d2-2e2e809a64d8"} {"msg":"[1]R flush_numbers[0..12]: [75, 81, 85, 85, 73, 52, 82, 73, 79, 80, 80, 80]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:48.256681162Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1358,"":"downstairs","":"downstairs","session_id":"5ccca124-2b36-4b48-96d2-2e2e809a64d8"} {"msg":"[1]R generation[0..12]: [37, 41, 43, 43, 36, 30, 42, 36, 39, 40, 40, 40]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:48.256694741Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1358,"":"downstairs","":"downstairs","session_id":"5ccca124-2b36-4b48-96d2-2e2e809a64d8"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:48.25670934Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1358,"":"downstairs","":"downstairs","session_id":"5ccca124-2b36-4b48-96d2-2e2e809a64d8"} {"msg":"[2]R flush_numbers[0..12]: [75, 81, 85, 85, 73, 52, 82, 73, 79, 80, 80, 80]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:48.256723749Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1358,"":"downstairs","":"downstairs","session_id":"5ccca124-2b36-4b48-96d2-2e2e809a64d8"} {"msg":"[2]R generation[0..12]: [37, 41, 43, 43, 36, 30, 42, 36, 39, 40, 40, 40]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:48.256737558Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1358,"":"downstairs","":"downstairs","session_id":"5ccca124-2b36-4b48-96d2-2e2e809a64d8"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:48.256751617Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1358,"":"downstairs","":"downstairs","session_id":"5ccca124-2b36-4b48-96d2-2e2e809a64d8"} {"msg":"Max found gen is 44","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:48.256764656Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1358,"":"downstairs","":"downstairs","session_id":"5ccca124-2b36-4b48-96d2-2e2e809a64d8"} {"msg":"Generation requested: 44 >= found:44","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:48.256777655Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1358,"":"downstairs","":"downstairs","session_id":"5ccca124-2b36-4b48-96d2-2e2e809a64d8"} {"msg":"Next flush: 86","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:48.256790555Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1358,"":"downstairs","":"downstairs","session_id":"5ccca124-2b36-4b48-96d2-2e2e809a64d8"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-12-12T20:34:48.256803174Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1358,"":"downstairs","":"downstairs","session_id":"5ccca124-2b36-4b48-96d2-2e2e809a64d8"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:48.256819553Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1358,"session_id":"5ccca124-2b36-4b48-96d2-2e2e809a64d8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:48.256831562Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1358,"client":"0","":"downstairs","session_id":"5ccca124-2b36-4b48-96d2-2e2e809a64d8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:48.256848541Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1358,"client":"1","":"downstairs","session_id":"5ccca124-2b36-4b48-96d2-2e2e809a64d8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:48.25686181Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1358,"client":"2","":"downstairs","session_id":"5ccca124-2b36-4b48-96d2-2e2e809a64d8"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:48.256874929Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1358,"session_id":"5ccca124-2b36-4b48-96d2-2e2e809a64d8"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:48.256887288Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1358,"session_id":"5ccca124-2b36-4b48-96d2-2e2e809a64d8"} {"msg":"a3c66af7-15f2-44b3-bdba-3611c718e890 is now active with session: 5ccca124-2b36-4b48-96d2-2e2e809a64d8","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:48.256899907Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1358,"session_id":"5ccca124-2b36-4b48-96d2-2e2e809a64d8"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:48.256912726Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1358,"session_id":"5ccca124-2b36-4b48-96d2-2e2e809a64d8"} {"msg":"The guest has finished waiting for activation with:44","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:48.256924885Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1358} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 9 len 8192 data: 5 5 02/30 Write block 300 len 16384 data: 4 4 3 4 03/30 Read block 567 len 20480 04/30 Flush 05/30 Write block 263 len 12288 data: 5 5 8 06/30 Write block 272 len 16384 data: 7 5 5 4 07/30 Write block 141 len 8192 data: 2 4 08/30 Read block 496 len 16384 09/30 Read block 348 len 28672 10/30 Write block 102 len 4096 data: 3 11/30 Read block 121 len 36864 12/30 Write block 423 len 16384 data: 4 5 5 4 13/30 Write block 417 len 28672 data: 3 5 4 4 4 5 5 14/30 Write block 151 len 16384 data: 3 3 3 4 15/30 Flush 16/30 Write block 213 len 12288 data: 5 4 4 17/30 Read block 36 len 4096 18/30 Flush 19/30 Write block 374 len 4096 data: 5 20/30 Read block 586 len 24576 21/30 Write block 18 len 28672 data: 5 4 4 4 4 3 4 22/30 Read block 36 len 12288 23/30 Write block 103 len 12288 data: 3 3 4 24/30 Write block 531 len 28672 data: 5 5 4 4 4 5 5 25/30 Write block 414 len 36864 data: 4 4 5 4 6 5 5 5 6 26/30 Read block 465 len 4096 27/30 Read block 63 len 4096 28/30 Write block 209 len 28672 data: 4 5 7 6 6 5 5 29/30 Write block 288 len 40960 data: 4 4 4 4 4 6 6 5 6 6 30/30 Write block 30 len 4096 data: 5 ---------------------------------------------------------------- Crucible gen:44 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1018 Acked Write 1 Done Done Done false 1019 Acked Read 6 Done Done Done false 1020 Acked Write 7 Done Done Done false 1021 Acked Read 3 Done Done Done false 1022 Acked Write 3 Done Done Done false 1023 Acked Write 7 Done Done Done false 1024 Acked Write 9 Done Done Done false 1025 Acked Read 1 Done Done Done false 1026 Acked Read 1 Done Done Done false 1027 Acked Write 7 Done Done Done false 1028 Acked Write 10 Sent Sent Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 10 10 10 30 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1017 1017 1017 Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:34:48.273579687Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1358} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 44 44 44 86 86 86 T T T 1 020-039 41 41 41 81 81 81 T T T 2 040-059 43 43 43 85 85 85 F F F 3 060-079 43 43 43 85 85 85 F F F 4 080-099 36 36 36 73 73 73 F F F 5 100-119 44 44 44 87 87 87 T T T 6 120-139 42 42 42 82 82 82 F F F 7 140-159 44 44 44 87 87 87 F F F 8 160-179 39 39 39 79 79 79 F F F 9 180-199 40 40 40 80 80 80 F F F 10 200-219 44 44 44 88 88 88 T T T 11 220-239 40 40 40 80 80 80 F F F 12 240-259 36 36 36 71 71 71 F F F 13 260-279 44 44 44 87 87 87 F F F 14 280-299 40 40 40 80 80 80 T T T 15 300-319 44 44 44 86 86 86 F F F 16 320-339 39 39 39 79 79 79 F F F 17 340-359 43 43 43 85 85 85 F F F 18 360-379 38 38 38 77 77 77 T T T 19 380-399 38 38 38 77 77 77 F F F 20 400-419 44 44 44 87 87 87 T T T 21 420-439 44 44 44 87 87 87 T T T 22 440-459 42 42 42 84 84 84 F F F 23 460-479 37 37 37 75 75 75 F F F 24 480-499 42 42 42 83 83 83 F F F 25 500-519 40 40 40 80 80 80 F F F 26 520-539 43 43 43 85 85 85 T T T 27 540-559 43 43 43 85 85 85 F F F 28 560-579 38 38 38 76 76 76 F F F 29 580-599 40 40 40 80 80 80 F F F Max gen: 44, Max flush: 88 A Difference in extent metadata is expected here On loop 22 Verifying data now No change after verify Loop: 22 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 45 45 45 89 89 89 F F F 1 020-039 45 45 45 89 89 89 F F F 2 040-059 43 43 43 85 85 85 F F F 3 060-079 43 43 43 85 85 85 F F F 4 080-099 36 36 36 73 73 73 F F F 5 100-119 45 45 45 89 89 89 F F F 6 120-139 42 42 42 82 82 82 F F F 7 140-159 44 44 44 87 87 87 F F F 8 160-179 39 39 39 79 79 79 F F F 9 180-199 40 40 40 80 80 80 F F F 10 200-219 45 45 45 89 89 89 F F F 11 220-239 40 40 40 80 80 80 F F F 12 240-259 36 36 36 71 71 71 F F F 13 260-279 44 44 44 87 87 87 F F F 14 280-299 45 45 45 89 89 89 F F F 15 300-319 44 44 44 86 86 86 F F F 16 320-339 39 39 39 79 79 79 F F F 17 340-359 43 43 43 85 85 85 F F F 18 360-379 45 45 45 89 89 89 F F F 19 380-399 38 38 38 77 77 77 F F F 20 400-419 45 45 45 89 89 89 F F F 21 420-439 45 45 45 89 89 89 F F F 22 440-459 42 42 42 84 84 84 F F F 23 460-479 37 37 37 75 75 75 F F F 24 480-499 42 42 42 83 83 83 F F F 25 500-519 40 40 40 80 80 80 F F F 26 520-539 45 45 45 89 89 89 F F F 27 540-559 43 43 43 85 85 85 F F F 28 560-579 38 38 38 76 76 76 F F F 29 580-599 40 40 40 80 80 80 F F F Max gen: 45, Max flush: 89 Begin loop 23 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:52.560931568Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1366,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:52.605259362Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1366,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:52.605946014Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1366} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"26e7d501b33f47379e53601720353c2983e7654b\",\n git_commit_timestamp: \"2024-12-12T20:16:40.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:52.605981972Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1366} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:52.605997741Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1366} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:52.60601184Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1366} {"msg":"Crucible 628c290f-d648-4d71-b26a-02e6b9da53b7 has session id: 115c57b1-49d0-4e1c-8fe4-90a7279b9ae1","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:52.606506436Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1366,"session_id":"115c57b1-49d0-4e1c-8fe4-90a7279b9ae1"} {"msg":"Upstairs opts: Upstairs UUID: 628c290f-d648-4d71-b26a-02e6b9da53b7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:52.606551183Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1366,"session_id":"115c57b1-49d0-4e1c-8fe4-90a7279b9ae1"} {"msg":"Crucible stats registered with UUID: 628c290f-d648-4d71-b26a-02e6b9da53b7","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:52.606575091Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1366,"session_id":"115c57b1-49d0-4e1c-8fe4-90a7279b9ae1"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:52.638148194Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1366,"task":"crutest"} {"msg":"The guest has requested activation with gen:46","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:52.638174732Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1366} {"msg":"628c290f-d648-4d71-b26a-02e6b9da53b7 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:52.638257996Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1366,"session_id":"115c57b1-49d0-4e1c-8fe4-90a7279b9ae1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:52.638276455Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1366,"client":"0","":"downstairs","session_id":"115c57b1-49d0-4e1c-8fe4-90a7279b9ae1"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:52.638299283Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1366,"client":"0","":"downstairs","session_id":"115c57b1-49d0-4e1c-8fe4-90a7279b9ae1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:52.638324512Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1366,"client":"1","":"downstairs","session_id":"115c57b1-49d0-4e1c-8fe4-90a7279b9ae1"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:52.63835362Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1366,"client":"1","":"downstairs","session_id":"115c57b1-49d0-4e1c-8fe4-90a7279b9ae1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:52.638381178Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1366,"client":"2","":"downstairs","session_id":"115c57b1-49d0-4e1c-8fe4-90a7279b9ae1"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:52.638409816Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1366,"client":"2","":"downstairs","session_id":"115c57b1-49d0-4e1c-8fe4-90a7279b9ae1"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:52.638510759Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1366,"":"io task","client":"2","":"downstairs","session_id":"115c57b1-49d0-4e1c-8fe4-90a7279b9ae1"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:52.638542787Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1366,"":"io task","client":"0","":"downstairs","session_id":"115c57b1-49d0-4e1c-8fe4-90a7279b9ae1"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:52.638672198Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1366,"":"io task","client":"1","":"downstairs","session_id":"115c57b1-49d0-4e1c-8fe4-90a7279b9ae1"} {"msg":"ds_connection connected from Ok(127.0.0.1:57968)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:52.638720504Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1366,"":"io task","client":"2","":"downstairs","session_id":"115c57b1-49d0-4e1c-8fe4-90a7279b9ae1"} {"msg":"ds_connection connected from Ok(127.0.0.1:35299)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:52.638758332Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1366,"":"io task","client":"0","":"downstairs","session_id":"115c57b1-49d0-4e1c-8fe4-90a7279b9ae1"} {"msg":"ds_connection connected from Ok(127.0.0.1:37723)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:52.638794529Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1366,"":"io task","client":"1","":"downstairs","session_id":"115c57b1-49d0-4e1c-8fe4-90a7279b9ae1"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:52.639315063Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1366,"client":"2","":"downstairs","session_id":"115c57b1-49d0-4e1c-8fe4-90a7279b9ae1"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:52.639486541Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1366,"client":"1","":"downstairs","session_id":"115c57b1-49d0-4e1c-8fe4-90a7279b9ae1"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:52.639616472Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1366,"client":"0","":"downstairs","session_id":"115c57b1-49d0-4e1c-8fe4-90a7279b9ae1"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:52.640075991Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1366,"client":"2","":"downstairs","session_id":"115c57b1-49d0-4e1c-8fe4-90a7279b9ae1"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:52.640119358Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1366,"client":"0","":"downstairs","session_id":"115c57b1-49d0-4e1c-8fe4-90a7279b9ae1"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:52.640199982Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1366,"client":"1","":"downstairs","session_id":"115c57b1-49d0-4e1c-8fe4-90a7279b9ae1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:52.640638032Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1366,"session_id":"115c57b1-49d0-4e1c-8fe4-90a7279b9ae1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:52.640752014Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1366,"session_id":"115c57b1-49d0-4e1c-8fe4-90a7279b9ae1"} {"msg":"[0]R flush_numbers[0..12]: [89, 89, 85, 85, 73, 89, 82, 87, 79, 80, 89, 80]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:52.640878896Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1366,"":"downstairs","":"downstairs","session_id":"115c57b1-49d0-4e1c-8fe4-90a7279b9ae1"} {"msg":"[0]R generation[0..12]: [45, 45, 43, 43, 36, 45, 42, 44, 39, 40, 45, 40]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:52.640899204Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1366,"":"downstairs","":"downstairs","session_id":"115c57b1-49d0-4e1c-8fe4-90a7279b9ae1"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:52.640913953Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1366,"":"downstairs","":"downstairs","session_id":"115c57b1-49d0-4e1c-8fe4-90a7279b9ae1"} {"msg":"[1]R flush_numbers[0..12]: [89, 89, 85, 85, 73, 89, 82, 87, 79, 80, 89, 80]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:52.640927732Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1366,"":"downstairs","":"downstairs","session_id":"115c57b1-49d0-4e1c-8fe4-90a7279b9ae1"} {"msg":"[1]R generation[0..12]: [45, 45, 43, 43, 36, 45, 42, 44, 39, 40, 45, 40]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:52.640941401Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1366,"":"downstairs","":"downstairs","session_id":"115c57b1-49d0-4e1c-8fe4-90a7279b9ae1"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:52.64095576Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1366,"":"downstairs","":"downstairs","session_id":"115c57b1-49d0-4e1c-8fe4-90a7279b9ae1"} {"msg":"[2]R flush_numbers[0..12]: [89, 89, 85, 85, 73, 89, 82, 87, 79, 80, 89, 80]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:52.640969719Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1366,"":"downstairs","":"downstairs","session_id":"115c57b1-49d0-4e1c-8fe4-90a7279b9ae1"} {"msg":"[2]R generation[0..12]: [45, 45, 43, 43, 36, 45, 42, 44, 39, 40, 45, 40]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:52.640983208Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1366,"":"downstairs","":"downstairs","session_id":"115c57b1-49d0-4e1c-8fe4-90a7279b9ae1"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:52.641005237Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1366,"":"downstairs","":"downstairs","session_id":"115c57b1-49d0-4e1c-8fe4-90a7279b9ae1"} {"msg":"Max found gen is 46","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:52.641024706Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1366,"":"downstairs","":"downstairs","session_id":"115c57b1-49d0-4e1c-8fe4-90a7279b9ae1"} {"msg":"Generation requested: 46 >= found:46","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:52.641041434Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1366,"":"downstairs","":"downstairs","session_id":"115c57b1-49d0-4e1c-8fe4-90a7279b9ae1"} Wait for a query_work_queue command to finish before sending IO {"msg":"Next flush: 90","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:52.641059343Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1366,"":"downstairs","":"downstairs","session_id":"115c57b1-49d0-4e1c-8fe4-90a7279b9ae1"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:52.641072592Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1366,"":"downstairs","":"downstairs","session_id":"115c57b1-49d0-4e1c-8fe4-90a7279b9ae1"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:52.641094611Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1366,"session_id":"115c57b1-49d0-4e1c-8fe4-90a7279b9ae1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:52.641110529Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1366,"client":"0","":"downstairs","session_id":"115c57b1-49d0-4e1c-8fe4-90a7279b9ae1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:52.641126648Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1366,"client":"1","":"downstairs","session_id":"115c57b1-49d0-4e1c-8fe4-90a7279b9ae1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:52.641143667Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1366,"client":"2","":"downstairs","session_id":"115c57b1-49d0-4e1c-8fe4-90a7279b9ae1"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:52.641157386Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1366,"session_id":"115c57b1-49d0-4e1c-8fe4-90a7279b9ae1"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:52.641169676Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1366,"session_id":"115c57b1-49d0-4e1c-8fe4-90a7279b9ae1"} {"msg":"628c290f-d648-4d71-b26a-02e6b9da53b7 is now active with session: 115c57b1-49d0-4e1c-8fe4-90a7279b9ae1","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:52.641182415Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1366,"session_id":"115c57b1-49d0-4e1c-8fe4-90a7279b9ae1"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:52.641194314Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1366,"session_id":"115c57b1-49d0-4e1c-8fe4-90a7279b9ae1"} {"msg":"The guest has finished waiting for activation with:46","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:52.641206323Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1366} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 431 len 32768 data: 3 3 3 3 3 3 3 4 02/30 Read block 315 len 32768 03/30 Read block 123 len 28672 04/30 Write block 570 len 28672 data: 2 2 2 2 2 2 3 05/30 Write block 475 len 4096 data: 3 06/30 Read block 362 len 8192 07/30 Write block 100 len 32768 data: 3 3 4 4 4 5 3 5 08/30 Write block 99 len 24576 data: 3 4 4 5 5 5 09/30 Flush 10/30 Write block 130 len 8192 data: 4 3 11/30 Read block 209 len 28672 12/30 Write block 18 len 12288 data: 6 5 5 13/30 Write block 59 len 8192 data: 9 7 14/30 Read block 32 len 20480 15/30 Read block 140 len 36864 16/30 Flush 17/30 Write block 212 len 8192 data: 7 7 18/30 Read block 354 len 36864 19/30 Read block 421 len 32768 20/30 Read block 422 len 24576 21/30 Read block 444 len 12288 22/30 Write block 357 len 8192 data: 6 4 23/30 Write block 213 len 4096 data: 8 24/30 Write block 36 len 12288 data: 8 8 9 25/30 Write block 158 len 4096 data: 5 26/30 Read block 308 len 12288 27/30 Read block 375 len 12288 28/30 Write block 400 len 36864 data: 4 5 7 6 5 6 6 6 6 29/30 Write block 7 len 24576 data: 3 5 6 6 5 4 30/30 Write block 107 len 40960 data: 6 5 5 6 7 7 7 7 5 5 ---------------------------------------------------------------- Crucible gen:46 GIO:true work queues: Upstairs:0 downstairs:14 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1016 Acked Write 2 Done Done Done false 1017 Acked Read 9 Done Done Done false 1018 Acked Read 8 Done Done Done false 1019 Acked Read 6 Done Done Done false 1020 Acked Read 3 Done Done Done false 1021 Acked Write 2 Done Done Done false 1022 Acked Write 1 Done Done Done false 1023 Acked Write 3 Done Done Done false 1024 Acked Write 1 Done Done Done false 1025 Acked Read 3 Done Done Done false 1026 Acked Read 3 Done Done Done false 1027 Acked Write 9 Done Done Done false 1028 Acked Write 6 Sent Sent Sent false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 12 12 12 36 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1015 1015 1015 Downstairs last five completed: 1015 1014 1013 1012 1011 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:34:52.658846746Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1366} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 46 46 46 91 91 91 T T T 1 020-039 46 46 46 91 91 91 T T T 2 040-059 46 46 46 91 91 91 F F F 3 060-079 46 46 46 91 91 91 F F F 4 080-099 46 46 46 90 90 90 F F F 5 100-119 46 46 46 90 90 90 T T T 6 120-139 46 46 46 91 91 91 F F F 7 140-159 44 44 44 87 87 87 T T T 8 160-179 39 39 39 79 79 79 F F F 9 180-199 40 40 40 80 80 80 F F F 10 200-219 45 45 45 89 89 89 T T T 11 220-239 40 40 40 80 80 80 F F F 12 240-259 36 36 36 71 71 71 F F F 13 260-279 44 44 44 87 87 87 F F F 14 280-299 45 45 45 89 89 89 F F F 15 300-319 44 44 44 86 86 86 F F F 16 320-339 39 39 39 79 79 79 F F F 17 340-359 43 43 43 85 85 85 T T T 18 360-379 45 45 45 89 89 89 F F F 19 380-399 38 38 38 77 77 77 F F F 20 400-419 45 45 45 89 89 89 T T T 21 420-439 46 46 46 90 90 90 F F F 22 440-459 42 42 42 84 84 84 F F F 23 460-479 46 46 46 90 90 90 F F F 24 480-499 42 42 42 83 83 83 F F F 25 500-519 40 40 40 80 80 80 F F F 26 520-539 45 45 45 89 89 89 F F F 27 540-559 43 43 43 85 85 85 F F F 28 560-579 46 46 46 90 90 90 F F F 29 580-599 40 40 40 80 80 80 F F F Max gen: 46, Max flush: 91 A Difference in extent metadata is expected here On loop 23 Verifying data now No change after verify Loop: 23 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 47 47 47 92 92 92 F F F 1 020-039 47 47 47 92 92 92 F F F 2 040-059 46 46 46 91 91 91 F F F 3 060-079 46 46 46 91 91 91 F F F 4 080-099 46 46 46 90 90 90 F F F 5 100-119 47 47 47 92 92 92 F F F 6 120-139 46 46 46 91 91 91 F F F 7 140-159 47 47 47 92 92 92 F F F 8 160-179 39 39 39 79 79 79 F F F 9 180-199 40 40 40 80 80 80 F F F 10 200-219 47 47 47 92 92 92 F F F 11 220-239 40 40 40 80 80 80 F F F 12 240-259 36 36 36 71 71 71 F F F 13 260-279 44 44 44 87 87 87 F F F 14 280-299 45 45 45 89 89 89 F F F 15 300-319 44 44 44 86 86 86 F F F 16 320-339 39 39 39 79 79 79 F F F 17 340-359 47 47 47 92 92 92 F F F 18 360-379 45 45 45 89 89 89 F F F 19 380-399 38 38 38 77 77 77 F F F 20 400-419 47 47 47 92 92 92 F F F 21 420-439 46 46 46 90 90 90 F F F 22 440-459 42 42 42 84 84 84 F F F 23 460-479 46 46 46 90 90 90 F F F 24 480-499 42 42 42 83 83 83 F F F 25 500-519 40 40 40 80 80 80 F F F 26 520-539 45 45 45 89 89 89 F F F 27 540-559 43 43 43 85 85 85 F F F 28 560-579 46 46 46 90 90 90 F F F 29 580-599 40 40 40 80 80 80 F F F Max gen: 47, Max flush: 92 Begin loop 24 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:56.938308331Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1374,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:56.983082603Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1374,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:56.983795744Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1374} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"26e7d501b33f47379e53601720353c2983e7654b\",\n git_commit_timestamp: \"2024-12-12T20:16:40.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:56.983832222Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1374} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:56.983846431Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1374} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:56.98386042Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1374} {"msg":"Crucible fc271dea-f799-4e2f-a982-f0364a2b4b87 has session id: ae7b8db9-b6a8-4d5b-a7a0-8a84832a5b49","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:56.984271481Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1374,"session_id":"ae7b8db9-b6a8-4d5b-a7a0-8a84832a5b49"} {"msg":"Upstairs opts: Upstairs UUID: fc271dea-f799-4e2f-a982-f0364a2b4b87, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:56.984319038Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1374,"session_id":"ae7b8db9-b6a8-4d5b-a7a0-8a84832a5b49"} {"msg":"Crucible stats registered with UUID: fc271dea-f799-4e2f-a982-f0364a2b4b87","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:56.984348556Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1374,"session_id":"ae7b8db9-b6a8-4d5b-a7a0-8a84832a5b49"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:57.01576374Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1374,"task":"crutest"} {"msg":"The guest has requested activation with gen:48","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:57.015793938Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1374} {"msg":"fc271dea-f799-4e2f-a982-f0364a2b4b87 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:57.015872342Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1374,"session_id":"ae7b8db9-b6a8-4d5b-a7a0-8a84832a5b49"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:57.015894761Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1374,"client":"0","":"downstairs","session_id":"ae7b8db9-b6a8-4d5b-a7a0-8a84832a5b49"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:57.015923089Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1374,"client":"0","":"downstairs","session_id":"ae7b8db9-b6a8-4d5b-a7a0-8a84832a5b49"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:57.015940798Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1374,"client":"1","":"downstairs","session_id":"ae7b8db9-b6a8-4d5b-a7a0-8a84832a5b49"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:57.015958907Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1374,"client":"1","":"downstairs","session_id":"ae7b8db9-b6a8-4d5b-a7a0-8a84832a5b49"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:57.015975775Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1374,"client":"2","":"downstairs","session_id":"ae7b8db9-b6a8-4d5b-a7a0-8a84832a5b49"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:57.015993214Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1374,"client":"2","":"downstairs","session_id":"ae7b8db9-b6a8-4d5b-a7a0-8a84832a5b49"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:57.016012913Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1374,"":"io task","client":"2","":"downstairs","session_id":"ae7b8db9-b6a8-4d5b-a7a0-8a84832a5b49"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:57.016045941Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1374,"":"io task","client":"0","":"downstairs","session_id":"ae7b8db9-b6a8-4d5b-a7a0-8a84832a5b49"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:57.016079068Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1374,"":"io task","client":"1","":"downstairs","session_id":"ae7b8db9-b6a8-4d5b-a7a0-8a84832a5b49"} {"msg":"ds_connection connected from Ok(127.0.0.1:34044)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:57.016285954Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1374,"":"io task","client":"1","":"downstairs","session_id":"ae7b8db9-b6a8-4d5b-a7a0-8a84832a5b49"} {"msg":"ds_connection connected from Ok(127.0.0.1:61038)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:57.016327621Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1374,"":"io task","client":"2","":"downstairs","session_id":"ae7b8db9-b6a8-4d5b-a7a0-8a84832a5b49"} {"msg":"ds_connection connected from Ok(127.0.0.1:61909)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:57.016363388Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1374,"":"io task","client":"0","":"downstairs","session_id":"ae7b8db9-b6a8-4d5b-a7a0-8a84832a5b49"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:57.016870844Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1374,"client":"0","":"downstairs","session_id":"ae7b8db9-b6a8-4d5b-a7a0-8a84832a5b49"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:57.016901602Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1374,"client":"1","":"downstairs","session_id":"ae7b8db9-b6a8-4d5b-a7a0-8a84832a5b49"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:57.016979996Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1374,"client":"2","":"downstairs","session_id":"ae7b8db9-b6a8-4d5b-a7a0-8a84832a5b49"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:57.017463473Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1374,"client":"0","":"downstairs","session_id":"ae7b8db9-b6a8-4d5b-a7a0-8a84832a5b49"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:57.017589074Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1374,"client":"1","":"downstairs","session_id":"ae7b8db9-b6a8-4d5b-a7a0-8a84832a5b49"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:57.017610182Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1374,"client":"2","":"downstairs","session_id":"ae7b8db9-b6a8-4d5b-a7a0-8a84832a5b49"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:57.018032904Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1374,"session_id":"ae7b8db9-b6a8-4d5b-a7a0-8a84832a5b49"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:57.018110758Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1374,"session_id":"ae7b8db9-b6a8-4d5b-a7a0-8a84832a5b49"} {"msg":"[0]R flush_numbers[0..12]: [92, 92, 91, 91, 90, 92, 91, 92, 79, 80, 92, 80]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:57.01823561Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1374,"":"downstairs","":"downstairs","session_id":"ae7b8db9-b6a8-4d5b-a7a0-8a84832a5b49"} {"msg":"[0]R generation[0..12]: [47, 47, 46, 46, 46, 47, 46, 47, 39, 40, 47, 40]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:57.018255908Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1374,"":"downstairs","":"downstairs","session_id":"ae7b8db9-b6a8-4d5b-a7a0-8a84832a5b49"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:57.018271477Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1374,"":"downstairs","":"downstairs","session_id":"ae7b8db9-b6a8-4d5b-a7a0-8a84832a5b49"} {"msg":"[1]R flush_numbers[0..12]: [92, 92, 91, 91, 90, 92, 91, 92, 79, 80, 92, 80]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:57.018285676Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1374,"":"downstairs","":"downstairs","session_id":"ae7b8db9-b6a8-4d5b-a7a0-8a84832a5b49"} {"msg":"[1]R generation[0..12]: [47, 47, 46, 46, 46, 47, 46, 47, 39, 40, 47, 40]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:57.018299495Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1374,"":"downstairs","":"downstairs","session_id":"ae7b8db9-b6a8-4d5b-a7a0-8a84832a5b49"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:57.018313344Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1374,"":"downstairs","":"downstairs","session_id":"ae7b8db9-b6a8-4d5b-a7a0-8a84832a5b49"} {"msg":"[2]R flush_numbers[0..12]: [92, 92, 91, 91, 90, 92, 91, 92, 79, 80, 92, 80]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:57.018330893Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1374,"":"downstairs","":"downstairs","session_id":"ae7b8db9-b6a8-4d5b-a7a0-8a84832a5b49"} {"msg":"[2]R generation[0..12]: [47, 47, 46, 46, 46, 47, 46, 47, 39, 40, 47, 40]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:57.018345252Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1374,"":"downstairs","":"downstairs","session_id":"ae7b8db9-b6a8-4d5b-a7a0-8a84832a5b49"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:57.018359211Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1374,"":"downstairs","":"downstairs","session_id":"ae7b8db9-b6a8-4d5b-a7a0-8a84832a5b49"} {"msg":"Max found gen is 48","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:57.01837216Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1374,"":"downstairs","":"downstairs","session_id":"ae7b8db9-b6a8-4d5b-a7a0-8a84832a5b49"} {"msg":"Generation requested: 48 >= found:48","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:57.018385369Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1374,"":"downstairs","":"downstairs","session_id":"ae7b8db9-b6a8-4d5b-a7a0-8a84832a5b49"} {"msg":"Next flush: 93","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:57.018398158Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1374,"":"downstairs","":"downstairs","session_id":"ae7b8db9-b6a8-4d5b-a7a0-8a84832a5b49"} {"msg":"All extents matchWait for a query_work_queue command to finish before sending IO ","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:57.018415327Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1374,"":"downstairs","":"downstairs","session_id":"ae7b8db9-b6a8-4d5b-a7a0-8a84832a5b49"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:57.018428586Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1374,"session_id":"ae7b8db9-b6a8-4d5b-a7a0-8a84832a5b49"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:57.018440645Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1374,"client":"0","":"downstairs","session_id":"ae7b8db9-b6a8-4d5b-a7a0-8a84832a5b49"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:57.018454004Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1374,"client":"1","":"downstairs","session_id":"ae7b8db9-b6a8-4d5b-a7a0-8a84832a5b49"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:57.018467134Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1374,"client":"2","":"downstairs","session_id":"ae7b8db9-b6a8-4d5b-a7a0-8a84832a5b49"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:57.018480193Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1374,"session_id":"ae7b8db9-b6a8-4d5b-a7a0-8a84832a5b49"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:57.018492362Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1374,"session_id":"ae7b8db9-b6a8-4d5b-a7a0-8a84832a5b49"} {"msg":"fc271dea-f799-4e2f-a982-f0364a2b4b87 is now active with session: ae7b8db9-b6a8-4d5b-a7a0-8a84832a5b49","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:57.018505181Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1374,"session_id":"ae7b8db9-b6a8-4d5b-a7a0-8a84832a5b49"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:57.01851719Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1374,"session_id":"ae7b8db9-b6a8-4d5b-a7a0-8a84832a5b49"} {"msg":"The guest has finished waiting for activation with:48","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:34:57.018531059Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1374} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 172 len 8192 data: 2 2 02/30 Write block 216 len 32768 data: 4 5 3 4 4 3 3 3 03/30 Write block 344 len 16384 data: 4 4 5 5 04/30 Write block 198 len 36864 data: 3 3 3 3 4 4 5 4 5 05/30 Read block 324 len 28672 06/30 Read block 426 len 24576 07/30 Write block 398 len 16384 data: 6 6 5 6 08/30 Read block 95 len 16384 09/30 Read block 412 len 24576 10/30 Flush 11/30 Write block 21 len 36864 data: 5 5 4 5 3 3 4 4 4 12/30 Read block 320 len 40960 13/30 Read block 59 len 36864 14/30 Write block 120 len 8192 data: 4 4 15/30 Read block 29 len 4096 16/30 Read block 508 len 32768 17/30 Read block 574 len 8192 18/30 Write block 108 len 8192 data: 6 6 19/30 Flush 20/30 Write block 440 len 20480 data: 5 4 5 5 5 21/30 Write block 110 len 16384 data: 7 8 8 8 22/30 Write block 184 len 8192 data: 6 6 23/30 Read block 307 len 28672 24/30 Read block 500 len 4096 25/30 Read block 566 len 36864 26/30 Read block 152 len 16384 27/30 Write block 374 len 40960 data: 6 4 6 5 7 8 8 7 8 7 28/30 Write block 46 len 24576 data: 5 5 6 7 7 5 29/30 Read block 96 len 8192 30/30 Write block 502 len 24576 data: 6 4 4 4 4 6 ---------------------------------------------------------------- Crucible gen:48 GIO:true work queues: Upstairs:0 downstairs:11 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1019 Acked Write 5 Done Done Done false 1020 Acked Write 4 Done Done Done false 1021 Acked Write 2 Done Done Done false 1022 Acked Read 7 Done Done Done false 1023 Acked Read 1 Done Done Done false 1024 Acked Read 9 Done Done Done false 1025 Acked Read 4 Done Done Done false 1026 Acked Write 10 Done Done Done false 1027 Acked Write 6 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 10 10 10 30 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1018 1018 1018 Downstairs last five completed: 1018 1017 1016 1015 1014 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:34:57.036763162Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1374} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 47 47 47 92 92 92 F F F 1 020-039 48 48 48 94 94 94 F F F 2 040-059 46 46 46 91 91 91 T T T 3 060-079 46 46 46 91 91 91 F F F 4 080-099 46 46 46 90 90 90 F F F 5 100-119 48 48 48 94 94 94 T T T 6 120-139 48 48 48 94 94 94 F F F 7 140-159 47 47 47 92 92 92 F F F 8 160-179 48 48 48 93 93 93 F F F 9 180-199 48 48 48 93 93 93 T T T 10 200-219 48 48 48 93 93 93 F F F 11 220-239 48 48 48 93 93 93 F F F 12 240-259 36 36 36 71 71 71 F F F 13 260-279 44 44 44 87 87 87 F F F 14 280-299 45 45 45 89 89 89 F F F 15 300-319 44 44 44 86 86 86 F F F 16 320-339 39 39 39 79 79 79 F F F 17 340-359 48 48 48 93 93 93 F F F 18 360-379 45 45 45 89 89 89 T T T 19 380-399 48 48 48 93 93 93 T T T 20 400-419 48 48 48 93 93 93 F F F 21 420-439 46 46 46 90 90 90 F F F 22 440-459 42 42 42 84 84 84 T T T 23 460-479 46 46 46 90 90 90 F F F 24 480-499 42 42 42 83 83 83 F F F 25 500-519 40 40 40 80 80 80 T T T 26 520-539 45 45 45 89 89 89 F F F 27 540-559 43 43 43 85 85 85 F F F 28 560-579 46 46 46 90 90 90 F F F 29 580-599 40 40 40 80 80 80 F F F Max gen: 48, Max flush: 94 A Difference in extent metadata is expected here On loop 24 Verifying data now No change after verify Loop: 24 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 47 47 47 92 92 92 F F F 1 020-039 48 48 48 94 94 94 F F F 2 040-059 49 49 49 95 95 95 F F F 3 060-079 46 46 46 91 91 91 F F F 4 080-099 46 46 46 90 90 90 F F F 5 100-119 49 49 49 95 95 95 F F F 6 120-139 48 48 48 94 94 94 F F F 7 140-159 47 47 47 92 92 92 F F F 8 160-179 48 48 48 93 93 93 F F F 9 180-199 49 49 49 95 95 95 F F F 10 200-219 48 48 48 93 93 93 F F F 11 220-239 48 48 48 93 93 93 F F F 12 240-259 36 36 36 71 71 71 F F F 13 260-279 44 44 44 87 87 87 F F F 14 280-299 45 45 45 89 89 89 F F F 15 300-319 44 44 44 86 86 86 F F F 16 320-339 39 39 39 79 79 79 F F F 17 340-359 48 48 48 93 93 93 F F F 18 360-379 49 49 49 95 95 95 F F F 19 380-399 49 49 49 95 95 95 F F F 20 400-419 48 48 48 93 93 93 F F F 21 420-439 46 46 46 90 90 90 F F F 22 440-459 49 49 49 95 95 95 F F F 23 460-479 46 46 46 90 90 90 F F F 24 480-499 42 42 42 83 83 83 F F F 25 500-519 49 49 49 95 95 95 F F F 26 520-539 45 45 45 89 89 89 F F F 27 540-559 43 43 43 85 85 85 F F F 28 560-579 46 46 46 90 90 90 F F F 29 580-599 40 40 40 80 80 80 F F F Max gen: 49, Max flush: 95 Begin loop 25 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:01.317275444Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1382,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:35:01.360576728Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1382,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:01.36098151Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1382,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:01.392563322Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1382,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:01.393144333Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1382} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"26e7d501b33f47379e53601720353c2983e7654b\",\n git_commit_timestamp: \"2024-12-12T20:16:40.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:01.39317922Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1382} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:01.393197799Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1382} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:01.393213048Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1382} {"msg":"Crucible 3a5187b8-11af-48e8-945c-a4f72b61c08f has session id: 9088dd70-e429-4c79-87de-28c526a25830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:01.393681196Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1382,"session_id":"9088dd70-e429-4c79-87de-28c526a25830"} {"msg":"Upstairs opts: Upstairs UUID: 3a5187b8-11af-48e8-945c-a4f72b61c08f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:01.393713803Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1382,"session_id":"9088dd70-e429-4c79-87de-28c526a25830"} {"msg":"Crucible stats registered with UUID: 3a5187b8-11af-48e8-945c-a4f72b61c08f","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:01.393729662Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1382,"session_id":"9088dd70-e429-4c79-87de-28c526a25830"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:01.424331672Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1382,"task":"crutest"} {"msg":"The guest has requested activation with gen:50","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:01.42435835Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1382} {"msg":"3a5187b8-11af-48e8-945c-a4f72b61c08f active request set","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:01.424436025Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1382,"session_id":"9088dd70-e429-4c79-87de-28c526a25830"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:01.424453983Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1382,"client":"0","":"downstairs","session_id":"9088dd70-e429-4c79-87de-28c526a25830"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:01.424477852Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1382,"client":"0","":"downstairs","session_id":"9088dd70-e429-4c79-87de-28c526a25830"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:01.424492091Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1382,"client":"1","":"downstairs","session_id":"9088dd70-e429-4c79-87de-28c526a25830"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:01.4245063Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1382,"client":"1","":"downstairs","session_id":"9088dd70-e429-4c79-87de-28c526a25830"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:01.424519729Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1382,"client":"2","":"downstairs","session_id":"9088dd70-e429-4c79-87de-28c526a25830"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:01.424533748Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1382,"client":"2","":"downstairs","session_id":"9088dd70-e429-4c79-87de-28c526a25830"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:01.424550797Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1382,"":"io task","client":"2","":"downstairs","session_id":"9088dd70-e429-4c79-87de-28c526a25830"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:01.424564936Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1382,"":"io task","client":"0","":"downstairs","session_id":"9088dd70-e429-4c79-87de-28c526a25830"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:01.424578555Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1382,"":"io task","client":"1","":"downstairs","session_id":"9088dd70-e429-4c79-87de-28c526a25830"} {"msg":"ds_connection connected from Ok(127.0.0.1:35953)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:01.424858046Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1382,"":"io task","client":"2","":"downstairs","session_id":"9088dd70-e429-4c79-87de-28c526a25830"} {"msg":"ds_connection connected from Ok(127.0.0.1:48751)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:01.424901233Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1382,"":"io task","client":"1","":"downstairs","session_id":"9088dd70-e429-4c79-87de-28c526a25830"} {"msg":"ds_connection connected from Ok(127.0.0.1:40293)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:01.424930241Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1382,"":"io task","client":"0","":"downstairs","session_id":"9088dd70-e429-4c79-87de-28c526a25830"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:01.425534589Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1382,"client":"1","":"downstairs","session_id":"9088dd70-e429-4c79-87de-28c526a25830"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:01.425566347Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1382,"client":"2","":"downstairs","session_id":"9088dd70-e429-4c79-87de-28c526a25830"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:01.425590895Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1382,"client":"0","":"downstairs","session_id":"9088dd70-e429-4c79-87de-28c526a25830"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:01.426286577Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1382,"client":"1","":"downstairs","session_id":"9088dd70-e429-4c79-87de-28c526a25830"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:01.426327524Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1382,"client":"0","":"downstairs","session_id":"9088dd70-e429-4c79-87de-28c526a25830"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:01.426355192Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1382,"client":"2","":"downstairs","session_id":"9088dd70-e429-4c79-87de-28c526a25830"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:01.426880516Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1382,"session_id":"9088dd70-e429-4c79-87de-28c526a25830"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:01.426914174Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1382,"session_id":"9088dd70-e429-4c79-87de-28c526a25830"} {"msg":"[0]R flush_numbers[0..12]: [92, 94, 95, 91, 90, 95, 94, 92, 93, 95, 93, 93]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:01.426949411Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1382,"":"downstairs","":"downstairs","session_id":"9088dd70-e429-4c79-87de-28c526a25830"} {"msg":"[0]R generation[0..12]: [47, 48, 49, 46, 46, 49, 48, 47, 48, 49, 48, 48]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:01.42696506Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1382,"":"downstairs","":"downstairs","session_id":"9088dd70-e429-4c79-87de-28c526a25830"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:01.426980309Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1382,"":"downstairs","":"downstairs","session_id":"9088dd70-e429-4c79-87de-28c526a25830"} {"msg":"[1]R flush_numbers[0..12]: [92, 94, 95, 91, 90, 95, 94, 92, 93, 95, 93, 93]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:01.426994218Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1382,"":"downstairs","":"downstairs","session_id":"9088dd70-e429-4c79-87de-28c526a25830"} {"msg":"[1]R generation[0..12]: [47, 48, 49, 46, 46, 49, 48, 47, 48, 49, 48, 48]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:01.427008057Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1382,"":"downstairs","":"downstairs","session_id":"9088dd70-e429-4c79-87de-28c526a25830"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:01.427022116Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1382,"":"downstairs","":"downstairs","session_id":"9088dd70-e429-4c79-87de-28c526a25830"} {"msg":"[2]R flush_numbers[0..12]: [92, 94, 95, 91, 90, 95, 94, 92, 93, 95, 93, 93]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:01.427035825Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1382,"":"downstairs","":"downstairs","session_id":"9088dd70-e429-4c79-87de-28c526a25830"} {"msg":"[2]R generation[0..12]: [47, 48, 49, 46, 46, 49, 48, 47, 48, 49, 48, 48]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:01.427049435Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1382,"":"downstairs","":"downstairs","session_id":"9088dd70-e429-4c79-87de-28c526a25830"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:01.427063274Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1382,"":"downstairs","":"downstairs","session_id":"9088dd70-e429-4c79-87de-28c526a25830"} {"msg":"Max found gen is 50","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:01.427076173Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1382,"":"downstairs","":"downstairs","session_id":"9088dd70-e429-4c79-87de-28c526a25830"} {"msg":"Generation requested: 50 >= found:50","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:01.427089162Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1382,"":"downstairs","":"downstairs","session_id":"9088dd70-e429-4c79-87de-28c526a25830"} {"msg":"Next flush: 96","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:01.427102051Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1382,"":"downstairs","":"downstairs","session_id":"9088dd70-e429-4c79-87de-28c526a25830"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:01.42711516Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1382,"":"downstairs","":"downstairs","session_id":"9088dd70-e429-4c79-87de-28c526a25830"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:01.427128449Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1382,"session_id":"9088dd70-e429-4c79-87de-28c526a25830"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:01.427140368Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1382,"client":"0","":"downstairs","session_id":"9088dd70-e429-4c79-87de-28c526a25830"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:01.427155687Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1382,"client":"1","":"downstairs","session_id":"9088dd70-e429-4c79-87de-28c526a25830"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:01.427169086Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1382,"client":"2","":"downstairs","session_id":"9088dd70-e429-4c79-87de-28c526a25830"} Wait for a query_work_queue command to finish before sending IO {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:01.427187445Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1382,"session_id":"9088dd70-e429-4c79-87de-28c526a25830"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:01.427199804Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1382,"session_id":"9088dd70-e429-4c79-87de-28c526a25830"} {"msg":"3a5187b8-11af-48e8-945c-a4f72b61c08f is now active with session: 9088dd70-e429-4c79-87de-28c526a25830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:01.427212463Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1382,"session_id":"9088dd70-e429-4c79-87de-28c526a25830"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:01.427224492Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1382,"session_id":"9088dd70-e429-4c79-87de-28c526a25830"} {"msg":"The guest has finished waiting for activation with:50","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:01.427236642Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1382} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 260 len 36864 data: 5 4 4 6 6 9 9 8 8 02/30 Read block 285 len 32768 03/30 Write block 193 len 20480 data: 7 4 4 5 4 04/30 Write block 569 len 16384 data: 3 3 3 3 05/30 Write block 101 len 36864 data: 5 6 6 6 6 4 7 7 7 06/30 Read block 38 len 8192 07/30 Read block 217 len 32768 08/30 Write block 295 len 12288 data: 6 7 7 09/30 Write block 1 len 40960 data: 3 4 4 4 3 3 4 6 7 7 10/30 Write block 93 len 16384 data: 8 7 5 3 11/30 Read block 343 len 32768 12/30 Read block 97 len 8192 13/30 Read block 502 len 16384 14/30 Write block 255 len 24576 data: 7 7 5 6 6 6 15/30 Write block 121 len 40960 data: 5 4 4 3 3 3 3 4 4 5 16/30 Write block 57 len 28672 data: 8 9 10 8 3 3 4 17/30 Write block 155 len 40960 data: 4 5 5 6 5 6 5 7 5 4 18/30 Read block 299 len 16384 19/30 Write block 79 len 28672 data: 3 4 4 3 4 5 6 20/30 Write block 179 len 32768 data: 6 4 4 4 4 7 7 6 21/30 Write block 181 len 32768 data: 5 5 5 8 8 7 7 8 22/30 Read block 187 len 8192 23/30 Read block 275 len 32768 24/30 Read block 17 len 32768 25/30 Write block 299 len 16384 data: 4 5 5 4 26/30 Read block 158 len 40960 27/30 Write block 286 len 40960 data: 5 4 5 5 5 5 5 7 7 7 28/30 Read block 537 len 12288 29/30 Write block 210 len 24576 data: 6 8 8 9 6 6 30/30 Write block 60 len 28672 data: 9 4 4 5 4 4 4 ---------------------------------------------------------------- Crucible gen:50 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 9 Done Done Done false 1001 Acked Read 8 Done Done Done false 1002 Acked Write 5 Done Done Done false 1003 Acked Write 4 Done Done Done false 1004 Acked Write 9 Done Done Done false 1005 Acked Read 2 Done Done Done false 1006 Acked Read 8 Done Done Done false 1007 Acked Write 3 Done Done Done false 1008 Acked Write 10 Done Done Done false 1009 Acked Write 4 Done Done Done false 1010 Acked Read 8 Done Done Done false 1011 Acked Read 2 Done Done Done false 1012 Acked Read 4 Done Done Done false 1013 Acked Write 6 Done Done Done false 1014 Acked Write 10 Done Done Done false 1015 Acked Write 7 Done Done Done false 1016 Acked Write 10 Done Done Done false 1017 Acked Read 4 Done Done Done false 1018 Acked Write 7 Done Done Done false 1019 Acked Write 8 Done Done Done false 1020 Acked Write 8 Done Done Done false 1021 Acked Read 2 Done Done Done false 1022 Acked Read 8 Done Done Done false 1023 Acked Read 8 Done Done Done false 1024 Acked Write 4 Done Done Done false 1025 Acked Read 10 Done Done Done false 1026 Acked Write 10 Done Done Done false 1027 Acked Read 3 Done Done Done false 1028 Acked Write 6 Sent Sent Sent false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 28 28 28 84 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 0 0 0 Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:35:01.447877848Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1382} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 47 47 47 92 92 92 T T T 1 020-039 48 48 48 94 94 94 F F F 2 040-059 49 49 49 95 95 95 T T T 3 060-079 46 46 46 91 91 91 T T T 4 080-099 46 46 46 90 90 90 T T T 5 100-119 49 49 49 95 95 95 T T T 6 120-139 48 48 48 94 94 94 T T T 7 140-159 47 47 47 92 92 92 T T T 8 160-179 48 48 48 93 93 93 T T T 9 180-199 49 49 49 95 95 95 T T T 10 200-219 48 48 48 93 93 93 T T T 11 220-239 48 48 48 93 93 93 F F F 12 240-259 36 36 36 71 71 71 T T T 13 260-279 44 44 44 87 87 87 T T T 14 280-299 45 45 45 89 89 89 T T T 15 300-319 44 44 44 86 86 86 T T T 16 320-339 39 39 39 79 79 79 F F F 17 340-359 48 48 48 93 93 93 F F F 18 360-379 49 49 49 95 95 95 F F F 19 380-399 49 49 49 95 95 95 F F F 20 400-419 48 48 48 93 93 93 F F F 21 420-439 46 46 46 90 90 90 F F F 22 440-459 49 49 49 95 95 95 F F F 23 460-479 46 46 46 90 90 90 F F F 24 480-499 42 42 42 83 83 83 F F F 25 500-519 49 49 49 95 95 95 F F F 26 520-539 45 45 45 89 89 89 F F F 27 540-559 43 43 43 85 85 85 F F F 28 560-579 46 46 46 90 90 90 T T T 29 580-599 40 40 40 80 80 80 F F F Max gen: 49, Max flush: 95 A Difference in extent metadata is expected here On loop 25 Verifying data now No change after verify Loop: 25 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 50 50 50 96 96 96 F F F 1 020-039 48 48 48 94 94 94 F F F 2 040-059 50 50 50 96 96 96 F F F 3 060-079 50 50 50 96 96 96 F F F 4 080-099 50 50 50 96 96 96 F F F 5 100-119 50 50 50 96 96 96 F F F 6 120-139 50 50 50 96 96 96 F F F 7 140-159 50 50 50 96 96 96 F F F 8 160-179 50 50 50 96 96 96 F F F 9 180-199 50 50 50 96 96 96 F F F 10 200-219 50 50 50 96 96 96 F F F 11 220-239 48 48 48 93 93 93 F F F 12 240-259 50 50 50 96 96 96 F F F 13 260-279 50 50 50 96 96 96 F F F 14 280-299 50 50 50 96 96 96 F F F 15 300-319 50 50 50 96 96 96 F F F 16 320-339 39 39 39 79 79 79 F F F 17 340-359 48 48 48 93 93 93 F F F 18 360-379 49 49 49 95 95 95 F F F 19 380-399 49 49 49 95 95 95 F F F 20 400-419 48 48 48 93 93 93 F F F 21 420-439 46 46 46 90 90 90 F F F 22 440-459 49 49 49 95 95 95 F F F 23 460-479 46 46 46 90 90 90 F F F 24 480-499 42 42 42 83 83 83 F F F 25 500-519 49 49 49 95 95 95 F F F 26 520-539 45 45 45 89 89 89 F F F 27 540-559 43 43 43 85 85 85 F F F 28 560-579 50 50 50 96 96 96 F F F 29 580-599 40 40 40 80 80 80 F F F Max gen: 50, Max flush: 96 Begin loop 26 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:05.801230948Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1390,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:05.845576531Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1390,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:05.846215906Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1390} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"26e7d501b33f47379e53601720353c2983e7654b\",\n git_commit_timestamp: \"2024-12-12T20:16:40.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:05.846251664Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1390} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:05.846267573Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1390} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:05.846282162Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1390} {"msg":"Crucible 229616df-e9d1-48fa-9362-e71857afa2d4 has session id: 0b847b79-4d80-4e95-8d45-e1472498a545","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:05.846678264Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1390,"session_id":"0b847b79-4d80-4e95-8d45-e1472498a545"} {"msg":"Upstairs opts: Upstairs UUID: 229616df-e9d1-48fa-9362-e71857afa2d4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:05.846723061Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1390,"session_id":"0b847b79-4d80-4e95-8d45-e1472498a545"} {"msg":"Crucible stats registered with UUID: 229616df-e9d1-48fa-9362-e71857afa2d4","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:05.846753539Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1390,"session_id":"0b847b79-4d80-4e95-8d45-e1472498a545"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:05.8779223Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1390,"task":"crutest"} {"msg":"The guest has requested activation with gen:52","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:05.877953618Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1390} {"msg":"229616df-e9d1-48fa-9362-e71857afa2d4 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:05.878003255Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1390,"session_id":"0b847b79-4d80-4e95-8d45-e1472498a545"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:05.878023713Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1390,"client":"0","":"downstairs","session_id":"0b847b79-4d80-4e95-8d45-e1472498a545"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:05.878053871Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1390,"client":"0","":"downstairs","session_id":"0b847b79-4d80-4e95-8d45-e1472498a545"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:05.878086659Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1390,"client":"1","":"downstairs","session_id":"0b847b79-4d80-4e95-8d45-e1472498a545"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:05.878122696Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1390,"client":"1","":"downstairs","session_id":"0b847b79-4d80-4e95-8d45-e1472498a545"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:05.878158724Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1390,"client":"2","":"downstairs","session_id":"0b847b79-4d80-4e95-8d45-e1472498a545"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:05.878179152Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1390,"client":"2","":"downstairs","session_id":"0b847b79-4d80-4e95-8d45-e1472498a545"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:05.878202001Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1390,"":"io task","client":"2","":"downstairs","session_id":"0b847b79-4d80-4e95-8d45-e1472498a545"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:05.878233839Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1390,"":"io task","client":"0","":"downstairs","session_id":"0b847b79-4d80-4e95-8d45-e1472498a545"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:05.878269466Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1390,"":"io task","client":"1","":"downstairs","session_id":"0b847b79-4d80-4e95-8d45-e1472498a545"} {"msg":"ds_connection connected from Ok(127.0.0.1:35009)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:05.878306834Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1390,"":"io task","client":"2","":"downstairs","session_id":"0b847b79-4d80-4e95-8d45-e1472498a545"} {"msg":"ds_connection connected from Ok(127.0.0.1:44912)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:05.878344511Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1390,"":"io task","client":"0","":"downstairs","session_id":"0b847b79-4d80-4e95-8d45-e1472498a545"} {"msg":"ds_connection connected from Ok(127.0.0.1:62535)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:05.878383728Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1390,"":"io task","client":"1","":"downstairs","session_id":"0b847b79-4d80-4e95-8d45-e1472498a545"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:05.879025144Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1390,"client":"0","":"downstairs","session_id":"0b847b79-4d80-4e95-8d45-e1472498a545"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:05.879058582Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1390,"client":"2","":"downstairs","session_id":"0b847b79-4d80-4e95-8d45-e1472498a545"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:05.87907605Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1390,"client":"1","":"downstairs","session_id":"0b847b79-4d80-4e95-8d45-e1472498a545"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:05.879640491Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1390,"client":"2","":"downstairs","session_id":"0b847b79-4d80-4e95-8d45-e1472498a545"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:05.879760043Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1390,"client":"1","":"downstairs","session_id":"0b847b79-4d80-4e95-8d45-e1472498a545"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:05.87980191Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1390,"client":"0","":"downstairs","session_id":"0b847b79-4d80-4e95-8d45-e1472498a545"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:05.880166765Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1390,"session_id":"0b847b79-4d80-4e95-8d45-e1472498a545"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:05.880252059Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1390,"session_id":"0b847b79-4d80-4e95-8d45-e1472498a545"} {"msg":"[0]R flush_numbers[0..12]: [96, 94, 96, 96, 96, 96, 96, 96, 96, 96, 96, 93]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:05.880373871Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1390,"":"downstairs","":"downstairs","session_id":"0b847b79-4d80-4e95-8d45-e1472498a545"} {"msg":"[0]R generation[0..12]: [50, 48, 50, 50, 50, 50, 50, 50, 50, 50, 50, 48]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:05.880394019Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1390,"":"downstairs","":"downstairs","session_id":"0b847b79-4d80-4e95-8d45-e1472498a545"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:05.880409278Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1390,"":"downstairs","":"downstairs","session_id":"0b847b79-4d80-4e95-8d45-e1472498a545"} {"msg":"[1]R flush_numbers[0..12]: [96, 94, 96, 96, 96, 96, 96, 96, 96, 96, 96, 93]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:05.880423608Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1390,"":"downstairs","":"downstairs","session_id":"0b847b79-4d80-4e95-8d45-e1472498a545"} {"msg":"[1]R generation[0..12]: [50, 48, 50, 50, 50, 50, 50, 50, 50, 50, 50, 48]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:05.880437287Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1390,"":"downstairs","":"downstairs","session_id":"0b847b79-4d80-4e95-8d45-e1472498a545"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:05.880451176Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1390,"":"downstairs","":"downstairs","session_id":"0b847b79-4d80-4e95-8d45-e1472498a545"} {"msg":"[2]R flush_numbers[0..12]: [96, 94, 96, 96, 96, 96, 96, 96, 96, 96, 96, 93]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:05.880467425Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1390,"":"downstairs","":"downstairs","session_id":"0b847b79-4d80-4e95-8d45-e1472498a545"} {"msg":"[2]R generation[0..12]: [50, 48, 50, 50, 50, 50, 50, 50, 50, 50, 50, 48]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:05.880481504Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1390,"":"downstairs","":"downstairs","session_id":"0b847b79-4d80-4e95-8d45-e1472498a545"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:05.880495523Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1390,"":"downstairs","":"downstairs","session_id":"0b847b79-4d80-4e95-8d45-e1472498a545"} {"msg":"Max found gen is 51","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:05.880508602Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1390,"":"downstairs","":"downstairs","session_id":"0b847b79-4d80-4e95-8d45-e1472498a545"} {"msg":"Generation requested: 52 >= found:51","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:05.880521851Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1390,"":"downstairs","":"downstairs","session_id":"0b847b79-4d80-4e95-8d45-e1472498a545"} {"msg":"Next flush: 97","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:05.88053471Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF"Wait for a query_work_queue command to finish before sending IO ,"pid":1390,"":"downstairs","":"downstairs","session_id":"0b847b79-4d80-4e95-8d45-e1472498a545"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:05.880558348Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1390,"":"downstairs","":"downstairs","session_id":"0b847b79-4d80-4e95-8d45-e1472498a545"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:05.880574337Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1390,"session_id":"0b847b79-4d80-4e95-8d45-e1472498a545"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:05.880586576Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1390,"client":"0","":"downstairs","session_id":"0b847b79-4d80-4e95-8d45-e1472498a545"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:05.880600565Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1390,"client":"1","":"downstairs","session_id":"0b847b79-4d80-4e95-8d45-e1472498a545"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:05.880613794Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1390,"client":"2","":"downstairs","session_id":"0b847b79-4d80-4e95-8d45-e1472498a545"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:05.880626894Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1390,"session_id":"0b847b79-4d80-4e95-8d45-e1472498a545"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:05.880639223Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1390,"session_id":"0b847b79-4d80-4e95-8d45-e1472498a545"} {"msg":"229616df-e9d1-48fa-9362-e71857afa2d4 is now active with session: 0b847b79-4d80-4e95-8d45-e1472498a545","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:05.880651992Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1390,"session_id":"0b847b79-4d80-4e95-8d45-e1472498a545"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:05.880664051Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1390,"session_id":"0b847b79-4d80-4e95-8d45-e1472498a545"} {"msg":"The guest has finished waiting for activation with:52","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:05.88067859Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1390} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 342 len 36864 data: 3 3 5 5 6 6 5 4 5 02/30 Flush 03/30 Flush 04/30 Write block 584 len 20480 data: 6 5 5 4 4 05/30 Flush 06/30 Write block 520 len 24576 data: 7 7 7 9 8 6 07/30 Write block 589 len 28672 data: 4 5 4 4 4 3 2 08/30 Write block 447 len 40960 data: 7 3 4 5 6 6 6 6 5 4 09/30 Write block 386 len 28672 data: 7 6 5 5 6 6 5 10/30 Read block 458 len 12288 11/30 Write block 516 len 20480 data: 4 4 7 8 8 12/30 Read block 489 len 24576 13/30 Read block 59 len 4096 14/30 Write block 380 len 20480 data: 9 8 9 8 6 15/30 Read block 579 len 40960 16/30 Flush 17/30 Write block 405 len 28672 data: 7 7 7 7 5 5 4 18/30 Read block 438 len 32768 19/30 Read block 479 len 28672 20/30 Read block 286 len 12288 21/30 Write block 520 len 28672 data: 9 8 8 10 9 7 4 22/30 Read block 455 len 20480 23/30 Write block 228 len 28672 data: 6 7 8 7 6 6 6 24/30 Write block 128 len 40960 data: 5 5 6 4 2 3 3 3 5 5 25/30 Read block 135 len 36864 26/30 Read block 568 len 8192 27/30 Read block 221 len 20480 28/30 Read block 451 len 24576 29/30 Write block 513 len 32768 data: 5 5 4 5 5 8 9 10 30/30 Write block 401 len 36864 data: 7 8 7 6 8 8 8 8 6 ---------------------------------------------------------------- Crucible gen:52 GIO:true work queues: Upstairs:0 downstairs:14 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1016 Acked Write 7 Done Done Done false 1017 Acked Read 8 Done Done Done false 1018 Acked Read 7 Done Done Done false 1019 Acked Read 3 Done Done Done false 1020 Acked Write 7 Done Done Done false 1021 Acked Read 5 Done Done Done false 1022 Acked Write 7 Done Done Done false 1023 Acked Write 10 Done Done Done false 1024 Acked Read 9 Done Done Done false 1025 Acked Read 2 Done Done Done false 1026 Acked Read 5 Done Done Done false 1027 Acked Read 6 Done Done Done false 1028 Acked Write 8 Sent Sent Sent false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 12 12 12 36 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1015 1015 1015 Downstairs last five completed: 1015 1014 1013 1012 1011 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:35:05.901500884Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1390} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 50 50 50 96 96 96 F F F 1 020-039 48 48 48 94 94 94 F F F 2 040-059 50 50 50 96 96 96 F F F 3 060-079 50 50 50 96 96 96 F F F 4 080-099 50 50 50 96 96 96 F F F 5 100-119 50 50 50 96 96 96 F F F 6 120-139 50 50 50 96 96 96 T T T 7 140-159 50 50 50 96 96 96 F F F 8 160-179 50 50 50 96 96 96 F F F 9 180-199 50 50 50 96 96 96 F F F 10 200-219 50 50 50 96 96 96 F F F 11 220-239 48 48 48 93 93 93 T T T 12 240-259 50 50 50 96 96 96 F F F 13 260-279 50 50 50 96 96 96 F F F 14 280-299 50 50 50 96 96 96 F F F 15 300-319 50 50 50 96 96 96 F F F 16 320-339 39 39 39 79 79 79 F F F 17 340-359 52 52 52 97 97 97 F F F 18 360-379 49 49 49 95 95 95 F F F 19 380-399 52 52 52 100 100 100 F F F 20 400-419 48 48 48 93 93 93 T T T 21 420-439 46 46 46 90 90 90 F F F 22 440-459 52 52 52 100 100 100 F F F 23 460-479 46 46 46 90 90 90 F F F 24 480-499 42 42 42 83 83 83 F F F 25 500-519 52 52 52 100 100 100 T T T 26 520-539 52 52 52 100 100 100 T T T 27 540-559 43 43 43 85 85 85 F F F 28 560-579 50 50 50 96 96 96 F F F 29 580-599 52 52 52 100 100 100 F F F Max gen: 52, Max flush: 100 A Difference in extent metadata is expected here On loop 26 Verifying data now No change after verify Loop: 26 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 50 50 50 96 96 96 F F F 1 020-039 48 48 48 94 94 94 F F F 2 040-059 50 50 50 96 96 96 F F F 3 060-079 50 50 50 96 96 96 F F F 4 080-099 50 50 50 96 96 96 F F F 5 100-119 50 50 50 96 96 96 F F F 6 120-139 53 53 53 101 101 101 F F F 7 140-159 50 50 50 96 96 96 F F F 8 160-179 50 50 50 96 96 96 F F F 9 180-199 50 50 50 96 96 96 F F F 10 200-219 50 50 50 96 96 96 F F F 11 220-239 53 53 53 101 101 101 F F F 12 240-259 50 50 50 96 96 96 F F F 13 260-279 50 50 50 96 96 96 F F F 14 280-299 50 50 50 96 96 96 F F F 15 300-319 50 50 50 96 96 96 F F F 16 320-339 39 39 39 79 79 79 F F F 17 340-359 52 52 52 97 97 97 F F F 18 360-379 49 49 49 95 95 95 F F F 19 380-399 52 52 52 100 100 100 F F F 20 400-419 53 53 53 101 101 101 F F F 21 420-439 46 46 46 90 90 90 F F F 22 440-459 52 52 52 100 100 100 F F F 23 460-479 46 46 46 90 90 90 F F F 24 480-499 42 42 42 83 83 83 F F F 25 500-519 53 53 53 101 101 101 F F F 26 520-539 53 53 53 101 101 101 F F F 27 540-559 43 43 43 85 85 85 F F F 28 560-579 50 50 50 96 96 96 F F F 29 580-599 52 52 52 100 100 100 F F F Max gen: 53, Max flush: 101 Begin loop 27 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:10.173065804Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1398,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:10.217545807Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1398,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:10.218158335Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1398} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"26e7d501b33f47379e53601720353c2983e7654b\",\n git_commit_timestamp: \"2024-12-12T20:16:40.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:10.218194862Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1398} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:10.218212661Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1398} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:10.21822729Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1398} {"msg":"Crucible 34f79bf4-9408-406f-ae91-57bb64be50b3 has session id: a97df90d-4de2-4573-96e8-d0c32afa7436","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:10.2186637Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1398,"session_id":"a97df90d-4de2-4573-96e8-d0c32afa7436"} {"msg":"Upstairs opts: Upstairs UUID: 34f79bf4-9408-406f-ae91-57bb64be50b3, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:10.218704437Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1398,"session_id":"a97df90d-4de2-4573-96e8-d0c32afa7436"} {"msg":"Crucible stats registered with UUID: 34f79bf4-9408-406f-ae91-57bb64be50b3","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:10.218720496Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1398,"session_id":"a97df90d-4de2-4573-96e8-d0c32afa7436"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:10.249857079Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1398,"task":"crutest"} {"msg":"The guest has requested activation with gen:54","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:10.249890967Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1398} {"msg":"34f79bf4-9408-406f-ae91-57bb64be50b3 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:10.249960452Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1398,"session_id":"a97df90d-4de2-4573-96e8-d0c32afa7436"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:10.24998356Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1398,"client":"0","":"downstairs","session_id":"a97df90d-4de2-4573-96e8-d0c32afa7436"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:10.250014728Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1398,"client":"0","":"downstairs","session_id":"a97df90d-4de2-4573-96e8-d0c32afa7436"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:10.250046966Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1398,"client":"1","":"downstairs","session_id":"a97df90d-4de2-4573-96e8-d0c32afa7436"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:10.250082053Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1398,"client":"1","":"downstairs","session_id":"a97df90d-4de2-4573-96e8-d0c32afa7436"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:10.250117351Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1398,"client":"2","":"downstairs","session_id":"a97df90d-4de2-4573-96e8-d0c32afa7436"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:10.250154398Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1398,"client":"2","":"downstairs","session_id":"a97df90d-4de2-4573-96e8-d0c32afa7436"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:10.250190106Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1398,"":"io task","client":"2","":"downstairs","session_id":"a97df90d-4de2-4573-96e8-d0c32afa7436"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:10.250231353Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1398,"":"io task","client":"0","":"downstairs","session_id":"a97df90d-4de2-4573-96e8-d0c32afa7436"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:10.25026673Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1398,"":"io task","client":"1","":"downstairs","session_id":"a97df90d-4de2-4573-96e8-d0c32afa7436"} {"msg":"ds_connection connected from Ok(127.0.0.1:52644)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:10.250407461Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1398,"":"io task","client":"2","":"downstairs","session_id":"a97df90d-4de2-4573-96e8-d0c32afa7436"} {"msg":"ds_connection connected from Ok(127.0.0.1:49535)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:10.250438489Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1398,"":"io task","client":"1","":"downstairs","session_id":"a97df90d-4de2-4573-96e8-d0c32afa7436"} {"msg":"ds_connection connected from Ok(127.0.0.1:37465)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:10.250453628Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1398,"":"io task","client":"0","":"downstairs","session_id":"a97df90d-4de2-4573-96e8-d0c32afa7436"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:10.250953253Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1398,"client":"2","":"downstairs","session_id":"a97df90d-4de2-4573-96e8-d0c32afa7436"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:10.2509958Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1398,"client":"0","":"downstairs","session_id":"a97df90d-4de2-4573-96e8-d0c32afa7436"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:10.251025738Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1398,"client":"1","":"downstairs","session_id":"a97df90d-4de2-4573-96e8-d0c32afa7436"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:10.251553652Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1398,"client":"0","":"downstairs","session_id":"a97df90d-4de2-4573-96e8-d0c32afa7436"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:10.251660545Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1398,"client":"1","":"downstairs","session_id":"a97df90d-4de2-4573-96e8-d0c32afa7436"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:10.251706901Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1398,"client":"2","":"downstairs","session_id":"a97df90d-4de2-4573-96e8-d0c32afa7436"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:10.252048928Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1398,"session_id":"a97df90d-4de2-4573-96e8-d0c32afa7436"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:10.252216356Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1398,"session_id":"a97df90d-4de2-4573-96e8-d0c32afa7436"} {"msg":"[0]R flush_numbers[0..12]: [96, 94, 96, 96, 96, 96, 101, 96, 96, 96, 96, 101]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:10.252343727Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1398,"":"downstairs","":"downstairs","session_id":"a97df90d-4de2-4573-96e8-d0c32afa7436"} {"msg":"[0]R generation[0..12]: [50, 48, 50, 50, 50, 50, 53, 50, 50, 50, 50, 53]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:10.252365946Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1398,"":"downstairs","":"downstairs","session_id":"a97df90d-4de2-4573-96e8-d0c32afa7436"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:10.252381425Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1398,"":"downstairs","":"downstairs","session_id":"a97df90d-4de2-4573-96e8-d0c32afa7436"} {"msg":"[1]R flush_numbers[0..12]: [96, 94, 96, 96, 96, 96, 101, 96, 96, 96, 96, 101]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:10.252395204Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1398,"":"downstairs","":"downstairs","session_id":"a97df90d-4de2-4573-96e8-d0c32afa7436"} {"msg":"[1]R generation[0..12]: [50, 48, 50, 50, 50, 50, 53, 50, 50, 50, 50, 53]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:10.252408983Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1398,"":"downstairs","":"downstairs","session_id":"a97df90d-4de2-4573-96e8-d0c32afa7436"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:10.252422772Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1398,"":"downstairs","":"downstairs","session_id":"a97df90d-4de2-4573-96e8-d0c32afa7436"} {"msg":"[2]R flush_numbers[0..12]: [96, 94, 96, 96, 96, 96, 101, 96, 96, 96, 96, 101]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:10.252436171Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1398,"":"downstairs","":"downstairs","session_id":"a97df90d-4de2-4573-96e8-d0c32afa7436"} {"msg":"[2]R generation[0..12]: [50, 48, 50, 50, 50, 50, 53, 50, 50, 50, 50, 53]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:10.25244979Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1398,"":"downstairs","":"downstairs","session_id":"a97df90d-4de2-4573-96e8-d0c32afa7436"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:10.252463939Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1398,"":"downstairs","":"downstairs","session_id":"a97df90d-4de2-4573-96e8-d0c32afa7436"} {"msg":"Max found gen is 54","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:10.252476788Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1398,"":"downstairs","":"downstairs","session_id":"a97df90d-4de2-4573-96e8-d0c32afa7436"} {"msg":"Generation requested: 54 >= found:54","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:10.252489847Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1398,"":"downstairs","":"downstairs","session_id":"a97df90d-4de2-4573-96e8-d0c32afa7436"} {"msg":"Next flush: 102","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:10.252502766Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1398,"":"downstairs","":"downstairs","session_id":"a97df90d-4de2-4573-96e8-d0c32afa7436"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-12-12T20:35:10.252515646Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1398,"":"downstairs","":"downstairs","session_id":"a97df90d-4de2-4573-96e8-d0c32afa7436"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:10.252532264Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1398,"session_id":"a97df90d-4de2-4573-96e8-d0c32afa7436"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:10.252544133Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1398,"client":"0","":"downstairs","session_id":"a97df90d-4de2-4573-96e8-d0c32afa7436"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:10.252559783Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1398,"client":"1","":"downstairs","session_id":"a97df90d-4de2-4573-96e8-d0c32afa7436"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:10.252573451Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1398,"client":"2","":"downstairs","session_id":"a97df90d-4de2-4573-96e8-d0c32afa7436"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:10.25258653Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1398,"session_id":"a97df90d-4de2-4573-96e8-d0c32afa7436"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:10.252609459Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1398,"session_id":"a97df90d-4de2-4573-96e8-d0c32afa7436"} {"msg":"34f79bf4-9408-406f-ae91-57bb64be50b3 is now active with session: a97df90d-4de2-4573-96e8-d0c32afa7436","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:10.252622818Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1398,"session_id":"a97df90d-4de2-4573-96e8-d0c32afa7436"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:10.252634787Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1398,"session_id":"a97df90d-4de2-4573-96e8-d0c32afa7436"} {"msg":"The guest has finished waiting for activation with:54","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:10.252646817Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1398} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 235 len 20480 data: 6 5 4 4 5 02/30 Read block 92 len 28672 03/30 Write block 493 len 20480 data: 4 3 3 3 3 04/30 Write block 525 len 20480 data: 8 5 4 4 5 05/30 Read block 325 len 28672 06/30 Read block 476 len 20480 07/30 Read block 187 len 16384 08/30 Read block 519 len 32768 09/30 Write block 431 len 20480 data: 4 4 4 4 4 10/30 Write block 143 len 32768 data: 3 4 5 5 5 6 4 4 11/30 Write block 122 len 32768 data: 5 5 4 4 4 4 6 6 12/30 Read block 185 len 36864 13/30 Read block 432 len 8192 14/30 Write block 492 len 24576 data: 4 5 4 4 4 4 15/30 Write block 346 len 8192 data: 7 7 16/30 Write block 285 len 32768 data: 5 6 5 6 6 6 6 6 17/30 Write block 484 len 28672 data: 5 6 6 5 4 5 5 18/30 Flush 19/30 Write block 229 len 36864 data: 8 9 8 7 7 7 7 6 5 20/30 Write block 329 len 40960 data: 5 5 5 4 6 6 5 5 4 5 21/30 Read block 488 len 16384 22/30 Write block 55 len 24576 data: 6 8 9 10 11 10 23/30 Flush 24/30 Write block 578 len 36864 data: 4 5 4 4 5 5 7 6 6 25/30 Read block 227 len 40960 26/30 Read block 255 len 24576 27/30 Write block 441 len 20480 data: 5 6 6 6 5 28/30 Write block 208 len 32768 data: 5 5 7 9 9 10 7 7 29/30 Write block 519 len 12288 data: 10 11 9 30/30 Write block 248 len 28672 data: 2 3 3 4 4 7 7 ---------------------------------------------------------------- Crucible gen:54 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 9 Done Done Done false 1024 Acked Read 10 Done Done Done false 1025 Acked Read 6 Done Done Done false 1026 Acked Write 5 Done Done Done false 1027 Acked Write 8 Done Done Done false 1028 Acked Write 3 Sent Sent Done false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 1 5 Done 5 5 6 16 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:35:10.27332926Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1398} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 50 50 50 96 96 96 F F F 1 020-039 48 48 48 94 94 94 F F F 2 040-059 54 54 54 103 103 103 F F F 3 060-079 54 54 54 103 103 103 F F F 4 080-099 50 50 50 96 96 96 F F F 5 100-119 50 50 50 96 96 96 F F F 6 120-139 54 54 54 102 102 102 F F F 7 140-159 54 54 54 102 102 102 F F F 8 160-179 50 50 50 96 96 96 F F F 9 180-199 50 50 50 96 96 96 F F F 10 200-219 50 50 50 96 96 96 T T T 11 220-239 54 54 54 103 103 103 F F F 12 240-259 50 50 50 96 96 96 T T T 13 260-279 50 50 50 96 96 96 F F F 14 280-299 54 54 54 102 102 102 F F F 15 300-319 50 50 50 96 96 96 F F F 16 320-339 54 54 54 103 103 103 F F F 17 340-359 54 54 54 102 102 102 F F F 18 360-379 49 49 49 95 95 95 F F F 19 380-399 52 52 52 100 100 100 F F F 20 400-419 53 53 53 101 101 101 F F F 21 420-439 54 54 54 102 102 102 F F F 22 440-459 52 52 52 100 100 100 T T T 23 460-479 46 46 46 90 90 90 F F F 24 480-499 54 54 54 102 102 102 F F F 25 500-519 53 53 53 101 101 101 T T T 26 520-539 54 54 54 102 102 102 T T T 27 540-559 43 43 43 85 85 85 F F F 28 560-579 50 50 50 96 96 96 T T T 29 580-599 52 52 52 100 100 100 T T T Max gen: 54, Max flush: 103 A Difference in extent metadata is expected here On loop 27 Verifying data now No change after verify Loop: 27 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 50 50 50 96 96 96 F F F 1 020-039 48 48 48 94 94 94 F F F 2 040-059 54 54 54 103 103 103 F F F 3 060-079 54 54 54 103 103 103 F F F 4 080-099 50 50 50 96 96 96 F F F 5 100-119 50 50 50 96 96 96 F F F 6 120-139 54 54 54 102 102 102 F F F 7 140-159 54 54 54 102 102 102 F F F 8 160-179 50 50 50 96 96 96 F F F 9 180-199 50 50 50 96 96 96 F F F 10 200-219 55 55 55 104 104 104 F F F 11 220-239 54 54 54 103 103 103 F F F 12 240-259 55 55 55 104 104 104 F F F 13 260-279 50 50 50 96 96 96 F F F 14 280-299 54 54 54 102 102 102 F F F 15 300-319 50 50 50 96 96 96 F F F 16 320-339 54 54 54 103 103 103 F F F 17 340-359 54 54 54 102 102 102 F F F 18 360-379 49 49 49 95 95 95 F F F 19 380-399 52 52 52 100 100 100 F F F 20 400-419 53 53 53 101 101 101 F F F 21 420-439 54 54 54 102 102 102 F F F 22 440-459 55 55 55 104 104 104 F F F 23 460-479 46 46 46 90 90 90 F F F 24 480-499 54 54 54 102 102 102 F F F 25 500-519 55 55 55 104 104 104 F F F 26 520-539 55 55 55 104 104 104 F F F 27 540-559 43 43 43 85 85 85 F F F 28 560-579 55 55 55 104 104 104 F F F 29 580-599 55 55 55 104 104 104 F F F Max gen: 55, Max flush: 104 Begin loop 28 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:14.551589708Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1406,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:14.596646852Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1406,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:14.5972543Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1406} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"26e7d501b33f47379e53601720353c2983e7654b\",\n git_commit_timestamp: \"2024-12-12T20:16:40.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:14.597291237Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1406} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:14.597305876Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1406} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:14.597320085Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1406} {"msg":"Crucible 35b71f1f-bfaf-4afc-91ff-cdbfa1dc8faa has session id: fc797ca8-3dcc-47b7-bb8b-3616c07a67a5","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:14.597863568Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1406,"session_id":"fc797ca8-3dcc-47b7-bb8b-3616c07a67a5"} {"msg":"Upstairs opts: Upstairs UUID: 35b71f1f-bfaf-4afc-91ff-cdbfa1dc8faa, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:14.597896706Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1406,"session_id":"fc797ca8-3dcc-47b7-bb8b-3616c07a67a5"} {"msg":"Crucible stats registered with UUID: 35b71f1f-bfaf-4afc-91ff-cdbfa1dc8faa","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:14.597911494Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1406,"session_id":"fc797ca8-3dcc-47b7-bb8b-3616c07a67a5"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:14.630154661Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1406,"task":"crutest"} {"msg":"The guest has requested activation with gen:56","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:14.630182119Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1406} {"msg":"35b71f1f-bfaf-4afc-91ff-cdbfa1dc8faa active request set","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:14.630246045Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1406,"session_id":"fc797ca8-3dcc-47b7-bb8b-3616c07a67a5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:14.630274113Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1406,"client":"0","":"downstairs","session_id":"fc797ca8-3dcc-47b7-bb8b-3616c07a67a5"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:14.630298601Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1406,"client":"0","":"downstairs","session_id":"fc797ca8-3dcc-47b7-bb8b-3616c07a67a5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:14.63031302Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1406,"client":"1","":"downstairs","session_id":"fc797ca8-3dcc-47b7-bb8b-3616c07a67a5"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:14.630331939Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1406,"client":"1","":"downstairs","session_id":"fc797ca8-3dcc-47b7-bb8b-3616c07a67a5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:14.630354687Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1406,"client":"2","":"downstairs","session_id":"fc797ca8-3dcc-47b7-bb8b-3616c07a67a5"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:14.630383065Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1406,"client":"2","":"downstairs","session_id":"fc797ca8-3dcc-47b7-bb8b-3616c07a67a5"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:14.630405414Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1406,"":"io task","client":"2","":"downstairs","session_id":"fc797ca8-3dcc-47b7-bb8b-3616c07a67a5"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:14.630429282Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1406,"":"io task","client":"0","":"downstairs","session_id":"fc797ca8-3dcc-47b7-bb8b-3616c07a67a5"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:14.63045749Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1406,"":"io task","client":"1","":"downstairs","session_id":"fc797ca8-3dcc-47b7-bb8b-3616c07a67a5"} {"msg":"ds_connection connected from Ok(127.0.0.1:61657)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:14.630485218Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1406,"":"io task","client":"2","":"downstairs","session_id":"fc797ca8-3dcc-47b7-bb8b-3616c07a67a5"} {"msg":"ds_connection connected from Ok(127.0.0.1:42769)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:14.630513036Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1406,"":"io task","client":"0","":"downstairs","session_id":"fc797ca8-3dcc-47b7-bb8b-3616c07a67a5"} {"msg":"ds_connection connected from Ok(127.0.0.1:63699)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:14.630538685Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1406,"":"io task","client":"1","":"downstairs","session_id":"fc797ca8-3dcc-47b7-bb8b-3616c07a67a5"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:14.631319971Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1406,"client":"0","":"downstairs","session_id":"fc797ca8-3dcc-47b7-bb8b-3616c07a67a5"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:14.631352039Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1406,"client":"1","":"downstairs","session_id":"fc797ca8-3dcc-47b7-bb8b-3616c07a67a5"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:14.631368257Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1406,"client":"2","":"downstairs","session_id":"fc797ca8-3dcc-47b7-bb8b-3616c07a67a5"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:14.631943378Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1406,"client":"0","":"downstairs","session_id":"fc797ca8-3dcc-47b7-bb8b-3616c07a67a5"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:14.632105507Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1406,"client":"1","":"downstairs","session_id":"fc797ca8-3dcc-47b7-bb8b-3616c07a67a5"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:14.632132985Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1406,"client":"2","":"downstairs","session_id":"fc797ca8-3dcc-47b7-bb8b-3616c07a67a5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:14.632444283Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1406,"session_id":"fc797ca8-3dcc-47b7-bb8b-3616c07a67a5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:14.632706285Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1406,"session_id":"fc797ca8-3dcc-47b7-bb8b-3616c07a67a5"} {"msg":"[0]R flush_numbers[0..12]: [96, 94, 103, 103, 96, 96, 102, 102, 96, 96, 104, 103]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:14.632832537Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1406,"":"downstairs","":"downstairs","session_id":"fc797ca8-3dcc-47b7-bb8b-3616c07a67a5"} {"msg":"[0]R generation[0..12]: [50, 48, 54, 54, 50, 50, 54, 54, 50, 50, 55, 54]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:14.632853635Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1406,"":"downstairs","":"downstairs","session_id":"fc797ca8-3dcc-47b7-bb8b-3616c07a67a5"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:14.632869604Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1406,"":"downstairs","":"downstairs","session_id":"fc797ca8-3dcc-47b7-bb8b-3616c07a67a5"} {"msg":"[1]R flush_numbers[0..12]: [96, 94, 103, 103, 96, 96, 102, 102, 96, 96, 104, 103]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:14.632883763Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1406,"":"downstairs","":"downstairs","session_id":"fc797ca8-3dcc-47b7-bb8b-3616c07a67a5"} {"msg":"[1]R generation[0..12]: [50, 48, 54, 54, 50, 50, 54, 54, 50, 50, 55, 54]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:14.632897432Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1406,"":"downstairs","":"downstairs","session_id":"fc797ca8-3dcc-47b7-bb8b-3616c07a67a5"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:14.632911171Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1406,"":"downstairs","":"downstairs","session_id":"fc797ca8-3dcc-47b7-bb8b-3616c07a67a5"} {"msg":"[2]R flush_numbers[0..12]: [96, 94, 103, 103, 96, 96, 102, 102, 96, 96, 104, 103]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:14.6329247Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1406,"":"downstairs","":"downstairs","session_id":"fc797ca8-3dcc-47b7-bb8b-3616c07a67a5"} {"msg":"[2]R generation[0..12]: [50, 48, 54, 54, 50, 50, 54, 54, 50, 50, 55, 54]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:14.632938299Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1406,"":"downstairs","":"downstairs","session_id":"fc797ca8-3dcc-47b7-bb8b-3616c07a67a5"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:14.632952098Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1406,"":"downstairs","":"downstairs","session_id":"fc797ca8-3dcc-47b7-bb8b-3616c07a67a5"} {"msg":"Max found gen is 56","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:14.632964967Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1406,"":"downstairs","":"downstairs","session_id":"fc797ca8-3dcc-47b7-bb8b-3616c07a67a5"} {"msg":"Generation requested: 56 >= found:56","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:14.632978196Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1406,"":"downstairs","":"downstairs","session_id":"fc797ca8-3dcc-47b7-bb8b-3616c07a67a5"} {"msg":"Next flush: 105","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:14.632994635Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1406,"":"downstairs","":"downstairs","session_id":"fc797ca8-3dcc-47b7-bb8b-3616c07a67a5"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:14.633009134Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1406,"":"downstairs","":"downstairs","session_id":"fc797ca8-3dcc-47b7-bb8b-3616c07a67a5"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-12-12T20:35:14.633022773Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1406,"session_id":"fc797ca8-3dcc-47b7-bb8b-3616c07a67a5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:14.633037822Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1406,"client":"0","":"downstairs","session_id":"fc797ca8-3dcc-47b7-bb8b-3616c07a67a5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:14.633051361Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1406,"client":"1","":"downstairs","session_id":"fc797ca8-3dcc-47b7-bb8b-3616c07a67a5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:14.63306437Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1406,"client":"2","":"downstairs","session_id":"fc797ca8-3dcc-47b7-bb8b-3616c07a67a5"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:14.633077809Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1406,"session_id":"fc797ca8-3dcc-47b7-bb8b-3616c07a67a5"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:14.633089989Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1406,"session_id":"fc797ca8-3dcc-47b7-bb8b-3616c07a67a5"} {"msg":"35b71f1f-bfaf-4afc-91ff-cdbfa1dc8faa is now active with session: fc797ca8-3dcc-47b7-bb8b-3616c07a67a5","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:14.633102618Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1406,"session_id":"fc797ca8-3dcc-47b7-bb8b-3616c07a67a5"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:14.633114587Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1406,"session_id":"fc797ca8-3dcc-47b7-bb8b-3616c07a67a5"} {"msg":"The guest has finished waiting for activation with:56","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:14.633127446Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1406} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 54 len 16384 data: 5 7 9 10 02/30 Flush 03/30 Flush 04/30 Flush 05/30 Write block 517 len 36864 data: 6 9 11 12 10 9 11 10 9 06/30 Write block 556 len 16384 data: 7 7 6 7 07/30 Read block 308 len 36864 08/30 Read block 82 len 40960 09/30 Read block 90 len 12288 10/30 Read block 206 len 28672 11/30 Flush 12/30 Write block 102 len 24576 data: 7 7 7 7 5 8 13/30 Read block 46 len 36864 14/30 Write block 402 len 4096 data: 9 15/30 Read block 197 len 8192 16/30 Read block 588 len 28672 17/30 Read block 168 len 36864 18/30 Read block 156 len 28672 19/30 Write block 448 len 8192 data: 4 5 20/30 Read block 573 len 28672 21/30 Write block 120 len 28672 data: 5 6 6 6 5 5 5 22/30 Read block 457 len 4096 23/30 Read block 577 len 12288 24/30 Write block 38 len 16384 data: 10 11 9 8 25/30 Read block 489 len 16384 26/30 Write block 458 len 24576 data: 4 4 5 5 5 5 27/30 Read block 417 len 20480 28/30 Read block 151 len 16384 29/30 Read block 393 len 8192 30/30 Write block 226 len 32768 data: 6 7 7 9 10 9 8 8 ---------------------------------------------------------------- Crucible gen:56 GIO:true work queues: Upstairs:0 downstairs:19 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1011 Acked Write 6 Done Done Done false 1012 Acked Read 9 Done Done Done false 1013 Acked Write 1 Done Done Done false 1014 Acked Read 2 Done Done Done false 1015 Acked Read 7 Done Done Done false 1016 Acked Read 9 Done Done Done false 1017 Acked Read 7 Done Done Done false 1018 Acked Write 2 Done Done Done false 1019 Acked Read 7 Done Done Done false 1020 Acked Write 7 Done Done Done false 1021 Acked Read 1 Done Done Done false 1022 Acked Read 3 Done Done Done false 1023 Acked Write 4 Done Done Done false 1024 Acked Read 4 Done Done Done false 1025 Acked Write 6 Done Done Done false 1026 Acked Read 5 Done Done Done false 1027 Acked Read 4 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 18 18 18 54 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1010 1010 1010 Downstairs last five completed: 1010 1009 1008 1007 1006 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:35:14.653209351Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1406} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 50 50 50 96 96 96 F F F 1 020-039 48 48 48 94 94 94 T T T 2 040-059 56 56 56 105 105 105 T T T 3 060-079 54 54 54 103 103 103 F F F 4 080-099 50 50 50 96 96 96 F F F 5 100-119 50 50 50 96 96 96 T T T 6 120-139 54 54 54 102 102 102 T T T 7 140-159 54 54 54 102 102 102 F F F 8 160-179 50 50 50 96 96 96 F F F 9 180-199 50 50 50 96 96 96 F F F 10 200-219 55 55 55 104 104 104 F F F 11 220-239 54 54 54 103 103 103 T T T 12 240-259 55 55 55 104 104 104 F F F 13 260-279 50 50 50 96 96 96 F F F 14 280-299 54 54 54 102 102 102 F F F 15 300-319 50 50 50 96 96 96 F F F 16 320-339 54 54 54 103 103 103 F F F 17 340-359 54 54 54 102 102 102 F F F 18 360-379 49 49 49 95 95 95 F F F 19 380-399 52 52 52 100 100 100 F F F 20 400-419 53 53 53 101 101 101 T T T 21 420-439 54 54 54 102 102 102 F F F 22 440-459 55 55 55 104 104 104 T T T 23 460-479 46 46 46 90 90 90 T T T 24 480-499 54 54 54 102 102 102 F F F 25 500-519 56 56 56 108 108 108 F F F 26 520-539 56 56 56 108 108 108 F F F 27 540-559 56 56 56 108 108 108 F F F 28 560-579 55 55 55 104 104 104 F F F 29 580-599 55 55 55 104 104 104 F F F Max gen: 56, Max flush: 108 A Difference in extent metadata is expected here On loop 28 Verifying data now No change after verify Loop: 28 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 50 50 50 96 96 96 F F F 1 020-039 57 57 57 109 109 109 F F F 2 040-059 57 57 57 109 109 109 F F F 3 060-079 54 54 54 103 103 103 F F F 4 080-099 50 50 50 96 96 96 F F F 5 100-119 57 57 57 109 109 109 F F F 6 120-139 57 57 57 109 109 109 F F F 7 140-159 54 54 54 102 102 102 F F F 8 160-179 50 50 50 96 96 96 F F F 9 180-199 50 50 50 96 96 96 F F F 10 200-219 55 55 55 104 104 104 F F F 11 220-239 57 57 57 109 109 109 F F F 12 240-259 55 55 55 104 104 104 F F F 13 260-279 50 50 50 96 96 96 F F F 14 280-299 54 54 54 102 102 102 F F F 15 300-319 50 50 50 96 96 96 F F F 16 320-339 54 54 54 103 103 103 F F F 17 340-359 54 54 54 102 102 102 F F F 18 360-379 49 49 49 95 95 95 F F F 19 380-399 52 52 52 100 100 100 F F F 20 400-419 57 57 57 109 109 109 F F F 21 420-439 54 54 54 102 102 102 F F F 22 440-459 57 57 57 109 109 109 F F F 23 460-479 57 57 57 109 109 109 F F F 24 480-499 54 54 54 102 102 102 F F F 25 500-519 56 56 56 108 108 108 F F F 26 520-539 56 56 56 108 108 108 F F F 27 540-559 56 56 56 108 108 108 F F F 28 560-579 55 55 55 104 104 104 F F F 29 580-599 55 55 55 104 104 104 F F F Max gen: 57, Max flush: 109 Begin loop 29 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:18.944091039Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1414,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:18.989799357Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1414,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:18.99048622Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1414} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"26e7d501b33f47379e53601720353c2983e7654b\",\n git_commit_timestamp: \"2024-12-12T20:16:40.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:18.990522867Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1414} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:18.990537086Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1414} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:18.990551055Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1414} {"msg":"Crucible 8479b163-b349-449e-b6ed-761539e378cc has session id: 1cb3b0f4-54c6-4b4a-84d1-d115a784a06d","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:18.990990305Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1414,"session_id":"1cb3b0f4-54c6-4b4a-84d1-d115a784a06d"} {"msg":"Upstairs opts: Upstairs UUID: 8479b163-b349-449e-b6ed-761539e378cc, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:18.991031162Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1414,"session_id":"1cb3b0f4-54c6-4b4a-84d1-d115a784a06d"} {"msg":"Crucible stats registered with UUID: 8479b163-b349-449e-b6ed-761539e378cc","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:18.991047391Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1414,"session_id":"1cb3b0f4-54c6-4b4a-84d1-d115a784a06d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:19.021604564Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1414,"task":"crutest"} {"msg":"The guest has requested activation with gen:58","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:19.021631412Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1414} {"msg":"8479b163-b349-449e-b6ed-761539e378cc active request set","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:19.021694508Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1414,"session_id":"1cb3b0f4-54c6-4b4a-84d1-d115a784a06d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:19.021728196Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1414,"client":"0","":"downstairs","session_id":"1cb3b0f4-54c6-4b4a-84d1-d115a784a06d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:19.021758134Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1414,"client":"0","":"downstairs","session_id":"1cb3b0f4-54c6-4b4a-84d1-d115a784a06d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:19.021776132Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1414,"client":"1","":"downstairs","session_id":"1cb3b0f4-54c6-4b4a-84d1-d115a784a06d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:19.021795991Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1414,"client":"1","":"downstairs","session_id":"1cb3b0f4-54c6-4b4a-84d1-d115a784a06d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:19.021826729Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1414,"client":"2","":"downstairs","session_id":"1cb3b0f4-54c6-4b4a-84d1-d115a784a06d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:19.021860916Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1414,"client":"2","":"downstairs","session_id":"1cb3b0f4-54c6-4b4a-84d1-d115a784a06d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:19.021893214Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1414,"":"io task","client":"0","":"downstairs","session_id":"1cb3b0f4-54c6-4b4a-84d1-d115a784a06d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:19.021922042Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1414,"":"io task","client":"2","":"downstairs","session_id":"1cb3b0f4-54c6-4b4a-84d1-d115a784a06d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:19.02194876Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1414,"":"io task","client":"1","":"downstairs","session_id":"1cb3b0f4-54c6-4b4a-84d1-d115a784a06d"} {"msg":"ds_connection connected from Ok(127.0.0.1:65431)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:19.021974679Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1414,"":"io task","client":"2","":"downstairs","session_id":"1cb3b0f4-54c6-4b4a-84d1-d115a784a06d"} {"msg":"ds_connection connected from Ok(127.0.0.1:36127)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:19.02210022Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1414,"":"io task","client":"1","":"downstairs","session_id":"1cb3b0f4-54c6-4b4a-84d1-d115a784a06d"} {"msg":"ds_connection connected from Ok(127.0.0.1:48205)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:19.022140767Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1414,"":"io task","client":"0","":"downstairs","session_id":"1cb3b0f4-54c6-4b4a-84d1-d115a784a06d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:19.022627223Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1414,"client":"0","":"downstairs","session_id":"1cb3b0f4-54c6-4b4a-84d1-d115a784a06d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:19.022656541Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1414,"client":"2","":"downstairs","session_id":"1cb3b0f4-54c6-4b4a-84d1-d115a784a06d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:19.022780653Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1414,"client":"1","":"downstairs","session_id":"1cb3b0f4-54c6-4b4a-84d1-d115a784a06d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:19.023269679Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1414,"client":"0","":"downstairs","session_id":"1cb3b0f4-54c6-4b4a-84d1-d115a784a06d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:19.023347364Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1414,"client":"1","":"downstairs","session_id":"1cb3b0f4-54c6-4b4a-84d1-d115a784a06d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:19.023410489Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1414,"client":"2","":"downstairs","session_id":"1cb3b0f4-54c6-4b4a-84d1-d115a784a06d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:19.023822761Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1414,"session_id":"1cb3b0f4-54c6-4b4a-84d1-d115a784a06d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:19.023930604Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1414,"session_id":"1cb3b0f4-54c6-4b4a-84d1-d115a784a06d"} {"msg":"[0]R flush_numbers[0..12]: [96, 109, 109, 103, 96, 109, 109, 102, 96, 96, 104, 109]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:19.024050556Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1414,"":"downstairs","":"downstairs","session_id":"1cb3b0f4-54c6-4b4a-84d1-d115a784a06d"} {"msg":"[0]R generation[0..12]: [50, 57, 57, 54, 50, 57, 57, 54, 50, 50, 55, 57]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:19.024071084Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1414,"":"downstairs","":"downstairs","session_id":"1cb3b0f4-54c6-4b4a-84d1-d115a784a06d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:19.024085653Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1414,"":"downstairs","":"downstairs","session_id":"1cb3b0f4-54c6-4b4a-84d1-d115a784a06d"} {"msg":"[1]R flush_numbers[0..12]: [96, 109, 109, 103, 96, 109, 109, 102, 96, 96, 104, 109]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:19.024099692Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1414,"":"downstairs","":"downstairs","session_id":"1cb3b0f4-54c6-4b4a-84d1-d115a784a06d"} {"msg":"[1]R generation[0..12]: [50, 57, 57, 54, 50, 57, 57, 54, 50, 50, 55, 57]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:19.024113341Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1414,"":"downstairs","":"downstairs","session_id":"1cb3b0f4-54c6-4b4a-84d1-d115a784a06d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:19.02412738Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1414,"":"downstairs","":"downstairs","session_id":"1cb3b0f4-54c6-4b4a-84d1-d115a784a06d"} {"msg":"[2]R flush_numbers[0..12]: [96, 109, 109, 103, 96, 109, 109, 102, 96, 96, 104, 109]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:19.024145069Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1414,"":"downstairs","":"downstairs","session_id":"1cb3b0f4-54c6-4b4a-84d1-d115a784a06d"} {"msg":"[2]R generation[0..12]: [50, 57, 57, 54, 50, 57, 57, 54, 50, 50, 55, 57]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:19.024159198Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1414,"":"downstairs","":"downstairs","session_id":"1cb3b0f4-54c6-4b4a-84d1-d115a784a06d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:19.024173127Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1414,"":"downstairs","":"downstairs","session_id":"1cb3b0f4-54c6-4b4a-84d1-d115a784a06d"} {"msg":"Max found gen is 58","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:19.024186096Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1414,"":"downstairs","":"downstairs","session_id":"1cb3b0f4-54c6-4b4a-84d1-d115a784a06d"} {"msg":"Generation requested: 58 >= found:58","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:19.024199205Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1414,"":"downstairs","":"downstairs","session_id":"1cb3b0f4-54c6-4b4a-84d1-d115a784a06d"} {"msg":"Next flush: 110","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:19.024212134Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1414,"":"downstairs","":"downstairs","session_id":"1cb3b0f4-54c6-4b4a-84d1-d115a784a06d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"Wait for a query_work_queue command to finish before sending IO time":"2024-12-12T20:35:19.024224993Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1414,"":"downstairs","":"downstairs","session_id":"1cb3b0f4-54c6-4b4a-84d1-d115a784a06d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:19.024242792Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1414,"session_id":"1cb3b0f4-54c6-4b4a-84d1-d115a784a06d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:19.024254881Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1414,"client":"0","":"downstairs","session_id":"1cb3b0f4-54c6-4b4a-84d1-d115a784a06d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:19.024268201Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1414,"client":"1","":"downstairs","session_id":"1cb3b0f4-54c6-4b4a-84d1-d115a784a06d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:19.02428128Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1414,"client":"2","":"downstairs","session_id":"1cb3b0f4-54c6-4b4a-84d1-d115a784a06d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:19.024294639Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1414,"session_id":"1cb3b0f4-54c6-4b4a-84d1-d115a784a06d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:19.024306868Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1414,"session_id":"1cb3b0f4-54c6-4b4a-84d1-d115a784a06d"} {"msg":"8479b163-b349-449e-b6ed-761539e378cc is now active with session: 1cb3b0f4-54c6-4b4a-84d1-d115a784a06d","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:19.024319627Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1414,"session_id":"1cb3b0f4-54c6-4b4a-84d1-d115a784a06d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:19.024331456Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1414,"session_id":"1cb3b0f4-54c6-4b4a-84d1-d115a784a06d"} {"msg":"The guest has finished waiting for activation with:58","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:19.024343535Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1414} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 590 len 28672 data: 6 5 5 5 4 3 2 02/30 Read block 235 len 32768 03/30 Read block 172 len 20480 04/30 Flush 05/30 Write block 354 len 40960 data: 6 6 5 7 5 3 3 3 4 5 06/30 Write block 189 len 16384 data: 7 7 7 7 07/30 Read block 249 len 40960 08/30 Write block 117 len 36864 data: 4 4 3 6 7 7 7 6 6 09/30 Read block 93 len 8192 10/30 Read block 135 len 16384 11/30 Flush 12/30 Write block 223 len 28672 data: 4 4 5 7 8 8 10 13/30 Write block 45 len 12288 data: 6 6 6 14/30 Flush 15/30 Write block 319 len 24576 data: 6 6 5 6 5 3 16/30 Write block 473 len 36864 data: 3 3 4 4 3 4 3 2 3 17/30 Write block 435 len 12288 data: 5 4 4 18/30 Read block 150 len 8192 19/30 Read block 321 len 20480 20/30 Read block 249 len 40960 21/30 Write block 291 len 20480 data: 7 7 8 8 8 22/30 Flush 23/30 Write block 459 len 40960 data: 5 6 6 6 6 4 4 3 3 3 24/30 Write block 569 len 4096 data: 4 25/30 Write block 350 len 12288 data: 6 3 4 26/30 Write block 298 len 12288 data: 5 5 6 27/30 Read block 148 len 24576 28/30 Read block 498 len 24576 29/30 Write block 100 len 32768 data: 5 6 8 8 8 8 6 9 30/30 Read block 91 len 8192 ---------------------------------------------------------------- Crucible gen:58 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 10 Done Done Done false 1023 Acked Write 1 Done Done Done false 1024 Acked Write 3 Done Done Done false 1025 Acked Write 3 Done Done Done false 1026 Acked Read 6 Done Done Done false 1027 Acked Read 6 Done Done Done false 1028 Acked Write 8 Done Done Done false 1029 Acked Read 2 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 8 8 8 24 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1021 1021 1021 Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:35:19.044117132Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1414} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 50 50 50 96 96 96 F F F 1 020-039 57 57 57 109 109 109 F F F 2 040-059 58 58 58 112 112 112 F F F 3 060-079 54 54 54 103 103 103 F F F 4 080-099 50 50 50 96 96 96 F F F 5 100-119 58 58 58 111 111 111 T T T 6 120-139 58 58 58 111 111 111 F F F 7 140-159 54 54 54 102 102 102 F F F 8 160-179 50 50 50 96 96 96 F F F 9 180-199 58 58 58 111 111 111 F F F 10 200-219 55 55 55 104 104 104 F F F 11 220-239 58 58 58 112 112 112 F F F 12 240-259 55 55 55 104 104 104 F F F 13 260-279 50 50 50 96 96 96 F F F 14 280-299 58 58 58 113 113 113 T T T 15 300-319 58 58 58 113 113 113 T T T 16 320-339 58 58 58 113 113 113 F F F 17 340-359 58 58 58 111 111 111 T T T 18 360-379 58 58 58 111 111 111 F F F 19 380-399 52 52 52 100 100 100 F F F 20 400-419 57 57 57 109 109 109 F F F 21 420-439 58 58 58 113 113 113 F F F 22 440-459 57 57 57 109 109 109 T T T 23 460-479 58 58 58 113 113 113 T T T 24 480-499 58 58 58 113 113 113 F F F 25 500-519 56 56 56 108 108 108 F F F 26 520-539 56 56 56 108 108 108 F F F 27 540-559 56 56 56 108 108 108 F F F 28 560-579 55 55 55 104 104 104 T T T 29 580-599 58 58 58 110 110 110 F F F Max gen: 58, Max flush: 113 A Difference in extent metadata is expected here On loop 29 Verifying data now No change after verify Loop: 29 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 50 50 50 96 96 96 F F F 1 020-039 57 57 57 109 109 109 F F F 2 040-059 58 58 58 112 112 112 F F F 3 060-079 54 54 54 103 103 103 F F F 4 080-099 50 50 50 96 96 96 F F F 5 100-119 59 59 59 114 114 114 F F F 6 120-139 58 58 58 111 111 111 F F F 7 140-159 54 54 54 102 102 102 F F F 8 160-179 50 50 50 96 96 96 F F F 9 180-199 58 58 58 111 111 111 F F F 10 200-219 55 55 55 104 104 104 F F F 11 220-239 58 58 58 112 112 112 F F F 12 240-259 55 55 55 104 104 104 F F F 13 260-279 50 50 50 96 96 96 F F F 14 280-299 59 59 59 114 114 114 F F F 15 300-319 59 59 59 114 114 114 F F F 16 320-339 58 58 58 113 113 113 F F F 17 340-359 59 59 59 114 114 114 F F F 18 360-379 58 58 58 111 111 111 F F F 19 380-399 52 52 52 100 100 100 F F F 20 400-419 57 57 57 109 109 109 F F F 21 420-439 58 58 58 113 113 113 F F F 22 440-459 59 59 59 114 114 114 F F F 23 460-479 59 59 59 114 114 114 F F F 24 480-499 58 58 58 113 113 113 F F F 25 500-519 56 56 56 108 108 108 F F F 26 520-539 56 56 56 108 108 108 F F F 27 540-559 56 56 56 108 108 108 F F F 28 560-579 59 59 59 114 114 114 F F F 29 580-599 58 58 58 110 110 110 F F F Max gen: 59, Max flush: 114 Begin loop 30 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:23.326502896Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1422,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:23.371495333Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1422,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:23.37211514Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1422} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"26e7d501b33f47379e53601720353c2983e7654b\",\n git_commit_timestamp: \"2024-12-12T20:16:40.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:23.372145578Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1422} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:23.372158768Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1422} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:23.372172607Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1422} {"msg":"Crucible 13c8b8ef-6319-403a-a250-0f22fb49e2b6 has session id: 5f0e02a2-9749-44c7-aab0-73b4d9667c89","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:23.3725647Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1422,"session_id":"5f0e02a2-9749-44c7-aab0-73b4d9667c89"} {"msg":"Upstairs opts: Upstairs UUID: 13c8b8ef-6319-403a-a250-0f22fb49e2b6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:23.372608846Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1422,"session_id":"5f0e02a2-9749-44c7-aab0-73b4d9667c89"} {"msg":"Crucible stats registered with UUID: 13c8b8ef-6319-403a-a250-0f22fb49e2b6","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:23.372634565Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1422,"session_id":"5f0e02a2-9749-44c7-aab0-73b4d9667c89"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:23.404085726Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1422,"task":"crutest"} {"msg":"The guest has requested activation with gen:60","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:23.404118194Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1422} {"msg":"13c8b8ef-6319-403a-a250-0f22fb49e2b6 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:23.404194339Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1422,"session_id":"5f0e02a2-9749-44c7-aab0-73b4d9667c89"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:23.404216447Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1422,"client":"0","":"downstairs","session_id":"5f0e02a2-9749-44c7-aab0-73b4d9667c89"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:23.404245085Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1422,"client":"0","":"downstairs","session_id":"5f0e02a2-9749-44c7-aab0-73b4d9667c89"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:23.404262684Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1422,"client":"1","":"downstairs","session_id":"5f0e02a2-9749-44c7-aab0-73b4d9667c89"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:23.404280742Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1422,"client":"1","":"downstairs","session_id":"5f0e02a2-9749-44c7-aab0-73b4d9667c89"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:23.404449391Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1422,"client":"2","":"downstairs","session_id":"5f0e02a2-9749-44c7-aab0-73b4d9667c89"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:23.404528385Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1422,"client":"2","":"downstairs","session_id":"5f0e02a2-9749-44c7-aab0-73b4d9667c89"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:23.404558483Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1422,"":"io task","client":"0","":"downstairs","session_id":"5f0e02a2-9749-44c7-aab0-73b4d9667c89"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:23.404585141Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1422,"":"io task","client":"2","":"downstairs","session_id":"5f0e02a2-9749-44c7-aab0-73b4d9667c89"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:23.404771649Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1422,"":"io task","client":"1","":"downstairs","session_id":"5f0e02a2-9749-44c7-aab0-73b4d9667c89"} {"msg":"ds_connection connected from Ok(127.0.0.1:39299)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:23.404803337Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1422,"":"io task","client":"1","":"downstairs","session_id":"5f0e02a2-9749-44c7-aab0-73b4d9667c89"} {"msg":"ds_connection connected from Ok(127.0.0.1:62784)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:23.404828045Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1422,"":"io task","client":"2","":"downstairs","session_id":"5f0e02a2-9749-44c7-aab0-73b4d9667c89"} {"msg":"ds_connection connected from Ok(127.0.0.1:40288)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:23.404852673Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1422,"":"io task","client":"0","":"downstairs","session_id":"5f0e02a2-9749-44c7-aab0-73b4d9667c89"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:23.405308652Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1422,"client":"2","":"downstairs","session_id":"5f0e02a2-9749-44c7-aab0-73b4d9667c89"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:23.40534016Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1422,"client":"0","":"downstairs","session_id":"5f0e02a2-9749-44c7-aab0-73b4d9667c89"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:23.405455672Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1422,"client":"1","":"downstairs","session_id":"5f0e02a2-9749-44c7-aab0-73b4d9667c89"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:23.4059216Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1422,"client":"2","":"downstairs","session_id":"5f0e02a2-9749-44c7-aab0-73b4d9667c89"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:23.406038181Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1422,"client":"1","":"downstairs","session_id":"5f0e02a2-9749-44c7-aab0-73b4d9667c89"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:23.40605874Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1422,"client":"0","":"downstairs","session_id":"5f0e02a2-9749-44c7-aab0-73b4d9667c89"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:23.40649153Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1422,"session_id":"5f0e02a2-9749-44c7-aab0-73b4d9667c89"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:23.406594033Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1422,"session_id":"5f0e02a2-9749-44c7-aab0-73b4d9667c89"} {"msg":"[0]R flush_numbers[0..12]: [96, 109, 112, 103, 96, 114, 111, 102, 96, 111, 104, 112]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:23.406667068Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1422,"":"downstairs","":"downstairs","session_id":"5f0e02a2-9749-44c7-aab0-73b4d9667c89"} {"msg":"[0]R generation[0..12]: [50, 57, 58, 54, 50, 59, 58, 54, 50, 58, 55, 58]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:23.406686697Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1422,"":"downstairs","":"downstairs","session_id":"5f0e02a2-9749-44c7-aab0-73b4d9667c89"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:23.406701586Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1422,"":"downstairs","":"downstairs","session_id":"5f0e02a2-9749-44c7-aab0-73b4d9667c89"} {"msg":"[1]R flush_numbers[0..12]: [96, 109, 112, 103, 96, 114, 111, 102, 96, 111, 104, 112]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:23.406719074Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1422,"":"downstairs","":"downstairs","session_id":"5f0e02a2-9749-44c7-aab0-73b4d9667c89"} {"msg":"[1]R generation[0..12]: [50, 57, 58, 54, 50, 59, 58, 54, 50, 58, 55, 58]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:23.406733344Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1422,"":"downstairs","":"downstairs","session_id":"5f0e02a2-9749-44c7-aab0-73b4d9667c89"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:23.406747583Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1422,"":"downstairs","":"downstairs","session_id":"5f0e02a2-9749-44c7-aab0-73b4d9667c89"} {"msg":"[2]R flush_numbers[0..12]: [96, 109, 112, 103, 96, 114, 111, 102, 96, 111, 104, 112]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:23.406761371Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1422,"":"downstairs","":"downstairs","session_id":"5f0e02a2-9749-44c7-aab0-73b4d9667c89"} {"msg":"[2]R generation[0..12]: [50, 57, 58, 54, 50, 59, 58, 54, 50, 58, 55, 58]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:23.40677479Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1422,"":"downstairs","":"downstairs","session_id":"5f0e02a2-9749-44c7-aab0-73b4d9667c89"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:23.40678873Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1422,"":"downstairs","":"downstairs","session_id":"5f0e02a2-9749-44c7-aab0-73b4d9667c89"} {"msg":"Max found gen is 60","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:23.406801649Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1422,"":"downstairs","":"downstairs","session_id":"5f0e02a2-9749-44c7-aab0-73b4d9667c89"} {"msg":"Generation requested: 60 >= found:60","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:23.406814828Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1422,"":"downstairs","":"downstairs","session_id":"5f0e02a2-9749-44c7-aab0-73b4d9667c89"} {"msg":"Next flush: 115","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:23.406827847Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1422,"":"downstairs","Wait for a query_work_queue command to finish before sending IO ":"downstairs","session_id":"5f0e02a2-9749-44c7-aab0-73b4d9667c89"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:23.406847236Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1422,"":"downstairs","":"downstairs","session_id":"5f0e02a2-9749-44c7-aab0-73b4d9667c89"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:23.406860695Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1422,"session_id":"5f0e02a2-9749-44c7-aab0-73b4d9667c89"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:23.406881083Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1422,"client":"0","":"downstairs","session_id":"5f0e02a2-9749-44c7-aab0-73b4d9667c89"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:23.406895902Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1422,"client":"1","":"downstairs","session_id":"5f0e02a2-9749-44c7-aab0-73b4d9667c89"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:23.406909481Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1422,"client":"2","":"downstairs","session_id":"5f0e02a2-9749-44c7-aab0-73b4d9667c89"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:23.40692293Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1422,"session_id":"5f0e02a2-9749-44c7-aab0-73b4d9667c89"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:23.406937069Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1422,"session_id":"5f0e02a2-9749-44c7-aab0-73b4d9667c89"} {"msg":"13c8b8ef-6319-403a-a250-0f22fb49e2b6 is now active with session: 5f0e02a2-9749-44c7-aab0-73b4d9667c89","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:23.406950009Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1422,"session_id":"5f0e02a2-9749-44c7-aab0-73b4d9667c89"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:23.406962268Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1422,"session_id":"5f0e02a2-9749-44c7-aab0-73b4d9667c89"} {"msg":"The guest has finished waiting for activation with:60","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:23.406975037Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1422} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 417 len 36864 data: 5 7 6 6 6 7 6 6 6 02/30 Read block 488 len 8192 03/30 Write block 483 len 16384 data: 3 6 7 7 04/30 Read block 143 len 24576 05/30 Read block 273 len 36864 06/30 Write block 579 len 4096 data: 6 07/30 Read block 189 len 8192 08/30 Write block 87 len 16384 data: 8 8 7 6 09/30 Read block 277 len 24576 10/30 Read block 32 len 20480 11/30 Write block 395 len 24576 data: 6 5 5 7 7 6 12/30 Read block 35 len 28672 13/30 Read block 567 len 36864 14/30 Write block 20 len 16384 data: 6 6 6 5 15/30 Read block 594 len 4096 16/30 Read block 278 len 40960 17/30 Read block 371 len 8192 18/30 Read block 387 len 4096 19/30 Read block 324 len 36864 20/30 Read block 100 len 20480 21/30 Read block 408 len 4096 22/30 Flush 23/30 Write block 81 len 20480 data: 5 4 5 6 7 24/30 Read block 347 len 20480 25/30 Write block 573 len 28672 data: 3 3 3 4 4 5 7 26/30 Read block 568 len 28672 27/30 Write block 449 len 24576 data: 6 6 7 7 7 7 28/30 Read block 434 len 24576 29/30 Read block 595 len 4096 30/30 Write block 362 len 32768 data: 5 6 5 4 5 4 3 4 ---------------------------------------------------------------- Crucible gen:60 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 5 Done Done Done false 1023 Acked Read 5 Done Done Done false 1024 Acked Write 7 Done Done Done false 1025 Acked Read 7 Done Done Done false 1026 Acked Write 6 Done Done Done false 1027 Acked Read 6 Done Done Done false 1028 Acked Read 1 Done Done Done false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 7 7 7 21 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1021 1021 1021 Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:35:23.426812449Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1422} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 50 50 50 96 96 96 F F F 1 020-039 60 60 60 115 115 115 F F F 2 040-059 58 58 58 112 112 112 F F F 3 060-079 54 54 54 103 103 103 F F F 4 080-099 60 60 60 115 115 115 T T T 5 100-119 59 59 59 114 114 114 F F F 6 120-139 58 58 58 111 111 111 F F F 7 140-159 54 54 54 102 102 102 F F F 8 160-179 50 50 50 96 96 96 F F F 9 180-199 58 58 58 111 111 111 F F F 10 200-219 55 55 55 104 104 104 F F F 11 220-239 58 58 58 112 112 112 F F F 12 240-259 55 55 55 104 104 104 F F F 13 260-279 50 50 50 96 96 96 F F F 14 280-299 59 59 59 114 114 114 F F F 15 300-319 59 59 59 114 114 114 F F F 16 320-339 58 58 58 113 113 113 F F F 17 340-359 59 59 59 114 114 114 F F F 18 360-379 58 58 58 111 111 111 T T T 19 380-399 60 60 60 115 115 115 F F F 20 400-419 60 60 60 115 115 115 F F F 21 420-439 60 60 60 115 115 115 F F F 22 440-459 59 59 59 114 114 114 T T T 23 460-479 59 59 59 114 114 114 F F F 24 480-499 60 60 60 115 115 115 F F F 25 500-519 56 56 56 108 108 108 F F F 26 520-539 56 56 56 108 108 108 F F F 27 540-559 56 56 56 108 108 108 F F F 28 560-579 60 60 60 115 115 115 T T T 29 580-599 58 58 58 110 110 110 F F F Max gen: 60, Max flush: 115 A Difference in extent metadata is expected here On loop 30 Verifying data now No change after verify Loop: 30 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 50 50 50 96 96 96 F F F 1 020-039 60 60 60 115 115 115 F F F 2 040-059 58 58 58 112 112 112 F F F 3 060-079 54 54 54 103 103 103 F F F 4 080-099 61 61 61 116 116 116 F F F 5 100-119 59 59 59 114 114 114 F F F 6 120-139 58 58 58 111 111 111 F F F 7 140-159 54 54 54 102 102 102 F F F 8 160-179 50 50 50 96 96 96 F F F 9 180-199 58 58 58 111 111 111 F F F 10 200-219 55 55 55 104 104 104 F F F 11 220-239 58 58 58 112 112 112 F F F 12 240-259 55 55 55 104 104 104 F F F 13 260-279 50 50 50 96 96 96 F F F 14 280-299 59 59 59 114 114 114 F F F 15 300-319 59 59 59 114 114 114 F F F 16 320-339 58 58 58 113 113 113 F F F 17 340-359 59 59 59 114 114 114 F F F 18 360-379 61 61 61 116 116 116 F F F 19 380-399 60 60 60 115 115 115 F F F 20 400-419 60 60 60 115 115 115 F F F 21 420-439 60 60 60 115 115 115 F F F 22 440-459 61 61 61 116 116 116 F F F 23 460-479 59 59 59 114 114 114 F F F 24 480-499 60 60 60 115 115 115 F F F 25 500-519 56 56 56 108 108 108 F F F 26 520-539 56 56 56 108 108 108 F F F 27 540-559 56 56 56 108 108 108 F F F 28 560-579 61 61 61 116 116 116 F F F 29 580-599 58 58 58 110 110 110 F F F Max gen: 61, Max flush: 116 Begin loop 31 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:27.692816402Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1430,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:35:27.736181332Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1430,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:27.736540167Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1430,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:27.768821221Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1430,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:27.769402971Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1430} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"26e7d501b33f47379e53601720353c2983e7654b\",\n git_commit_timestamp: \"2024-12-12T20:16:40.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:27.769437819Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1430} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:27.769451408Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1430} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:27.769465517Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1430} {"msg":"Crucible 6ce9a6cb-8671-4067-acc4-4614f98248bf has session id: 9a05bc68-28d3-44f3-86b1-55a9cdeb1494","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:27.76986193Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1430,"session_id":"9a05bc68-28d3-44f3-86b1-55a9cdeb1494"} {"msg":"Upstairs opts: Upstairs UUID: 6ce9a6cb-8671-4067-acc4-4614f98248bf, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:27.769907117Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1430,"session_id":"9a05bc68-28d3-44f3-86b1-55a9cdeb1494"} {"msg":"Crucible stats registered with UUID: 6ce9a6cb-8671-4067-acc4-4614f98248bf","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:27.769929235Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1430,"session_id":"9a05bc68-28d3-44f3-86b1-55a9cdeb1494"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:27.800988403Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1430,"task":"crutest"} {"msg":"The guest has requested activation with gen:62","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:27.801024171Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1430} {"msg":"6ce9a6cb-8671-4067-acc4-4614f98248bf active request set","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:27.801110394Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1430,"session_id":"9a05bc68-28d3-44f3-86b1-55a9cdeb1494"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:27.801128903Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1430,"client":"0","":"downstairs","session_id":"9a05bc68-28d3-44f3-86b1-55a9cdeb1494"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:27.801153131Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1430,"client":"0","":"downstairs","session_id":"9a05bc68-28d3-44f3-86b1-55a9cdeb1494"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:27.801168501Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1430,"client":"1","":"downstairs","session_id":"9a05bc68-28d3-44f3-86b1-55a9cdeb1494"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:27.80118354Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1430,"client":"1","":"downstairs","session_id":"9a05bc68-28d3-44f3-86b1-55a9cdeb1494"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:27.801200518Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1430,"client":"2","":"downstairs","session_id":"9a05bc68-28d3-44f3-86b1-55a9cdeb1494"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:27.801215677Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1430,"client":"2","":"downstairs","session_id":"9a05bc68-28d3-44f3-86b1-55a9cdeb1494"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:27.801229326Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1430,"":"io task","client":"2","":"downstairs","session_id":"9a05bc68-28d3-44f3-86b1-55a9cdeb1494"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:27.801243315Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1430,"":"io task","client":"0","":"downstairs","session_id":"9a05bc68-28d3-44f3-86b1-55a9cdeb1494"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:27.80131862Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1430,"":"io task","client":"1","":"downstairs","session_id":"9a05bc68-28d3-44f3-86b1-55a9cdeb1494"} {"msg":"ds_connection connected from Ok(127.0.0.1:35841)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:27.801528476Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1430,"":"io task","client":"2","":"downstairs","session_id":"9a05bc68-28d3-44f3-86b1-55a9cdeb1494"} {"msg":"ds_connection connected from Ok(127.0.0.1:63687)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:27.801569093Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1430,"":"io task","client":"1","":"downstairs","session_id":"9a05bc68-28d3-44f3-86b1-55a9cdeb1494"} {"msg":"ds_connection connected from Ok(127.0.0.1:53971)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:27.801597271Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1430,"":"io task","client":"0","":"downstairs","session_id":"9a05bc68-28d3-44f3-86b1-55a9cdeb1494"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:27.802153253Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1430,"client":"1","":"downstairs","session_id":"9a05bc68-28d3-44f3-86b1-55a9cdeb1494"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:27.80218298Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1430,"client":"2","":"downstairs","session_id":"9a05bc68-28d3-44f3-86b1-55a9cdeb1494"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:27.802221028Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1430,"client":"0","":"downstairs","session_id":"9a05bc68-28d3-44f3-86b1-55a9cdeb1494"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:27.80276914Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1430,"client":"1","":"downstairs","session_id":"9a05bc68-28d3-44f3-86b1-55a9cdeb1494"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:27.802909641Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1430,"client":"0","":"downstairs","session_id":"9a05bc68-28d3-44f3-86b1-55a9cdeb1494"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:27.803027862Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1430,"client":"2","":"downstairs","session_id":"9a05bc68-28d3-44f3-86b1-55a9cdeb1494"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:27.803369979Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1430,"session_id":"9a05bc68-28d3-44f3-86b1-55a9cdeb1494"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:27.80349583Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1430,"session_id":"9a05bc68-28d3-44f3-86b1-55a9cdeb1494"} {"msg":"[0]R flush_numbers[0..12]: [96, 115, 112, 103, 116, 114, 111, 102, 96, 111, 104, 112]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:27.803633191Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1430,"":"downstairs","":"downstairs","session_id":"9a05bc68-28d3-44f3-86b1-55a9cdeb1494"} {"msg":"[0]R generation[0..12]: [50, 60, 58, 54, 61, 59, 58, 54, 50, 58, 55, 58]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:27.803652939Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1430,"":"downstairs","":"downstairs","session_id":"9a05bc68-28d3-44f3-86b1-55a9cdeb1494"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:27.803667428Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1430,"":"downstairs","":"downstairs","session_id":"9a05bc68-28d3-44f3-86b1-55a9cdeb1494"} {"msg":"[1]R flush_numbers[0..12]: [96, 115, 112, 103, 116, 114, 111, 102, 96, 111, 104, 112]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:27.803681377Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1430,"":"downstairs","":"downstairs","session_id":"9a05bc68-28d3-44f3-86b1-55a9cdeb1494"} {"msg":"[1]R generation[0..12]: [50, 60, 58, 54, 61, 59, 58, 54, 50, 58, 55, 58]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:27.803695066Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1430,"":"downstairs","":"downstairs","session_id":"9a05bc68-28d3-44f3-86b1-55a9cdeb1494"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:27.803709135Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1430,"":"downstairs","":"downstairs","session_id":"9a05bc68-28d3-44f3-86b1-55a9cdeb1494"} {"msg":"[2]R flush_numbers[0..12]: [96, 115, 112, 103, 116, 114, 111, 102, 96, 111, 104, 112]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:27.803722804Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1430,"":"downstairs","":"downstairs","session_id":"9a05bc68-28d3-44f3-86b1-55a9cdeb1494"} {"msg":"[2]R generation[0..12]: [50, 60, 58, 54, 61, 59, 58, 54, 50, 58, 55, 58]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:27.803736204Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1430,"":"downstairs","":"downstairs","session_id":"9a05bc68-28d3-44f3-86b1-55a9cdeb1494"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:27.803750312Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1430,"":"downstairs","":"downstairs","session_id":"9a05bc68-28d3-44f3-86b1-55a9cdeb1494"} {"msg":"Max found gen is 62","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:27.803763191Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1430,"":"downstairs","":"downstairs","session_id":"9a05bc68-28d3-44f3-86b1-55a9cdeb1494"} {"msg":"Generation requested: 62 >= found:62","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:27.803776371Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1430,"":"downstairs","":"downstairs","session_id":"9a05bc68-28d3-44f3-86b1-55a9cdeb1494"} {"msg":"Next flush: 117","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:27.80378919Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1430,"":"downstairs","":"downstairs","session_id":"9a05bc68-28d3-44f3-86b1-55a9cdeb1494"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-12-12T20:35:27.803805039Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1430,"":"downstairs","":"downstairs","session_id":"9a05bc68-28d3-44f3-86b1-55a9cdeb1494"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:27.803821958Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1430,"session_id":"9a05bc68-28d3-44f3-86b1-55a9cdeb1494"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:27.803833967Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1430,"client":"0","":"downstairs","session_id":"9a05bc68-28d3-44f3-86b1-55a9cdeb1494"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:27.803847266Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1430,"client":"1","":"downstairs","session_id":"9a05bc68-28d3-44f3-86b1-55a9cdeb1494"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:27.803860245Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1430,"client":"2","":"downstairs","session_id":"9a05bc68-28d3-44f3-86b1-55a9cdeb1494"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:27.803873454Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1430,"session_id":"9a05bc68-28d3-44f3-86b1-55a9cdeb1494"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:27.803885793Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1430,"session_id":"9a05bc68-28d3-44f3-86b1-55a9cdeb1494"} {"msg":"6ce9a6cb-8671-4067-acc4-4614f98248bf is now active with session: 9a05bc68-28d3-44f3-86b1-55a9cdeb1494","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:27.803898852Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1430,"session_id":"9a05bc68-28d3-44f3-86b1-55a9cdeb1494"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:27.803910901Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1430,"session_id":"9a05bc68-28d3-44f3-86b1-55a9cdeb1494"} {"msg":"The guest has finished waiting for activation with:62","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:27.803923011Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1430} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 466 len 8192 data: 4 4 02/30 Write block 478 len 24576 data: 5 4 3 4 3 4 03/30 Read block 289 len 40960 04/30 Write block 464 len 32768 data: 5 5 5 5 4 4 3 3 05/30 Write block 588 len 20480 data: 5 5 7 6 6 06/30 Flush 07/30 Write block 108 len 36864 data: 8 8 8 9 9 9 8 6 6 08/30 Read block 184 len 36864 09/30 Read block 589 len 16384 10/30 Flush 11/30 Flush 12/30 Write block 51 len 4096 data: 6 13/30 Flush 14/30 Write block 24 len 12288 data: 6 4 4 15/30 Write block 359 len 32768 data: 4 4 4 6 7 6 5 6 16/30 Read block 298 len 24576 17/30 Read block 41 len 4096 18/30 Write block 426 len 32768 data: 5 3 3 4 3 5 5 5 19/30 Write block 348 len 36864 data: 6 5 7 4 5 5 7 7 6 20/30 Read block 474 len 28672 21/30 Read block 467 len 36864 22/30 Write block 56 len 4096 data: 10 23/30 Flush 24/30 Write block 55 len 28672 data: 8 11 11 11 12 11 5 25/30 Flush 26/30 Write block 427 len 28672 data: 4 4 5 4 6 6 6 27/30 Write block 515 len 12288 data: 5 6 7 28/30 Write block 279 len 20480 data: 3 3 4 4 3 29/30 Write block 580 len 32768 data: 5 5 6 6 8 7 7 5 30/30 Read block 14 len 4096 ---------------------------------------------------------------- Crucible gen:62 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 7 Done Done Done false 1026 Acked Write 3 Done Done Done false 1027 Acked Write 5 Done Done Done false 1028 Acked Write 8 Done Done Sent false 1029 Acked Read 1 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 2 2 Done 5 5 3 13 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:35:27.822862814Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1430} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 50 50 50 96 96 96 F F F 1 020-039 62 62 62 121 121 121 F F F 2 040-059 62 62 62 122 122 122 F F F 3 060-079 62 62 62 122 122 122 F F F 4 080-099 61 61 61 116 116 116 F F F 5 100-119 62 62 62 118 118 118 F F F 6 120-139 58 58 58 111 111 111 F F F 7 140-159 54 54 54 102 102 102 F F F 8 160-179 50 50 50 96 96 96 F F F 9 180-199 58 58 58 111 111 111 F F F 10 200-219 55 55 55 104 104 104 F F F 11 220-239 58 58 58 112 112 112 F F F 12 240-259 55 55 55 104 104 104 F F F 13 260-279 50 50 50 96 96 96 T T T 14 280-299 59 59 59 114 114 114 T T T 15 300-319 59 59 59 114 114 114 F F F 16 320-339 58 58 58 113 113 113 F F F 17 340-359 62 62 62 121 121 121 F F F 18 360-379 62 62 62 121 121 121 F F F 19 380-399 60 60 60 115 115 115 F F F 20 400-419 60 60 60 115 115 115 F F F 21 420-439 62 62 62 121 121 121 T T T 22 440-459 61 61 61 116 116 116 F F F 23 460-479 62 62 62 117 117 117 F F F 24 480-499 62 62 62 117 117 117 F F F 25 500-519 56 56 56 108 108 108 T T T 26 520-539 56 56 56 108 108 108 F F F 27 540-559 56 56 56 108 108 108 F F F 28 560-579 61 61 61 116 116 116 F F F 29 580-599 62 62 62 117 117 117 T T T Max gen: 62, Max flush: 122 A Difference in extent metadata is expected here On loop 31 Verifying data now No change after verify Loop: 31 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 50 50 50 96 96 96 F F F 1 020-039 62 62 62 121 121 121 F F F 2 040-059 62 62 62 122 122 122 F F F 3 060-079 62 62 62 122 122 122 F F F 4 080-099 61 61 61 116 116 116 F F F 5 100-119 62 62 62 118 118 118 F F F 6 120-139 58 58 58 111 111 111 F F F 7 140-159 54 54 54 102 102 102 F F F 8 160-179 50 50 50 96 96 96 F F F 9 180-199 58 58 58 111 111 111 F F F 10 200-219 55 55 55 104 104 104 F F F 11 220-239 58 58 58 112 112 112 F F F 12 240-259 55 55 55 104 104 104 F F F 13 260-279 63 63 63 123 123 123 F F F 14 280-299 63 63 63 123 123 123 F F F 15 300-319 59 59 59 114 114 114 F F F 16 320-339 58 58 58 113 113 113 F F F 17 340-359 62 62 62 121 121 121 F F F 18 360-379 62 62 62 121 121 121 F F F 19 380-399 60 60 60 115 115 115 F F F 20 400-419 60 60 60 115 115 115 F F F 21 420-439 63 63 63 123 123 123 F F F 22 440-459 61 61 61 116 116 116 F F F 23 460-479 62 62 62 117 117 117 F F F 24 480-499 62 62 62 117 117 117 F F F 25 500-519 63 63 63 123 123 123 F F F 26 520-539 56 56 56 108 108 108 F F F 27 540-559 56 56 56 108 108 108 F F F 28 560-579 61 61 61 116 116 116 F F F 29 580-599 63 63 63 123 123 123 F F F Max gen: 63, Max flush: 123 Begin loop 32 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:32.092337009Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1438,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:35:32.135459645Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1438,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:32.135855528Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1438,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:32.167412392Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1438,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:32.167983583Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1438} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"26e7d501b33f47379e53601720353c2983e7654b\",\n git_commit_timestamp: \"2024-12-12T20:16:40.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:32.16801846Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1438} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:32.168032269Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1438} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:32.168046518Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1438} {"msg":"Crucible 8e9d8f1d-7f93-4762-a781-6805e32abdfc has session id: a4e0496e-da6d-42b0-a661-d75ced70653c","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:32.168517336Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1438,"session_id":"a4e0496e-da6d-42b0-a661-d75ced70653c"} {"msg":"Upstairs opts: Upstairs UUID: 8e9d8f1d-7f93-4762-a781-6805e32abdfc, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:32.168563523Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1438,"session_id":"a4e0496e-da6d-42b0-a661-d75ced70653c"} {"msg":"Crucible stats registered with UUID: 8e9d8f1d-7f93-4762-a781-6805e32abdfc","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:32.168587991Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1438,"session_id":"a4e0496e-da6d-42b0-a661-d75ced70653c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:32.199942449Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1438,"task":"crutest"} {"msg":"The guest has requested activation with gen:64","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:32.199976106Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1438} {"msg":"8e9d8f1d-7f93-4762-a781-6805e32abdfc active request set","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:32.200057451Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1438,"session_id":"a4e0496e-da6d-42b0-a661-d75ced70653c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:32.200080719Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1438,"client":"0","":"downstairs","session_id":"a4e0496e-da6d-42b0-a661-d75ced70653c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:32.200107787Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1438,"client":"0","":"downstairs","session_id":"a4e0496e-da6d-42b0-a661-d75ced70653c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:32.200122016Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1438,"client":"1","":"downstairs","session_id":"a4e0496e-da6d-42b0-a661-d75ced70653c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:32.200136075Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1438,"client":"1","":"downstairs","session_id":"a4e0496e-da6d-42b0-a661-d75ced70653c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:32.200149165Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1438,"client":"2","":"downstairs","session_id":"a4e0496e-da6d-42b0-a661-d75ced70653c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:32.200162944Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1438,"client":"2","":"downstairs","session_id":"a4e0496e-da6d-42b0-a661-d75ced70653c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:32.200175912Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1438,"":"io task","client":"2","":"downstairs","session_id":"a4e0496e-da6d-42b0-a661-d75ced70653c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:32.200193341Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1438,"":"io task","client":"0","":"downstairs","session_id":"a4e0496e-da6d-42b0-a661-d75ced70653c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:32.2002171Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1438,"":"io task","client":"1","":"downstairs","session_id":"a4e0496e-da6d-42b0-a661-d75ced70653c"} {"msg":"ds_connection connected from Ok(127.0.0.1:44198)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:32.200260167Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1438,"":"io task","client":"0","":"downstairs","session_id":"a4e0496e-da6d-42b0-a661-d75ced70653c"} {"msg":"ds_connection connected from Ok(127.0.0.1:57367)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:32.200287385Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1438,"":"io task","client":"2","":"downstairs","session_id":"a4e0496e-da6d-42b0-a661-d75ced70653c"} {"msg":"ds_connection connected from Ok(127.0.0.1:39247)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:32.200313543Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1438,"":"io task","client":"1","":"downstairs","session_id":"a4e0496e-da6d-42b0-a661-d75ced70653c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:32.201151475Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1438,"client":"0","":"downstairs","session_id":"a4e0496e-da6d-42b0-a661-d75ced70653c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:32.201179733Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1438,"client":"1","":"downstairs","session_id":"a4e0496e-da6d-42b0-a661-d75ced70653c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:32.201198702Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1438,"client":"2","":"downstairs","session_id":"a4e0496e-da6d-42b0-a661-d75ced70653c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:32.201821699Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1438,"client":"1","":"downstairs","session_id":"a4e0496e-da6d-42b0-a661-d75ced70653c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:32.201865206Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1438,"client":"2","":"downstairs","session_id":"a4e0496e-da6d-42b0-a661-d75ced70653c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:32.201898764Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1438,"client":"0","":"downstairs","session_id":"a4e0496e-da6d-42b0-a661-d75ced70653c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:32.202369821Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1438,"session_id":"a4e0496e-da6d-42b0-a661-d75ced70653c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:32.202466265Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1438,"session_id":"a4e0496e-da6d-42b0-a661-d75ced70653c"} {"msg":"[0]R flush_numbers[0..12]: [96, 121, 122, 122, 116, 118, 111, 102, 96, 111, 104, 112]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:32.202558368Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1438,"":"downstairs","":"downstairs","session_id":"a4e0496e-da6d-42b0-a661-d75ced70653c"} {"msg":"[0]R generation[0..12]: [50, 62, 62, 62, 61, 62, 58, 54, 50, 58, 55, 58]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:32.202578277Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1438,"":"downstairs","":"downstairs","session_id":"a4e0496e-da6d-42b0-a661-d75ced70653c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:32.202593166Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1438,"":"downstairs","":"downstairs","session_id":"a4e0496e-da6d-42b0-a661-d75ced70653c"} {"msg":"[1]R flush_numbers[0..12]: [96, 121, 122, 122, 116, 118, 111, 102, 96, 111, 104, 112]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:32.202607365Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1438,"":"downstairs","":"downstairs","session_id":"a4e0496e-da6d-42b0-a661-d75ced70653c"} {"msg":"[1]R generation[0..12]: [50, 62, 62, 62, 61, 62, 58, 54, 50, 58, 55, 58]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:32.202621024Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1438,"":"downstairs","":"downstairs","session_id":"a4e0496e-da6d-42b0-a661-d75ced70653c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:32.202634763Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1438,"":"downstairs","":"downstairs","session_id":"a4e0496e-da6d-42b0-a661-d75ced70653c"} {"msg":"[2]R flush_numbers[0..12]: [96, 121, 122, 122, 116, 118, 111, 102, 96, 111, 104, 112]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:32.202648422Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1438,"":"downstairs","":"downstairs","session_id":"a4e0496e-da6d-42b0-a661-d75ced70653c"} {"msg":"[2]R generation[0..12]: [50, 62, 62, 62, 61, 62, 58, 54, 50, 58, 55, 58]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:32.202669681Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1438,"":"downstairs","":"downstairs","session_id":"a4e0496e-da6d-42b0-a661-d75ced70653c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:32.202709438Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1438,"":"downstairs","":"downstairs","session_id":"a4e0496e-da6d-42b0-a661-d75ced70653c"} {"msg":"Max found gen is 64","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:32.202736926Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1438,"":"downstairs","":"downstairs","session_id":"a4e0496e-da6d-42b0-a661-d75ced70653c"} {"msg":"Generation requested: 64 >= found:64","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:32.202759445Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1438,"":"Wait for a query_work_queue command to finish before sending IO downstairs","":"downstairs","session_id":"a4e0496e-da6d-42b0-a661-d75ced70653c"} {"msg":"Next flush: 124","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:32.202788552Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1438,"":"downstairs","":"downstairs","session_id":"a4e0496e-da6d-42b0-a661-d75ced70653c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:32.202809921Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1438,"":"downstairs","":"downstairs","session_id":"a4e0496e-da6d-42b0-a661-d75ced70653c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:32.202834579Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1438,"session_id":"a4e0496e-da6d-42b0-a661-d75ced70653c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:32.202858058Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1438,"client":"0","":"downstairs","session_id":"a4e0496e-da6d-42b0-a661-d75ced70653c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:32.202882946Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1438,"client":"1","":"downstairs","session_id":"a4e0496e-da6d-42b0-a661-d75ced70653c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:32.202907114Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1438,"client":"2","":"downstairs","session_id":"a4e0496e-da6d-42b0-a661-d75ced70653c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2024-12-12T20:35:32.202932502Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1438,"session_id":"a4e0496e-da6d-42b0-a661-d75ced70653c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:32.202956641Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1438,"session_id":"a4e0496e-da6d-42b0-a661-d75ced70653c"} {"msg":"8e9d8f1d-7f93-4762-a781-6805e32abdfc is now active with session: a4e0496e-da6d-42b0-a661-d75ced70653c","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:32.202978089Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1438,"session_id":"a4e0496e-da6d-42b0-a661-d75ced70653c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:32.203019347Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1438,"session_id":"a4e0496e-da6d-42b0-a661-d75ced70653c"} {"msg":"The guest has finished waiting for activation with:64","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:32.203043635Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1438} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 521 len 4096 data: 11 02/30 Write block 5 len 20480 data: 4 4 5 7 8 03/30 Flush 04/30 Write block 408 len 16384 data: 9 7 6 5 05/30 Read block 352 len 40960 06/30 Write block 523 len 28672 data: 12 11 10 6 5 5 6 07/30 Write block 562 len 32768 data: 5 5 5 5 5 4 4 5 08/30 Write block 183 len 12288 data: 6 9 9 09/30 Write block 519 len 36864 data: 12 13 12 10 13 12 11 7 6 10/30 Read block 393 len 40960 11/30 Write block 131 len 8192 data: 5 3 12/30 Write block 90 len 8192 data: 7 6 13/30 Write block 578 len 24576 data: 6 8 6 6 7 7 14/30 Read block 347 len 4096 15/30 Write block 457 len 12288 data: 5 5 6 16/30 Read block 489 len 20480 17/30 Read block 183 len 16384 18/30 Write block 278 len 12288 data: 4 4 4 19/30 Read block 311 len 4096 20/30 Flush 21/30 Write block 86 len 24576 data: 6 9 9 8 8 7 22/30 Write block 549 len 8192 data: 5 5 23/30 Read block 190 len 4096 24/30 Read block 555 len 24576 25/30 Read block 571 len 20480 26/30 Read block 403 len 16384 27/30 Write block 311 len 16384 data: 7 7 9 10 28/30 Write block 266 len 12288 data: 10 9 9 29/30 Read block 525 len 40960 30/30 Read block 354 len 24576 ---------------------------------------------------------------- Crucible gen:64 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1020 Acked Write 6 Done Done Done false 1021 Acked Write 2 Done Done Done false 1022 Acked Read 1 Done Done Done false 1023 Acked Read 6 Done Done Done false 1024 Acked Read 5 Done Done Done false 1025 Acked Read 4 Done Done Done false 1026 Acked Write 4 Done Done Done false 1027 Acked Write 3 Done Done Done false 1028 Acked Read 10 Done Done Done false 1029 Acked Read 6 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 10 10 9 29 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1019 1019 1019 Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:35:32.223083683Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1438} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 64 64 64 124 124 124 F F F 1 020-039 62 62 62 121 121 121 F F F 2 040-059 62 62 62 122 122 122 F F F 3 060-079 62 62 62 122 122 122 F F F 4 080-099 64 64 64 125 125 125 T T T 5 100-119 62 62 62 118 118 118 F F F 6 120-139 64 64 64 125 125 125 F F F 7 140-159 54 54 54 102 102 102 F F F 8 160-179 50 50 50 96 96 96 F F F 9 180-199 64 64 64 125 125 125 F F F 10 200-219 55 55 55 104 104 104 F F F 11 220-239 58 58 58 112 112 112 F F F 12 240-259 55 55 55 104 104 104 F F F 13 260-279 64 64 64 125 125 125 T T T 14 280-299 64 64 64 125 125 125 F F F 15 300-319 59 59 59 114 114 114 T T T 16 320-339 58 58 58 113 113 113 F F F 17 340-359 62 62 62 121 121 121 F F F 18 360-379 62 62 62 121 121 121 F F F 19 380-399 60 60 60 115 115 115 F F F 20 400-419 64 64 64 125 125 125 F F F 21 420-439 63 63 63 123 123 123 F F F 22 440-459 64 64 64 125 125 125 F F F 23 460-479 62 62 62 117 117 117 F F F 24 480-499 62 62 62 117 117 117 F F F 25 500-519 64 64 64 125 125 125 F F F 26 520-539 64 64 64 125 125 125 F F F 27 540-559 56 56 56 108 108 108 T T T 28 560-579 64 64 64 125 125 125 F F F 29 580-599 64 64 64 125 125 125 F F F Max gen: 64, Max flush: 125 A Difference in extent metadata is expected here On loop 32 Verifying data now No change after verify Loop: 32 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 64 64 64 124 124 124 F F F 1 020-039 62 62 62 121 121 121 F F F 2 040-059 62 62 62 122 122 122 F F F 3 060-079 62 62 62 122 122 122 F F F 4 080-099 65 65 65 126 126 126 F F F 5 100-119 62 62 62 118 118 118 F F F 6 120-139 64 64 64 125 125 125 F F F 7 140-159 54 54 54 102 102 102 F F F 8 160-179 50 50 50 96 96 96 F F F 9 180-199 64 64 64 125 125 125 F F F 10 200-219 55 55 55 104 104 104 F F F 11 220-239 58 58 58 112 112 112 F F F 12 240-259 55 55 55 104 104 104 F F F 13 260-279 65 65 65 126 126 126 F F F 14 280-299 64 64 64 125 125 125 F F F 15 300-319 65 65 65 126 126 126 F F F 16 320-339 58 58 58 113 113 113 F F F 17 340-359 62 62 62 121 121 121 F F F 18 360-379 62 62 62 121 121 121 F F F 19 380-399 60 60 60 115 115 115 F F F 20 400-419 64 64 64 125 125 125 F F F 21 420-439 63 63 63 123 123 123 F F F 22 440-459 64 64 64 125 125 125 F F F 23 460-479 62 62 62 117 117 117 F F F 24 480-499 62 62 62 117 117 117 F F F 25 500-519 64 64 64 125 125 125 F F F 26 520-539 64 64 64 125 125 125 F F F 27 540-559 65 65 65 126 126 126 F F F 28 560-579 64 64 64 125 125 125 F F F 29 580-599 64 64 64 125 125 125 F F F Max gen: 65, Max flush: 126 Begin loop 33 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:36.523985009Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1446,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:35:36.567061009Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1446,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:36.567466131Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1446,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:36.598986507Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1446,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:36.599553998Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1446} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"26e7d501b33f47379e53601720353c2983e7654b\",\n git_commit_timestamp: \"2024-12-12T20:16:40.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:36.599586896Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1446} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:36.599600455Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1446} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:36.599618224Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1446} {"msg":"Crucible 73595229-3a45-4c27-8319-a5f956332aca has session id: 1dbdd0fa-d184-4b54-a4e4-8401f5a7f6e9","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:36.600085552Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1446,"session_id":"1dbdd0fa-d184-4b54-a4e4-8401f5a7f6e9"} {"msg":"Upstairs opts: Upstairs UUID: 73595229-3a45-4c27-8319-a5f956332aca, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:36.600125959Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1446,"session_id":"1dbdd0fa-d184-4b54-a4e4-8401f5a7f6e9"} {"msg":"Crucible stats registered with UUID: 73595229-3a45-4c27-8319-a5f956332aca","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:36.600143168Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1446,"session_id":"1dbdd0fa-d184-4b54-a4e4-8401f5a7f6e9"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:36.631025798Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1446,"task":"crutest"} {"msg":"The guest has requested activation with gen:66","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:36.631057996Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1446} {"msg":"73595229-3a45-4c27-8319-a5f956332aca active request set","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:36.631089944Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1446,"session_id":"1dbdd0fa-d184-4b54-a4e4-8401f5a7f6e9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:36.631106312Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1446,"client":"0","":"downstairs","session_id":"1dbdd0fa-d184-4b54-a4e4-8401f5a7f6e9"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:36.631134121Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1446,"client":"0","":"downstairs","session_id":"1dbdd0fa-d184-4b54-a4e4-8401f5a7f6e9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:36.631151549Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1446,"client":"1","":"downstairs","session_id":"1dbdd0fa-d184-4b54-a4e4-8401f5a7f6e9"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:36.631169038Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1446,"client":"1","":"downstairs","session_id":"1dbdd0fa-d184-4b54-a4e4-8401f5a7f6e9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:36.631214595Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1446,"client":"2","":"downstairs","session_id":"1dbdd0fa-d184-4b54-a4e4-8401f5a7f6e9"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:36.631252953Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1446,"client":"2","":"downstairs","session_id":"1dbdd0fa-d184-4b54-a4e4-8401f5a7f6e9"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:36.63128333Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1446,"":"io task","client":"2","":"downstairs","session_id":"1dbdd0fa-d184-4b54-a4e4-8401f5a7f6e9"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:36.631321368Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1446,"":"io task","client":"0","":"downstairs","session_id":"1dbdd0fa-d184-4b54-a4e4-8401f5a7f6e9"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:36.631358465Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1446,"":"io task","client":"1","":"downstairs","session_id":"1dbdd0fa-d184-4b54-a4e4-8401f5a7f6e9"} {"msg":"ds_connection connected from Ok(127.0.0.1:47168)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:36.631416551Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1446,"":"io task","client":"0","":"downstairs","session_id":"1dbdd0fa-d184-4b54-a4e4-8401f5a7f6e9"} {"msg":"ds_connection connected from Ok(127.0.0.1:59646)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:36.631562961Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1446,"":"io task","client":"1","":"downstairs","session_id":"1dbdd0fa-d184-4b54-a4e4-8401f5a7f6e9"} {"msg":"ds_connection connected from Ok(127.0.0.1:32859)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:36.631602388Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1446,"":"io task","client":"2","":"downstairs","session_id":"1dbdd0fa-d184-4b54-a4e4-8401f5a7f6e9"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:36.632180969Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1446,"client":"1","":"downstairs","session_id":"1dbdd0fa-d184-4b54-a4e4-8401f5a7f6e9"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:36.632210736Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1446,"client":"2","":"downstairs","session_id":"1dbdd0fa-d184-4b54-a4e4-8401f5a7f6e9"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:36.632230815Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1446,"client":"0","":"downstairs","session_id":"1dbdd0fa-d184-4b54-a4e4-8401f5a7f6e9"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:36.632841483Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1446,"client":"0","":"downstairs","session_id":"1dbdd0fa-d184-4b54-a4e4-8401f5a7f6e9"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:36.632871381Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1446,"client":"1","":"downstairs","session_id":"1dbdd0fa-d184-4b54-a4e4-8401f5a7f6e9"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:36.63288739Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1446,"client":"2","":"downstairs","session_id":"1dbdd0fa-d184-4b54-a4e4-8401f5a7f6e9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:36.633424973Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1446,"session_id":"1dbdd0fa-d184-4b54-a4e4-8401f5a7f6e9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:36.633451361Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1446,"session_id":"1dbdd0fa-d184-4b54-a4e4-8401f5a7f6e9"} {"msg":"[0]R flush_numbers[0..12]: [124, 121, 122, 122, 126, 118, 125, 102, 96, 125, 104, 112]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:36.633538435Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1446,"":"downstairs","":"downstairs","session_id":"1dbdd0fa-d184-4b54-a4e4-8401f5a7f6e9"} {"msg":"[0]R generation[0..12]: [64, 62, 62, 62, 65, 62, 64, 54, 50, 64, 55, 58]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:36.633557754Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1446,"":"downstairs","":"downstairs","session_id":"1dbdd0fa-d184-4b54-a4e4-8401f5a7f6e9"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:36.633576722Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1446,"":"downstairs","":"downstairs","session_id":"1dbdd0fa-d184-4b54-a4e4-8401f5a7f6e9"} {"msg":"[1]R flush_numbers[0..12]: [124, 121, 122, 122, 126, 118, 125, 102, 96, 125, 104, 112]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:36.633591421Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1446,"":"downstairs","":"downstairs","session_id":"1dbdd0fa-d184-4b54-a4e4-8401f5a7f6e9"} {"msg":"[1]R generation[0..12]: [64, 62, 62, 62, 65, 62, 64, 54, 50, 64, 55, 58]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:36.63360499Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1446,"":"downstairs","":"downstairs","session_id":"1dbdd0fa-d184-4b54-a4e4-8401f5a7f6e9"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:36.633618939Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1446,"":"downstairs","":"downstairs","session_id":"1dbdd0fa-d184-4b54-a4e4-8401f5a7f6e9"} {"msg":"[2]R flush_numbers[0..12]: [124, 121, 122, 122, 126, 118, 125, 102, 96, 125, 104, 112]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:36.633632698Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1446,"":"downstairs","":"downstairs","session_id":"1dbdd0fa-d184-4b54-a4e4-8401f5a7f6e9"} {"msg":"[2]R generation[0..12]: [64, 62, 62, 62, 65, 62, 64, 54, 50, 64, 55, 58]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:36.633646208Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1446,"":"downstairs","":"downstairs","session_id":"1dbdd0fa-d184-4b54-a4e4-8401f5a7f6e9"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:36.633659907Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1446,"":"downstairs","":"downstairs","session_id":"1dbdd0fa-d184-4b54-a4e4-8401f5a7f6e9"} {"msg":"Max found gen is 66","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:36.633672846Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1446,"":"downstairs","":"downstairs","session_id":"1dbdd0fa-d184-4b54-a4e4-8401f5a7f6e9"} {"msg":"Generation requested: 66 >= found:66","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:36.633686015Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1446,"":"downstairs","":"downstairs","session_id":"1dbdd0fa-d184-4b54-a4e4-8401f5a7f6e9"} {"msg":"Next flush: 127","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:36.633698824Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1446,"":"downstairs","":"downstairs","session_id":"1dbdd0fa-d184-4b54-a4e4-8401f5a7f6e9"} {Wait for a query_work_queue command to finish before sending IO "msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:36.633717412Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1446,"":"downstairs","":"downstairs","session_id":"1dbdd0fa-d184-4b54-a4e4-8401f5a7f6e9"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:36.633730582Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1446,"session_id":"1dbdd0fa-d184-4b54-a4e4-8401f5a7f6e9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:36.633742531Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1446,"client":"0","":"downstairs","session_id":"1dbdd0fa-d184-4b54-a4e4-8401f5a7f6e9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:36.6337561Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1446,"client":"1","":"downstairs","session_id":"1dbdd0fa-d184-4b54-a4e4-8401f5a7f6e9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:36.633771779Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1446,"client":"2","":"downstairs","session_id":"1dbdd0fa-d184-4b54-a4e4-8401f5a7f6e9"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:36.633785258Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1446,"session_id":"1dbdd0fa-d184-4b54-a4e4-8401f5a7f6e9"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:36.633797357Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1446,"session_id":"1dbdd0fa-d184-4b54-a4e4-8401f5a7f6e9"} {"msg":"73595229-3a45-4c27-8319-a5f956332aca is now active with session: 1dbdd0fa-d184-4b54-a4e4-8401f5a7f6e9","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:36.633810206Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1446,"session_id":"1dbdd0fa-d184-4b54-a4e4-8401f5a7f6e9"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:36.633823055Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1446,"session_id":"1dbdd0fa-d184-4b54-a4e4-8401f5a7f6e9"} {"msg":"The guest has finished waiting for activation with:66","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:36.633835334Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1446} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 142 len 8192 data: 5 4 02/30 Read block 430 len 4096 03/30 Read block 43 len 36864 04/30 Read block 318 len 20480 05/30 Read block 481 len 4096 06/30 Read block 318 len 28672 07/30 Read block 499 len 40960 08/30 Flush 09/30 Write block 364 len 16384 data: 7 6 7 5 10/30 Read block 136 len 12288 11/30 Write block 194 len 20480 data: 5 5 6 5 4 12/30 Write block 199 len 40960 data: 4 4 4 5 5 6 5 6 5 6 13/30 Read block 264 len 24576 14/30 Read block 77 len 40960 15/30 Write block 77 len 12288 data: 5 4 4 16/30 Read block 165 len 12288 17/30 Flush 18/30 Write block 11 len 28672 data: 6 5 4 5 7 4 4 19/30 Read block 179 len 12288 20/30 Write block 166 len 16384 data: 3 2 2 2 21/30 Write block 523 len 24576 data: 14 13 12 8 7 6 22/30 Write block 502 len 12288 data: 7 5 5 23/30 Write block 361 len 28672 data: 5 7 8 8 7 8 6 24/30 Read block 537 len 28672 25/30 Read block 93 len 28672 26/30 Read block 309 len 4096 27/30 Read block 426 len 4096 28/30 Read block 85 len 16384 29/30 Write block 564 len 12288 data: 6 6 6 30/30 Read block 255 len 36864 ---------------------------------------------------------------- Crucible gen:66 GIO:true work queues: Upstairs:0 downstairs:13 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1017 Acked Write 7 Done Done Done false 1018 Acked Read 3 Done Done Done false 1019 Acked Write 4 Done Done Done false 1020 Acked Write 6 Done Done Done false 1021 Acked Write 3 Done Done Done false 1022 Acked Write 7 Done Done Done false 1023 Acked Read 7 Done Done Done false 1024 Acked Read 7 Done Done Done false 1025 Acked Read 1 Done Done Done false 1026 Acked Read 1 Done Done Done false 1027 Acked Read 4 Done Done Done false 1028 Acked Write 3 Done Done Sent false 1029 Acked Read 9 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 2 3 Done 13 12 11 36 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1016 1016 1016 Downstairs last five completed: 1016 1015 1014 1013 1012 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:35:36.653501978Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1446} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 64 64 64 124 124 124 T T T 1 020-039 62 62 62 121 121 121 F F F 2 040-059 62 62 62 122 122 122 F F F 3 060-079 66 66 66 128 128 128 F F F 4 080-099 65 65 65 126 126 126 F F F 5 100-119 62 62 62 118 118 118 F F F 6 120-139 64 64 64 125 125 125 F F F 7 140-159 66 66 66 127 127 127 F F F 8 160-179 50 50 50 96 96 96 T T T 9 180-199 66 66 66 128 128 128 F F F 10 200-219 66 66 66 128 128 128 F F F 11 220-239 58 58 58 112 112 112 F F F 12 240-259 55 55 55 104 104 104 F F F 13 260-279 65 65 65 126 126 126 F F F 14 280-299 64 64 64 125 125 125 F F F 15 300-319 65 65 65 126 126 126 F F F 16 320-339 58 58 58 113 113 113 F F F 17 340-359 62 62 62 121 121 121 F F F 18 360-379 66 66 66 128 128 128 T T T 19 380-399 60 60 60 115 115 115 F F F 20 400-419 64 64 64 125 125 125 F F F 21 420-439 63 63 63 123 123 123 F F F 22 440-459 64 64 64 125 125 125 F F F 23 460-479 62 62 62 117 117 117 F F F 24 480-499 62 62 62 117 117 117 F F F 25 500-519 64 64 64 125 125 125 T T T 26 520-539 64 64 64 125 125 125 T T T 27 540-559 65 65 65 126 126 126 F F F 28 560-579 64 64 64 125 125 125 T T T 29 580-599 64 64 64 125 125 125 F F F Max gen: 66, Max flush: 128 A Difference in extent metadata is expected here On loop 33 Verifying data now No change after verify Loop: 33 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 67 67 67 129 129 129 F F F 1 020-039 62 62 62 121 121 121 F F F 2 040-059 62 62 62 122 122 122 F F F 3 060-079 66 66 66 128 128 128 F F F 4 080-099 65 65 65 126 126 126 F F F 5 100-119 62 62 62 118 118 118 F F F 6 120-139 64 64 64 125 125 125 F F F 7 140-159 66 66 66 127 127 127 F F F 8 160-179 67 67 67 129 129 129 F F F 9 180-199 66 66 66 128 128 128 F F F 10 200-219 66 66 66 128 128 128 F F F 11 220-239 58 58 58 112 112 112 F F F 12 240-259 55 55 55 104 104 104 F F F 13 260-279 65 65 65 126 126 126 F F F 14 280-299 64 64 64 125 125 125 F F F 15 300-319 65 65 65 126 126 126 F F F 16 320-339 58 58 58 113 113 113 F F F 17 340-359 62 62 62 121 121 121 F F F 18 360-379 67 67 67 129 129 129 F F F 19 380-399 60 60 60 115 115 115 F F F 20 400-419 64 64 64 125 125 125 F F F 21 420-439 63 63 63 123 123 123 F F F 22 440-459 64 64 64 125 125 125 F F F 23 460-479 62 62 62 117 117 117 F F F 24 480-499 62 62 62 117 117 117 F F F 25 500-519 67 67 67 129 129 129 F F F 26 520-539 67 67 67 129 129 129 F F F 27 540-559 65 65 65 126 126 126 F F F 28 560-579 67 67 67 129 129 129 F F F 29 580-599 64 64 64 125 125 125 F F F Max gen: 67, Max flush: 129 Begin loop 34 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:40.959838039Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1454,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:41.004559876Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1454,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:41.00521417Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1454} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"26e7d501b33f47379e53601720353c2983e7654b\",\n git_commit_timestamp: \"2024-12-12T20:16:40.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:41.005247758Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1454} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:41.005261787Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1454} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:41.005275996Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1454} {"msg":"Crucible d52b5a98-8466-4ee5-abb9-0e43bf47f02b has session id: 77fe4ef3-1429-4f7f-87d2-368152d7f33e","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:41.005765532Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1454,"session_id":"77fe4ef3-1429-4f7f-87d2-368152d7f33e"} {"msg":"Upstairs opts: Upstairs UUID: d52b5a98-8466-4ee5-abb9-0e43bf47f02b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:41.005811899Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1454,"session_id":"77fe4ef3-1429-4f7f-87d2-368152d7f33e"} {"msg":"Crucible stats registered with UUID: d52b5a98-8466-4ee5-abb9-0e43bf47f02b","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:41.005832238Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1454,"session_id":"77fe4ef3-1429-4f7f-87d2-368152d7f33e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:41.036914525Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1454,"task":"crutest"} {"msg":"The guest has requested activation with gen:68","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:41.036943902Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1454} {"msg":"d52b5a98-8466-4ee5-abb9-0e43bf47f02b active request set","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:41.037023377Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1454,"session_id":"77fe4ef3-1429-4f7f-87d2-368152d7f33e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:41.037044436Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1454,"client":"0","":"downstairs","session_id":"77fe4ef3-1429-4f7f-87d2-368152d7f33e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:41.037068644Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1454,"client":"0","":"downstairs","session_id":"77fe4ef3-1429-4f7f-87d2-368152d7f33e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:41.037086723Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1454,"client":"1","":"downstairs","session_id":"77fe4ef3-1429-4f7f-87d2-368152d7f33e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:41.037101442Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1454,"client":"1","":"downstairs","session_id":"77fe4ef3-1429-4f7f-87d2-368152d7f33e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:41.03712311Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1454,"client":"2","":"downstairs","session_id":"77fe4ef3-1429-4f7f-87d2-368152d7f33e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:41.037151248Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1454,"client":"2","":"downstairs","session_id":"77fe4ef3-1429-4f7f-87d2-368152d7f33e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:41.037178006Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1454,"":"io task","client":"2","":"downstairs","session_id":"77fe4ef3-1429-4f7f-87d2-368152d7f33e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:41.037205615Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1454,"":"io task","client":"0","":"downstairs","session_id":"77fe4ef3-1429-4f7f-87d2-368152d7f33e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:41.037231653Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1454,"":"io task","client":"1","":"downstairs","session_id":"77fe4ef3-1429-4f7f-87d2-368152d7f33e"} {"msg":"ds_connection connected from Ok(127.0.0.1:50399)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:41.037256691Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1454,"":"io task","client":"2","":"downstairs","session_id":"77fe4ef3-1429-4f7f-87d2-368152d7f33e"} {"msg":"ds_connection connected from Ok(127.0.0.1:33257)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:41.037283309Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1454,"":"io task","client":"1","":"downstairs","session_id":"77fe4ef3-1429-4f7f-87d2-368152d7f33e"} {"msg":"ds_connection connected from Ok(127.0.0.1:37441)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:41.037543981Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1454,"":"io task","client":"0","":"downstairs","session_id":"77fe4ef3-1429-4f7f-87d2-368152d7f33e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:41.037947733Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1454,"client":"0","":"downstairs","session_id":"77fe4ef3-1429-4f7f-87d2-368152d7f33e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:41.03799763Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1454,"client":"1","":"downstairs","session_id":"77fe4ef3-1429-4f7f-87d2-368152d7f33e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:41.038215155Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1454,"client":"2","":"downstairs","session_id":"77fe4ef3-1429-4f7f-87d2-368152d7f33e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:41.038754598Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1454,"client":"1","":"downstairs","session_id":"77fe4ef3-1429-4f7f-87d2-368152d7f33e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:41.038793095Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1454,"client":"0","":"downstairs","session_id":"77fe4ef3-1429-4f7f-87d2-368152d7f33e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:41.038883269Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1454,"client":"2","":"downstairs","session_id":"77fe4ef3-1429-4f7f-87d2-368152d7f33e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:41.039329368Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1454,"session_id":"77fe4ef3-1429-4f7f-87d2-368152d7f33e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:41.039365406Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1454,"session_id":"77fe4ef3-1429-4f7f-87d2-368152d7f33e"} {"msg":"[0]R flush_numbers[0..12]: [129, 121, 122, 128, 126, 118, 125, 127, 129, 128, 128, 112]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:41.039423931Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1454,"":"downstairs","":"downstairs","session_id":"77fe4ef3-1429-4f7f-87d2-368152d7f33e"} {"msg":"[0]R generation[0..12]: [67, 62, 62, 66, 65, 62, 64, 66, 67, 66, 66, 58]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:41.03945352Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1454,"":"downstairs","":"downstairs","session_id":"77fe4ef3-1429-4f7f-87d2-368152d7f33e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:41.039469808Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1454,"":"downstairs","":"downstairs","session_id":"77fe4ef3-1429-4f7f-87d2-368152d7f33e"} {"msg":"[1]R flush_numbers[0..12]: [129, 121, 122, 128, 126, 118, 125, 127, 129, 128, 128, 112]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:41.039484157Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1454,"":"downstairs","":"downstairs","session_id":"77fe4ef3-1429-4f7f-87d2-368152d7f33e"} {"msg":"[1]R generation[0..12]: [67, 62, 62, 66, 65, 62, 64, 66, 67, 66, 66, 58]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:41.039498006Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1454,"":"downstairs","":"downstairs","session_id":"77fe4ef3-1429-4f7f-87d2-368152d7f33e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:41.039512136Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1454,"":"downstairs","":"downstairs","session_id":"77fe4ef3-1429-4f7f-87d2-368152d7f33e"} {"msg":"[2]R flush_numbers[0..12]: [129, 121, 122, 128, 126, 118, 125, 127, 129, 128, 128, 112]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:41.039525984Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1454,"":"downstairs","":"downstairs","session_id":"77fe4ef3-1429-4f7f-87d2-368152d7f33e"} {"msg":"[2]R generation[0..12]: [67, 62, 62, 66, 65, 62, 64, 66, 67, 66, 66, 58]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:41.039539594Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1454,"":"downstairs","":"downstairs","session_id":"77fe4ef3-1429-4f7f-87d2-368152d7f33e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:41.039553613Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1454,"":"downstairs","":"downstairs","session_id":"77fe4ef3-1429-4f7f-87d2-368152d7f33e"} {"msg":"Max found gen is 68","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:41.039566892Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1454,"":"downstairs","":"downstairs","session_id":"77fe4ef3-1429-4f7f-87d2-368152d7f33e"} {"msg":"Generation requested: 68 >= found:68","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-12-12T20:35:41.039621038Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1454,"":"downstairs","":"downstairs","session_id":"77fe4ef3-1429-4f7f-87d2-368152d7f33e"} {"msg":"Next flush: 130","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:41.039646386Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1454,"":"downstairs","":"downstairs","session_id":"77fe4ef3-1429-4f7f-87d2-368152d7f33e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:41.039660665Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1454,"":"downstairs","":"downstairs","session_id":"77fe4ef3-1429-4f7f-87d2-368152d7f33e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:41.039674304Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1454,"session_id":"77fe4ef3-1429-4f7f-87d2-368152d7f33e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:41.039686553Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1454,"client":"0","":"downstairs","session_id":"77fe4ef3-1429-4f7f-87d2-368152d7f33e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:41.039699952Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1454,"client":"1","":"downstairs","session_id":"77fe4ef3-1429-4f7f-87d2-368152d7f33e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:41.039713242Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1454,"client":"2","":"downstairs","session_id":"77fe4ef3-1429-4f7f-87d2-368152d7f33e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:41.039726581Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1454,"session_id":"77fe4ef3-1429-4f7f-87d2-368152d7f33e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:41.03973882Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1454,"session_id":"77fe4ef3-1429-4f7f-87d2-368152d7f33e"} {"msg":"d52b5a98-8466-4ee5-abb9-0e43bf47f02b is now active with session: 77fe4ef3-1429-4f7f-87d2-368152d7f33e","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:41.039751849Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1454,"session_id":"77fe4ef3-1429-4f7f-87d2-368152d7f33e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:41.039763878Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1454,"session_id":"77fe4ef3-1429-4f7f-87d2-368152d7f33e"} {"msg":"The guest has finished waiting for activation with:68","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:41.039776127Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1454} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 413 len 8192 data: 4 5 02/30 Write block 213 len 36864 data: 11 8 8 5 6 4 5 5 4 03/30 Write block 598 len 8192 data: 3 3 04/30 Write block 586 len 8192 data: 8 6 05/30 Write block 542 len 36864 data: 5 5 6 5 5 5 5 6 6 06/30 Read block 346 len 8192 07/30 Read block 301 len 40960 08/30 Write block 518 len 32768 data: 10 13 14 13 11 15 14 13 09/30 Read block 115 len 4096 10/30 Read block 314 len 28672 11/30 Flush 12/30 Write block 445 len 32768 data: 6 7 8 5 7 7 8 8 13/30 Write block 399 len 12288 data: 8 7 8 14/30 Write block 0 len 8192 data: 3 4 15/30 Read block 548 len 32768 16/30 Write block 87 len 16384 data: 10 10 9 9 17/30 Read block 62 len 16384 18/30 Write block 112 len 4096 data: 10 19/30 Read block 101 len 12288 20/30 Write block 540 len 4096 data: 6 21/30 Read block 167 len 24576 22/30 Flush 23/30 Write block 561 len 8192 data: 4 6 24/30 Read block 157 len 8192 25/30 Read block 321 len 32768 26/30 Write block 495 len 12288 data: 5 5 5 27/30 Write block 566 len 40960 data: 7 5 5 6 4 4 4 4 4 4 28/30 Read block 448 len 4096 29/30 Write block 448 len 36864 data: 6 8 8 9 9 8 8 6 5 30/30 Read block 334 len 12288 ---------------------------------------------------------------- Crucible gen:68 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 2 Done Done Done false 1023 Acked Read 2 Done Done Done false 1024 Acked Read 8 Done Done Done false 1025 Acked Write 3 Done Done Done false 1026 Acked Write 10 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Write 9 Done Done Done false 1029 Acked Read 3 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 1 2 Done 7 8 7 22 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1021 1021 1021 Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:35:41.057616897Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1454} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 68 68 68 131 131 131 F F F 1 020-039 62 62 62 121 121 121 F F F 2 040-059 62 62 62 122 122 122 F F F 3 060-079 66 66 66 128 128 128 F F F 4 080-099 68 68 68 131 131 131 F F F 5 100-119 68 68 68 131 131 131 F F F 6 120-139 64 64 64 125 125 125 F F F 7 140-159 66 66 66 127 127 127 F F F 8 160-179 67 67 67 129 129 129 F F F 9 180-199 66 66 66 128 128 128 F F F 10 200-219 68 68 68 130 130 130 F F F 11 220-239 68 68 68 130 130 130 F F F 12 240-259 55 55 55 104 104 104 F F F 13 260-279 65 65 65 126 126 126 F F F 14 280-299 64 64 64 125 125 125 F F F 15 300-319 65 65 65 126 126 126 F F F 16 320-339 58 58 58 113 113 113 F F F 17 340-359 62 62 62 121 121 121 F F F 18 360-379 67 67 67 129 129 129 F F F 19 380-399 68 68 68 131 131 131 F F F 20 400-419 68 68 68 131 131 131 F F F 21 420-439 63 63 63 123 123 123 F F F 22 440-459 68 68 68 131 131 131 T T T 23 460-479 62 62 62 117 117 117 F F F 24 480-499 62 62 62 117 117 117 T T T 25 500-519 68 68 68 130 130 130 F F F 26 520-539 68 68 68 130 130 130 F F F 27 540-559 68 68 68 131 131 131 F F F 28 560-579 67 67 67 129 129 129 T T T 29 580-599 68 68 68 130 130 130 F F F Max gen: 68, Max flush: 131 A Difference in extent metadata is expected here On loop 34 Verifying data now No change after verify Loop: 34 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 68 68 68 131 131 131 F F F 1 020-039 62 62 62 121 121 121 F F F 2 040-059 62 62 62 122 122 122 F F F 3 060-079 66 66 66 128 128 128 F F F 4 080-099 68 68 68 131 131 131 F F F 5 100-119 68 68 68 131 131 131 F F F 6 120-139 64 64 64 125 125 125 F F F 7 140-159 66 66 66 127 127 127 F F F 8 160-179 67 67 67 129 129 129 F F F 9 180-199 66 66 66 128 128 128 F F F 10 200-219 68 68 68 130 130 130 F F F 11 220-239 68 68 68 130 130 130 F F F 12 240-259 55 55 55 104 104 104 F F F 13 260-279 65 65 65 126 126 126 F F F 14 280-299 64 64 64 125 125 125 F F F 15 300-319 65 65 65 126 126 126 F F F 16 320-339 58 58 58 113 113 113 F F F 17 340-359 62 62 62 121 121 121 F F F 18 360-379 67 67 67 129 129 129 F F F 19 380-399 68 68 68 131 131 131 F F F 20 400-419 68 68 68 131 131 131 F F F 21 420-439 63 63 63 123 123 123 F F F 22 440-459 69 69 69 132 132 132 F F F 23 460-479 62 62 62 117 117 117 F F F 24 480-499 69 69 69 132 132 132 F F F 25 500-519 68 68 68 130 130 130 F F F 26 520-539 68 68 68 130 130 130 F F F 27 540-559 68 68 68 131 131 131 F F F 28 560-579 69 69 69 132 132 132 F F F 29 580-599 68 68 68 130 130 130 F F F Max gen: 69, Max flush: 132 Begin loop 35 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:45.316872206Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1462,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:45.361490179Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1462,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:45.362091357Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1462} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"26e7d501b33f47379e53601720353c2983e7654b\",\n git_commit_timestamp: \"2024-12-12T20:16:40.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:45.362126595Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1462} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:45.362140484Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1462} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:45.362154823Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1462} {"msg":"Crucible cbef032a-6fab-4e11-93e8-a9e5f6dce265 has session id: 0430fb5c-446e-42ff-8631-26b2d1f4e4e7","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:45.362757072Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1462,"session_id":"0430fb5c-446e-42ff-8631-26b2d1f4e4e7"} {"msg":"Upstairs opts: Upstairs UUID: cbef032a-6fab-4e11-93e8-a9e5f6dce265, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:45.362798919Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1462,"session_id":"0430fb5c-446e-42ff-8631-26b2d1f4e4e7"} {"msg":"Crucible stats registered with UUID: cbef032a-6fab-4e11-93e8-a9e5f6dce265","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:45.362815118Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1462,"session_id":"0430fb5c-446e-42ff-8631-26b2d1f4e4e7"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:45.393393439Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1462,"task":"crutest"} {"msg":"The guest has requested activation with gen:70","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:45.393423887Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1462} {"msg":"cbef032a-6fab-4e11-93e8-a9e5f6dce265 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:45.393479133Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1462,"session_id":"0430fb5c-446e-42ff-8631-26b2d1f4e4e7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:45.393507321Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1462,"client":"0","":"downstairs","session_id":"0430fb5c-446e-42ff-8631-26b2d1f4e4e7"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:45.393541239Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1462,"client":"0","":"downstairs","session_id":"0430fb5c-446e-42ff-8631-26b2d1f4e4e7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:45.393556798Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1462,"client":"1","":"downstairs","session_id":"0430fb5c-446e-42ff-8631-26b2d1f4e4e7"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:45.393571587Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1462,"client":"1","":"downstairs","session_id":"0430fb5c-446e-42ff-8631-26b2d1f4e4e7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:45.393585126Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1462,"client":"2","":"downstairs","session_id":"0430fb5c-446e-42ff-8631-26b2d1f4e4e7"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:45.393599165Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1462,"client":"2","":"downstairs","session_id":"0430fb5c-446e-42ff-8631-26b2d1f4e4e7"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:45.393612204Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1462,"":"io task","client":"2","":"downstairs","session_id":"0430fb5c-446e-42ff-8631-26b2d1f4e4e7"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:45.393681939Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1462,"":"io task","client":"0","":"downstairs","session_id":"0430fb5c-446e-42ff-8631-26b2d1f4e4e7"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:45.393730646Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1462,"":"io task","client":"1","":"downstairs","session_id":"0430fb5c-446e-42ff-8631-26b2d1f4e4e7"} {"msg":"ds_connection connected from Ok(127.0.0.1:49983)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:45.393768103Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1462,"":"io task","client":"0","":"downstairs","session_id":"0430fb5c-446e-42ff-8631-26b2d1f4e4e7"} {"msg":"ds_connection connected from Ok(127.0.0.1:38749)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:45.393803971Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1462,"":"io task","client":"2","":"downstairs","session_id":"0430fb5c-446e-42ff-8631-26b2d1f4e4e7"} {"msg":"ds_connection connected from Ok(127.0.0.1:46518)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:45.393974209Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1462,"":"io task","client":"1","":"downstairs","session_id":"0430fb5c-446e-42ff-8631-26b2d1f4e4e7"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:45.394418279Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1462,"client":"2","":"downstairs","session_id":"0430fb5c-446e-42ff-8631-26b2d1f4e4e7"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:45.394452726Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1462,"client":"0","":"downstairs","session_id":"0430fb5c-446e-42ff-8631-26b2d1f4e4e7"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:45.394758595Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1462,"client":"1","":"downstairs","session_id":"0430fb5c-446e-42ff-8631-26b2d1f4e4e7"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:45.395160927Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1462,"client":"0","":"downstairs","session_id":"0430fb5c-446e-42ff-8631-26b2d1f4e4e7"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:45.395194745Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1462,"client":"2","":"downstairs","session_id":"0430fb5c-446e-42ff-8631-26b2d1f4e4e7"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:45.395307907Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1462,"client":"1","":"downstairs","session_id":"0430fb5c-446e-42ff-8631-26b2d1f4e4e7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:45.395756736Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1462,"session_id":"0430fb5c-446e-42ff-8631-26b2d1f4e4e7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:45.395788894Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1462,"session_id":"0430fb5c-446e-42ff-8631-26b2d1f4e4e7"} {"msg":"[0]R flush_numbers[0..12]: [131, 121, 122, 128, 131, 131, 125, 127, 129, 128, 130, 130]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:45.395871698Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1462,"":"downstairs","":"downstairs","session_id":"0430fb5c-446e-42ff-8631-26b2d1f4e4e7"} {"msg":"[0]R generation[0..12]: [68, 62, 62, 66, 68, 68, 64, 66, 67, 66, 68, 68]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:45.395891337Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1462,"":"downstairs","":"downstairs","session_id":"0430fb5c-446e-42ff-8631-26b2d1f4e4e7"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:45.395906176Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1462,"":"downstairs","":"downstairs","session_id":"0430fb5c-446e-42ff-8631-26b2d1f4e4e7"} {"msg":"[1]R flush_numbers[0..12]: [131, 121, 122, 128, 131, 131, 125, 127, 129, 128, 130, 130]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:45.395920225Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1462,"":"downstairs","":"downstairs","session_id":"0430fb5c-446e-42ff-8631-26b2d1f4e4e7"} {"msg":"[1]R generation[0..12]: [68, 62, 62, 66, 68, 68, 64, 66, 67, 66, 68, 68]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:45.395934044Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1462,"":"downstairs","":"downstairs","session_id":"0430fb5c-446e-42ff-8631-26b2d1f4e4e7"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:45.395948213Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1462,"":"downstairs","":"downstairs","session_id":"0430fb5c-446e-42ff-8631-26b2d1f4e4e7"} {"msg":"[2]R flush_numbers[0..12]: [131, 121, 122, 128, 131, 131, 125, 127, 129, 128, 130, 130]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:45.395961922Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1462,"":"downstairs","":"downstairs","session_id":"0430fb5c-446e-42ff-8631-26b2d1f4e4e7"} {"msg":"[2]R generation[0..12]: [68, 62, 62, 66, 68, 68, 64, 66, 67, 66, 68, 68]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:45.395978971Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1462,"":"downstairs","":"downstairs","session_id":"0430fb5c-446e-42ff-8631-26b2d1f4e4e7"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:45.39599313Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1462,"":"downstairs","":"downstairs","session_id":"0430fb5c-446e-42ff-8631-26b2d1f4e4e7"} {"msg":"Max found gen is 70","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:45.396006199Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1462,"":"downstairs","":"downstairs","session_id":"0430fb5c-446e-42ff-8631-26b2d1f4e4e7"} {"msg":"Generation requested: 70 >= found:70","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:45.396019498Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1462,"":"downstairs","":"downstairs","session_id":"0430fb5c-446e-42ff-8631-26b2d1f4e4e7"} {"msg":"Next flush: 133","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:45.396032407Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1462,"":"downstairs","":"downstairs","session_id":"0430fb5c-446e-42ff-8631-26b2d1f4e4e7"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:45.396045396Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1462,"":"downstairs","":"downstairs","session_id":"0430fb5c-446e-42ff-8631-26b2d1f4e4e7"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:45.396058555Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1462,"session_id":"0430fb5c-446e-42ff-8631-26b2d1f4e4e7"} Wait for a query_work_queue command to finish before sending IO {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:45.396075064Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1462,"client":"0","":"downstairs","session_id":"0430fb5c-446e-42ff-8631-26b2d1f4e4e7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:45.396088473Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1462,"client":"1","":"downstairs","session_id":"0430fb5c-446e-42ff-8631-26b2d1f4e4e7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:45.396101522Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1462,"client":"2","":"downstairs","session_id":"0430fb5c-446e-42ff-8631-26b2d1f4e4e7"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:45.396114582Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1462,"session_id":"0430fb5c-446e-42ff-8631-26b2d1f4e4e7"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:45.396126741Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1462,"session_id":"0430fb5c-446e-42ff-8631-26b2d1f4e4e7"} {"msg":"cbef032a-6fab-4e11-93e8-a9e5f6dce265 is now active with session: 0430fb5c-446e-42ff-8631-26b2d1f4e4e7","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:45.39613939Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1462,"session_id":"0430fb5c-446e-42ff-8631-26b2d1f4e4e7"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:45.396151499Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1462,"session_id":"0430fb5c-446e-42ff-8631-26b2d1f4e4e7"} {"msg":"The guest has finished waiting for activation with:70","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:45.396163528Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1462} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 350 len 32768 data: 8 5 6 6 8 8 7 8 03/30 Read block 204 len 16384 04/30 Read block 42 len 32768 05/30 Read block 39 len 36864 06/30 Read block 391 len 16384 07/30 Write block 45 len 4096 data: 7 08/30 Read block 413 len 28672 09/30 Write block 217 len 28672 data: 7 5 6 6 5 4 5 10/30 Read block 396 len 20480 11/30 Read block 521 len 12288 12/30 Read block 379 len 40960 13/30 Write block 203 len 20480 data: 6 7 6 7 6 14/30 Write block 231 len 40960 data: 10 9 9 8 8 7 6 5 6 5 15/30 Read block 391 len 20480 16/30 Write block 218 len 16384 data: 6 7 7 6 17/30 Flush 18/30 Write block 275 len 4096 data: 5 19/30 Write block 424 len 24576 data: 7 7 6 5 5 6 20/30 Read block 55 len 12288 21/30 Read block 163 len 4096 22/30 Read block 377 len 40960 23/30 Write block 201 len 24576 data: 5 6 7 8 7 8 24/30 Read block 491 len 8192 25/30 Read block 345 len 36864 26/30 Read block 214 len 16384 27/30 Write block 236 len 36864 data: 8 7 6 7 6 5 4 4 3 28/30 Read block 94 len 36864 29/30 Write block 153 len 20480 data: 4 5 5 6 6 30/30 Write block 225 len 8192 data: 6 8 ---------------------------------------------------------------- Crucible gen:70 GIO:true work queues: Upstairs:0 downstairs:13 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1017 Acked Write 1 Done Done Done false 1018 Acked Write 6 Done Done Done false 1019 Acked Read 3 Done Done Done false 1020 Acked Read 1 Done Done Done false 1021 Acked Read 10 Done Done Done false 1022 Acked Write 6 Done Done Done false 1023 Acked Read 2 Done Done Done false 1024 Acked Read 9 Done Done Done false 1025 Acked Read 4 Done Done Done false 1026 Acked Write 9 Done Done Done false 1027 Acked Read 9 Done Done Done false 1028 Acked Write 5 Sent Sent Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 11 11 11 33 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1016 1016 1016 Downstairs last five completed: 1016 1015 1014 1013 1012 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:35:45.416519464Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1462} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 68 68 68 131 131 131 F F F 1 020-039 62 62 62 121 121 121 F F F 2 040-059 70 70 70 134 134 134 F F F 3 060-079 66 66 66 128 128 128 F F F 4 080-099 68 68 68 131 131 131 F F F 5 100-119 68 68 68 131 131 131 F F F 6 120-139 64 64 64 125 125 125 F F F 7 140-159 66 66 66 127 127 127 T T T 8 160-179 67 67 67 129 129 129 F F F 9 180-199 66 66 66 128 128 128 F F F 10 200-219 70 70 70 134 134 134 T T T 11 220-239 70 70 70 134 134 134 T T T 12 240-259 70 70 70 134 134 134 T T T 13 260-279 65 65 65 126 126 126 T T T 14 280-299 64 64 64 125 125 125 F F F 15 300-319 65 65 65 126 126 126 F F F 16 320-339 58 58 58 113 113 113 F F F 17 340-359 70 70 70 134 134 134 F F F 18 360-379 67 67 67 129 129 129 F F F 19 380-399 68 68 68 131 131 131 F F F 20 400-419 68 68 68 131 131 131 F F F 21 420-439 63 63 63 123 123 123 T T T 22 440-459 69 69 69 132 132 132 F F F 23 460-479 62 62 62 117 117 117 F F F 24 480-499 69 69 69 132 132 132 F F F 25 500-519 68 68 68 130 130 130 F F F 26 520-539 68 68 68 130 130 130 F F F 27 540-559 68 68 68 131 131 131 F F F 28 560-579 69 69 69 132 132 132 F F F 29 580-599 68 68 68 130 130 130 F F F Max gen: 70, Max flush: 134 A Difference in extent metadata is expected here On loop 35 Verifying data now No change after verify Loop: 35 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 68 68 68 131 131 131 F F F 1 020-039 62 62 62 121 121 121 F F F 2 040-059 70 70 70 134 134 134 F F F 3 060-079 66 66 66 128 128 128 F F F 4 080-099 68 68 68 131 131 131 F F F 5 100-119 68 68 68 131 131 131 F F F 6 120-139 64 64 64 125 125 125 F F F 7 140-159 71 71 71 135 135 135 F F F 8 160-179 67 67 67 129 129 129 F F F 9 180-199 66 66 66 128 128 128 F F F 10 200-219 71 71 71 135 135 135 F F F 11 220-239 71 71 71 135 135 135 F F F 12 240-259 71 71 71 135 135 135 F F F 13 260-279 71 71 71 135 135 135 F F F 14 280-299 64 64 64 125 125 125 F F F 15 300-319 65 65 65 126 126 126 F F F 16 320-339 58 58 58 113 113 113 F F F 17 340-359 70 70 70 134 134 134 F F F 18 360-379 67 67 67 129 129 129 F F F 19 380-399 68 68 68 131 131 131 F F F 20 400-419 68 68 68 131 131 131 F F F 21 420-439 71 71 71 135 135 135 F F F 22 440-459 69 69 69 132 132 132 F F F 23 460-479 62 62 62 117 117 117 F F F 24 480-499 69 69 69 132 132 132 F F F 25 500-519 68 68 68 130 130 130 F F F 26 520-539 68 68 68 130 130 130 F F F 27 540-559 68 68 68 131 131 131 F F F 28 560-579 69 69 69 132 132 132 F F F 29 580-599 68 68 68 130 130 130 F F F Max gen: 71, Max flush: 135 Begin loop 36 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:49.692209409Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1470,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:49.738294891Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1470,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:49.738972815Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1470} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"26e7d501b33f47379e53601720353c2983e7654b\",\n git_commit_timestamp: \"2024-12-12T20:16:40.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:49.739009142Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1470} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:49.739023221Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1470} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:49.73904119Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1470} {"msg":"Crucible 82e7dc69-3105-4482-9084-b8ac79fa6bd1 has session id: a04648bb-f4b5-4a10-bf54-c355792a55a2","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:49.73947621Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1470,"session_id":"a04648bb-f4b5-4a10-bf54-c355792a55a2"} {"msg":"Upstairs opts: Upstairs UUID: 82e7dc69-3105-4482-9084-b8ac79fa6bd1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:49.739523377Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1470,"session_id":"a04648bb-f4b5-4a10-bf54-c355792a55a2"} {"msg":"Crucible stats registered with UUID: 82e7dc69-3105-4482-9084-b8ac79fa6bd1","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:49.739542215Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1470,"session_id":"a04648bb-f4b5-4a10-bf54-c355792a55a2"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:49.770520259Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1470,"task":"crutest"} {"msg":"The guest has requested activation with gen:72","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:49.770546797Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1470} {"msg":"82e7dc69-3105-4482-9084-b8ac79fa6bd1 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:49.770616213Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1470,"session_id":"a04648bb-f4b5-4a10-bf54-c355792a55a2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:49.7706481Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1470,"client":"0","":"downstairs","session_id":"a04648bb-f4b5-4a10-bf54-c355792a55a2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:49.770677098Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1470,"client":"0","":"downstairs","session_id":"a04648bb-f4b5-4a10-bf54-c355792a55a2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:49.770700907Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1470,"client":"1","":"downstairs","session_id":"a04648bb-f4b5-4a10-bf54-c355792a55a2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:49.770733565Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1470,"client":"1","":"downstairs","session_id":"a04648bb-f4b5-4a10-bf54-c355792a55a2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:49.770766232Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1470,"client":"2","":"downstairs","session_id":"a04648bb-f4b5-4a10-bf54-c355792a55a2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:49.77080204Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1470,"client":"2","":"downstairs","session_id":"a04648bb-f4b5-4a10-bf54-c355792a55a2"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:49.770834527Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1470,"":"io task","client":"2","":"downstairs","session_id":"a04648bb-f4b5-4a10-bf54-c355792a55a2"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:49.770869585Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1470,"":"io task","client":"0","":"downstairs","session_id":"a04648bb-f4b5-4a10-bf54-c355792a55a2"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:49.770908353Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1470,"":"io task","client":"1","":"downstairs","session_id":"a04648bb-f4b5-4a10-bf54-c355792a55a2"} {"msg":"ds_connection connected from Ok(127.0.0.1:51863)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:49.77094272Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1470,"":"io task","client":"2","":"downstairs","session_id":"a04648bb-f4b5-4a10-bf54-c355792a55a2"} {"msg":"ds_connection connected from Ok(127.0.0.1:62244)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:49.770976858Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1470,"":"io task","client":"1","":"downstairs","session_id":"a04648bb-f4b5-4a10-bf54-c355792a55a2"} {"msg":"ds_connection connected from Ok(127.0.0.1:48216)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:49.771055492Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1470,"":"io task","client":"0","":"downstairs","session_id":"a04648bb-f4b5-4a10-bf54-c355792a55a2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:49.771594885Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1470,"client":"1","":"downstairs","session_id":"a04648bb-f4b5-4a10-bf54-c355792a55a2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:49.771622713Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1470,"client":"0","":"downstairs","session_id":"a04648bb-f4b5-4a10-bf54-c355792a55a2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:49.771638282Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1470,"client":"2","":"downstairs","session_id":"a04648bb-f4b5-4a10-bf54-c355792a55a2"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:49.772284998Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1470,"client":"1","":"downstairs","session_id":"a04648bb-f4b5-4a10-bf54-c355792a55a2"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:49.772311716Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1470,"client":"2","":"downstairs","session_id":"a04648bb-f4b5-4a10-bf54-c355792a55a2"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:49.772368072Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1470,"client":"0","":"downstairs","session_id":"a04648bb-f4b5-4a10-bf54-c355792a55a2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:49.772850839Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1470,"session_id":"a04648bb-f4b5-4a10-bf54-c355792a55a2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:49.772947562Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1470,"session_id":"a04648bb-f4b5-4a10-bf54-c355792a55a2"} {"msg":"[0]R flush_numbers[0..12]: [131, 121, 134, 128, 131, 131, 125, 135, 129, 128, 135, 135]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:49.773021377Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1470,"":"downstairs","":"downstairs","session_id":"a04648bb-f4b5-4a10-bf54-c355792a55a2"} {"msg":"[0]R generation[0..12]: [68, 62, 70, 66, 68, 68, 64, 71, 67, 66, 71, 71]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:49.773040055Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1470,"":"downstairs","":"downstairs","session_id":"a04648bb-f4b5-4a10-bf54-c355792a55a2"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:49.773057504Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1470,"":"downstairs","":"downstairs","session_id":"a04648bb-f4b5-4a10-bf54-c355792a55a2"} {"msg":"[1]R flush_numbers[0..12]: [131, 121, 134, 128, 131, 131, 125, 135, 129, 128, 135, 135]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:49.773071803Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1470,"":"downstairs","":"downstairs","session_id":"a04648bb-f4b5-4a10-bf54-c355792a55a2"} {"msg":"[1]R generation[0..12]: [68, 62, 70, 66, 68, 68, 64, 71, 67, 66, 71, 71]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:49.773085362Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1470,"":"downstairs","":"downstairs","session_id":"a04648bb-f4b5-4a10-bf54-c355792a55a2"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:49.773099431Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1470,"":"downstairs","":"downstairs","session_id":"a04648bb-f4b5-4a10-bf54-c355792a55a2"} {"msg":"[2]R flush_numbers[0..12]: [131, 121, 134, 128, 131, 131, 125, 135, 129, 128, 135, 135]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:49.77311329Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1470,"":"downstairs","":"downstairs","session_id":"a04648bb-f4b5-4a10-bf54-c355792a55a2"} {"msg":"[2]R generation[0..12]: [68, 62, 70, 66, 68, 68, 64, 71, 67, 66, 71, 71]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:49.773126809Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1470,"":"downstairs","":"downstairs","session_id":"a04648bb-f4b5-4a10-bf54-c355792a55a2"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:49.773140639Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1470,"":"downstairs","":"downstairs","session_id":"a04648bb-f4b5-4a10-bf54-c355792a55a2"} {"msg":"Max found gen is 72","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:49.773165717Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1470,"":"downstairs","":"downstairs","session_id":"a04648bb-f4b5-4a10-bf54-c355792a55a2"} {"msg":"Generation requested: 72 >= found:72","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:49.773179756Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1470,"":"downstairs","":"downstairs","session_id":"a04648bb-f4b5-4a10-bf54-c355792a55a2"} {"msg":"Next flush: 136","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:49.773192895Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1470,"":"downstairs","":"downstairs","session_id":"a04648bb-f4b5-4a10-bf54-c355792a55a2"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-12-12T20:35:49.773206184Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1470,"":"downstairs","":"downstairs","session_id":"a04648bb-f4b5-4a10-bf54-c355792a55a2"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:49.773220703Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1470,"session_id":"a04648bb-f4b5-4a10-bf54-c355792a55a2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:49.773232722Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1470,"client":"0","":"downstairs","session_id":"a04648bb-f4b5-4a10-bf54-c355792a55a2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:49.773245961Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1470,"client":"1","":"downstairs","session_id":"a04648bb-f4b5-4a10-bf54-c355792a55a2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:49.7732609Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1470,"client":"2","":"downstairs","session_id":"a04648bb-f4b5-4a10-bf54-c355792a55a2"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:49.773274329Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1470,"session_id":"a04648bb-f4b5-4a10-bf54-c355792a55a2"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:49.773286578Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1470,"session_id":"a04648bb-f4b5-4a10-bf54-c355792a55a2"} {"msg":"82e7dc69-3105-4482-9084-b8ac79fa6bd1 is now active with session: a04648bb-f4b5-4a10-bf54-c355792a55a2","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:49.773299228Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1470,"session_id":"a04648bb-f4b5-4a10-bf54-c355792a55a2"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:49.773311117Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1470,"session_id":"a04648bb-f4b5-4a10-bf54-c355792a55a2"} {"msg":"The guest has finished waiting for activation with:72","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:49.773323296Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1470} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 397 len 16384 data: 6 8 9 8 02/30 Flush 03/30 Write block 591 len 8192 data: 7 7 04/30 Read block 392 len 36864 05/30 Read block 30 len 32768 06/30 Read block 396 len 36864 07/30 Write block 581 len 20480 data: 7 8 8 9 8 08/30 Read block 15 len 12288 09/30 Read block 540 len 4096 10/30 Read block 222 len 20480 11/30 Read block 468 len 12288 12/30 Write block 367 len 32768 data: 7 4 5 4 4 5 5 7 13/30 Read block 505 len 8192 14/30 Read block 159 len 24576 15/30 Read block 248 len 8192 16/30 Read block 569 len 8192 17/30 Read block 270 len 32768 18/30 Read block 459 len 36864 19/30 Read block 502 len 20480 20/30 Read block 573 len 8192 21/30 Read block 418 len 24576 22/30 Write block 517 len 36864 data: 8 11 14 15 14 12 16 15 14 23/30 Write block 381 len 4096 data: 9 24/30 Read block 57 len 28672 25/30 Write block 53 len 8192 data: 5 6 26/30 Read block 547 len 20480 27/30 Write block 123 len 40960 data: 8 7 7 6 5 7 7 7 6 4 28/30 Read block 276 len 8192 29/30 Write block 246 len 12288 data: 3 2 3 30/30 Read block 381 len 32768 ---------------------------------------------------------------- Crucible gen:72 GIO:true work queues: Upstairs:0 downstairs:28 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1002 Acked Write 2 Done Done Done false 1003 Acked Read 9 Done Done Done false 1004 Acked Read 8 Done Done Done false 1005 Acked Read 9 Done Done Done false 1006 Acked Write 5 Done Done Done false 1007 Acked Read 3 Done Done Done false 1008 Acked Read 1 Done Done Done false 1009 Acked Read 5 Done Done Done false 1010 Acked Read 3 Done Done Done false 1011 Acked Write 8 Done Done Done false 1012 Acked Read 2 Done Done Done false 1013 Acked Read 6 Done Done Done false 1014 Acked Read 2 Done Done Done false 1015 Acked Read 2 Done Done Done false 1016 Acked Read 8 Done Done Done false 1017 Acked Read 9 Done Done Done false 1018 Acked Read 5 Done Done Done false 1019 Acked Read 2 Done Done Done false 1020 Acked Read 6 Done Done Done false 1021 Acked Write 9 Done Done Done false 1022 Acked Write 1 Done Done Done false 1023 Acked Read 7 Done Done Done false 1024 Acked Write 2 Done Done Done false 1025 Acked Read 5 Done Done Done false 1026 Acked Write 10 Done Done Done false 1027 Acked Read 2 Done Done Done false 1028 Acked Write 3 Done Done Sent false 1029 Acked Read 8 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 2 3 Done 28 27 26 81 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1001 1001 1001 Downstairs last five completed: 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:35:49.792488564Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1470} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 68 68 68 131 131 131 F F F 1 020-039 62 62 62 121 121 121 F F F 2 040-059 70 70 70 134 134 134 T T T 3 060-079 66 66 66 128 128 128 F F F 4 080-099 68 68 68 131 131 131 F F F 5 100-119 68 68 68 131 131 131 F F F 6 120-139 64 64 64 125 125 125 T T T 7 140-159 71 71 71 135 135 135 F F F 8 160-179 67 67 67 129 129 129 F F F 9 180-199 66 66 66 128 128 128 F F F 10 200-219 71 71 71 135 135 135 F F F 11 220-239 71 71 71 135 135 135 F F F 12 240-259 71 71 71 135 135 135 T T T 13 260-279 71 71 71 135 135 135 F F F 14 280-299 64 64 64 125 125 125 F F F 15 300-319 65 65 65 126 126 126 F F F 16 320-339 58 58 58 113 113 113 F F F 17 340-359 70 70 70 134 134 134 F F F 18 360-379 67 67 67 129 129 129 T T T 19 380-399 72 72 72 136 136 136 T T T 20 400-419 72 72 72 136 136 136 F F F 21 420-439 71 71 71 135 135 135 F F F 22 440-459 69 69 69 132 132 132 F F F 23 460-479 62 62 62 117 117 117 F F F 24 480-499 69 69 69 132 132 132 F F F 25 500-519 68 68 68 130 130 130 T T T 26 520-539 68 68 68 130 130 130 T T T 27 540-559 68 68 68 131 131 131 F F F 28 560-579 69 69 69 132 132 132 F F F 29 580-599 68 68 68 130 130 130 T T T Max gen: 72, Max flush: 136 A Difference in extent metadata is expected here On loop 36 Verifying data now No change after verify Loop: 36 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 68 68 68 131 131 131 F F F 1 020-039 62 62 62 121 121 121 F F F 2 040-059 73 73 73 137 137 137 F F F 3 060-079 66 66 66 128 128 128 F F F 4 080-099 68 68 68 131 131 131 F F F 5 100-119 68 68 68 131 131 131 F F F 6 120-139 73 73 73 137 137 137 F F F 7 140-159 71 71 71 135 135 135 F F F 8 160-179 67 67 67 129 129 129 F F F 9 180-199 66 66 66 128 128 128 F F F 10 200-219 71 71 71 135 135 135 F F F 11 220-239 71 71 71 135 135 135 F F F 12 240-259 73 73 73 137 137 137 F F F 13 260-279 71 71 71 135 135 135 F F F 14 280-299 64 64 64 125 125 125 F F F 15 300-319 65 65 65 126 126 126 F F F 16 320-339 58 58 58 113 113 113 F F F 17 340-359 70 70 70 134 134 134 F F F 18 360-379 73 73 73 137 137 137 F F F 19 380-399 73 73 73 137 137 137 F F F 20 400-419 72 72 72 136 136 136 F F F 21 420-439 71 71 71 135 135 135 F F F 22 440-459 69 69 69 132 132 132 F F F 23 460-479 62 62 62 117 117 117 F F F 24 480-499 69 69 69 132 132 132 F F F 25 500-519 73 73 73 137 137 137 F F F 26 520-539 73 73 73 137 137 137 F F F 27 540-559 68 68 68 131 131 131 F F F 28 560-579 69 69 69 132 132 132 F F F 29 580-599 73 73 73 137 137 137 F F F Max gen: 73, Max flush: 137 Begin loop 37 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:54.076950305Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1478,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:54.121562829Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1478,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:54.122248491Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1478} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"26e7d501b33f47379e53601720353c2983e7654b\",\n git_commit_timestamp: \"2024-12-12T20:16:40.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:54.122299798Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1478} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:54.122322326Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1478} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:54.122346495Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1478} {"msg":"Crucible 006bacf6-51d9-479e-bc4e-f4ee2b9fae53 has session id: 0b31bdec-0d0a-4807-92ca-215b37386d59","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:54.122818922Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1478,"session_id":"0b31bdec-0d0a-4807-92ca-215b37386d59"} {"msg":"Upstairs opts: Upstairs UUID: 006bacf6-51d9-479e-bc4e-f4ee2b9fae53, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:54.122855009Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1478,"session_id":"0b31bdec-0d0a-4807-92ca-215b37386d59"} {"msg":"Crucible stats registered with UUID: 006bacf6-51d9-479e-bc4e-f4ee2b9fae53","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:54.122870719Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1478,"session_id":"0b31bdec-0d0a-4807-92ca-215b37386d59"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:54.153913828Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1478,"task":"crutest"} {"msg":"The guest has requested activation with gen:74","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:54.153945226Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1478} {"msg":"006bacf6-51d9-479e-bc4e-f4ee2b9fae53 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:54.153997892Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1478,"session_id":"0b31bdec-0d0a-4807-92ca-215b37386d59"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:54.154018001Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1478,"client":"0","":"downstairs","session_id":"0b31bdec-0d0a-4807-92ca-215b37386d59"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:54.154046749Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1478,"client":"0","":"downstairs","session_id":"0b31bdec-0d0a-4807-92ca-215b37386d59"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:54.154064677Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1478,"client":"1","":"downstairs","session_id":"0b31bdec-0d0a-4807-92ca-215b37386d59"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:54.154083436Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1478,"client":"1","":"downstairs","session_id":"0b31bdec-0d0a-4807-92ca-215b37386d59"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:54.154114314Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1478,"client":"2","":"downstairs","session_id":"0b31bdec-0d0a-4807-92ca-215b37386d59"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:54.154147782Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1478,"client":"2","":"downstairs","session_id":"0b31bdec-0d0a-4807-92ca-215b37386d59"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:54.154192249Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1478,"":"io task","client":"2","":"downstairs","session_id":"0b31bdec-0d0a-4807-92ca-215b37386d59"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:54.154227336Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1478,"":"io task","client":"0","":"downstairs","session_id":"0b31bdec-0d0a-4807-92ca-215b37386d59"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:54.154261044Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1478,"":"io task","client":"1","":"downstairs","session_id":"0b31bdec-0d0a-4807-92ca-215b37386d59"} {"msg":"ds_connection connected from Ok(127.0.0.1:37211)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:54.15446707Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1478,"":"io task","client":"0","":"downstairs","session_id":"0b31bdec-0d0a-4807-92ca-215b37386d59"} {"msg":"ds_connection connected from Ok(127.0.0.1:39074)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:54.154489708Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1478,"":"io task","client":"2","":"downstairs","session_id":"0b31bdec-0d0a-4807-92ca-215b37386d59"} {"msg":"ds_connection connected from Ok(127.0.0.1:47893)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:54.154507787Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1478,"":"io task","client":"1","":"downstairs","session_id":"0b31bdec-0d0a-4807-92ca-215b37386d59"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:54.155095097Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1478,"client":"1","":"downstairs","session_id":"0b31bdec-0d0a-4807-92ca-215b37386d59"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:54.155137863Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1478,"client":"2","":"downstairs","session_id":"0b31bdec-0d0a-4807-92ca-215b37386d59"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:54.155178161Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1478,"client":"0","":"downstairs","session_id":"0b31bdec-0d0a-4807-92ca-215b37386d59"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:54.155678056Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1478,"client":"1","":"downstairs","session_id":"0b31bdec-0d0a-4807-92ca-215b37386d59"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:54.155872563Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1478,"client":"0","":"downstairs","session_id":"0b31bdec-0d0a-4807-92ca-215b37386d59"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:54.156009283Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1478,"client":"2","":"downstairs","session_id":"0b31bdec-0d0a-4807-92ca-215b37386d59"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:54.156224099Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1478,"session_id":"0b31bdec-0d0a-4807-92ca-215b37386d59"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:54.156308823Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1478,"session_id":"0b31bdec-0d0a-4807-92ca-215b37386d59"} {"msg":"[0]R flush_numbers[0..12]: [131, 121, 137, 128, 131, 131, 137, 135, 129, 128, 135, 135]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:54.156513699Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1478,"":"downstairs","":"downstairs","session_id":"0b31bdec-0d0a-4807-92ca-215b37386d59"} {"msg":"[0]R generation[0..12]: [68, 62, 73, 66, 68, 68, 73, 71, 67, 66, 71, 71]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:54.156533347Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1478,"":"downstairs","":"downstairs","session_id":"0b31bdec-0d0a-4807-92ca-215b37386d59"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:54.156548586Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1478,"":"downstairs","":"downstairs","session_id":"0b31bdec-0d0a-4807-92ca-215b37386d59"} {"msg":"[1]R flush_numbers[0..12]: [131, 121, 137, 128, 131, 131, 137, 135, 129, 128, 135, 135]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:54.156562915Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1478,"":"downstairs","":"downstairs","session_id":"0b31bdec-0d0a-4807-92ca-215b37386d59"} {"msg":"[1]R generation[0..12]: [68, 62, 73, 66, 68, 68, 73, 71, 67, 66, 71, 71]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:54.156576844Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1478,"":"downstairs","":"downstairs","session_id":"0b31bdec-0d0a-4807-92ca-215b37386d59"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:54.156590783Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1478,"":"downstairs","":"downstairs","session_id":"0b31bdec-0d0a-4807-92ca-215b37386d59"} {"msg":"[2]R flush_numbers[0..12]: [131, 121, 137, 128, 131, 131, 137, 135, 129, 128, 135, 135]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:54.156604382Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1478,"":"downstairs","":"downstairs","session_id":"0b31bdec-0d0a-4807-92ca-215b37386d59"} {"msg":"[2]R generation[0..12]: [68, 62, 73, 66, 68, 68, 73, 71, 67, 66, 71, 71]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:54.156617921Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1478,"":"downstairs","":"downstairs","session_id":"0b31bdec-0d0a-4807-92ca-215b37386d59"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:54.15663464Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1478,"":"downstairs","":"downstairs","session_id":"0b31bdec-0d0a-4807-92ca-215b37386d59"} {"msg":"Max found gen is 74","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:54.156647989Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1478,"":"downstairs","":"downstairs","session_id":"0b31bdec-0d0a-4807-92ca-215b37386d59"} {"msg":"Generation requested: 74 >= found:74","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:54.156661138Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1478,"":"downstairs","":"downstairs","session_id":"0b31bdec-0d0a-4807-92ca-215b37386d59"} {"msg":"Next flush: 138","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:54.156674097Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1478,"":"downstairs","":"downstairs","session_id":"0b31bdec-0d0a-4807-92ca-215b37386d59"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"timeWait for a query_work_queue command to finish before sending IO ":"2024-12-12T20:35:54.156686847Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1478,"":"downstairs","":"downstairs","session_id":"0b31bdec-0d0a-4807-92ca-215b37386d59"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:54.156704806Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1478,"session_id":"0b31bdec-0d0a-4807-92ca-215b37386d59"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:54.156716934Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1478,"client":"0","":"downstairs","session_id":"0b31bdec-0d0a-4807-92ca-215b37386d59"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:54.156730284Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1478,"client":"1","":"downstairs","session_id":"0b31bdec-0d0a-4807-92ca-215b37386d59"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:54.156743453Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1478,"client":"2","":"downstairs","session_id":"0b31bdec-0d0a-4807-92ca-215b37386d59"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:54.156756722Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1478,"session_id":"0b31bdec-0d0a-4807-92ca-215b37386d59"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:54.15678195Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1478,"session_id":"0b31bdec-0d0a-4807-92ca-215b37386d59"} {"msg":"006bacf6-51d9-479e-bc4e-f4ee2b9fae53 is now active with session: 0b31bdec-0d0a-4807-92ca-215b37386d59","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:54.156795499Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1478,"session_id":"0b31bdec-0d0a-4807-92ca-215b37386d59"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:54.156807638Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1478,"session_id":"0b31bdec-0d0a-4807-92ca-215b37386d59"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 {"msg":"The guest has finished waiting for activation with:74","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:54.156822877Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1478} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 414 len 36864 data: 6 5 6 6 8 7 7 7 8 02/30 Write block 536 len 28672 data: 6 6 4 4 7 4 6 03/30 Read block 569 len 32768 04/30 Write block 466 len 4096 data: 6 05/30 Read block 360 len 20480 06/30 Write block 406 len 40960 data: 9 9 10 8 7 6 4 5 7 6 07/30 Flush 08/30 Write block 27 len 8192 data: 5 5 09/30 Write block 199 len 20480 data: 5 5 6 7 8 10/30 Read block 550 len 8192 11/30 Flush 12/30 Write block 536 len 20480 data: 7 7 5 5 8 13/30 Read block 581 len 40960 14/30 Write block 321 len 24576 data: 6 7 6 4 3 3 15/30 Write block 470 len 36864 data: 4 4 3 4 4 5 5 4 6 16/30 Write block 146 len 12288 data: 6 6 7 17/30 Read block 352 len 36864 18/30 Write block 160 len 28672 data: 7 6 8 6 5 4 4 19/30 Write block 540 len 8192 data: 9 5 20/30 Read block 161 len 32768 21/30 Read block 137 len 8192 22/30 Write block 246 len 12288 data: 4 3 4 23/30 Read block 259 len 20480 24/30 Flush 25/30 Write block 267 len 32768 data: 10 10 5 5 5 8 6 6 26/30 Flush 27/30 Write block 344 len 28672 data: 6 6 8 8 7 6 9 28/30 Read block 335 len 40960 29/30 Read block 513 len 40960 30/30 Read block 46 len 4096 ---------------------------------------------------------------- Crucible gen:74 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 7 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Read 10 Done Done Done false 1029 Acked Read 1 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:35:54.176729435Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1478} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 68 68 68 131 131 131 F F F 1 020-039 74 74 74 139 139 139 F F F 2 040-059 73 73 73 137 137 137 F F F 3 060-079 66 66 66 128 128 128 F F F 4 080-099 68 68 68 131 131 131 F F F 5 100-119 68 68 68 131 131 131 F F F 6 120-139 73 73 73 137 137 137 F F F 7 140-159 74 74 74 140 140 140 F F F 8 160-179 74 74 74 140 140 140 F F F 9 180-199 74 74 74 139 139 139 F F F 10 200-219 74 74 74 139 139 139 F F F 11 220-239 71 71 71 135 135 135 F F F 12 240-259 74 74 74 140 140 140 F F F 13 260-279 74 74 74 141 141 141 F F F 14 280-299 64 64 64 125 125 125 F F F 15 300-319 65 65 65 126 126 126 F F F 16 320-339 74 74 74 140 140 140 F F F 17 340-359 70 70 70 134 134 134 T T T 18 360-379 73 73 73 137 137 137 F F F 19 380-399 73 73 73 137 137 137 F F F 20 400-419 74 74 74 138 138 138 F F F 21 420-439 74 74 74 138 138 138 F F F 22 440-459 69 69 69 132 132 132 F F F 23 460-479 74 74 74 140 140 140 F F F 24 480-499 69 69 69 132 132 132 F F F 25 500-519 73 73 73 137 137 137 F F F 26 520-539 74 74 74 140 140 140 F F F 27 540-559 74 74 74 140 140 140 F F F 28 560-579 69 69 69 132 132 132 F F F 29 580-599 73 73 73 137 137 137 F F F Max gen: 74, Max flush: 141 A Difference in extent metadata is expected here On loop 37 Verifying data now No change after verify Loop: 37 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 68 68 68 131 131 131 F F F 1 020-039 74 74 74 139 139 139 F F F 2 040-059 73 73 73 137 137 137 F F F 3 060-079 66 66 66 128 128 128 F F F 4 080-099 68 68 68 131 131 131 F F F 5 100-119 68 68 68 131 131 131 F F F 6 120-139 73 73 73 137 137 137 F F F 7 140-159 74 74 74 140 140 140 F F F 8 160-179 74 74 74 140 140 140 F F F 9 180-199 74 74 74 139 139 139 F F F 10 200-219 74 74 74 139 139 139 F F F 11 220-239 71 71 71 135 135 135 F F F 12 240-259 74 74 74 140 140 140 F F F 13 260-279 74 74 74 141 141 141 F F F 14 280-299 64 64 64 125 125 125 F F F 15 300-319 65 65 65 126 126 126 F F F 16 320-339 74 74 74 140 140 140 F F F 17 340-359 75 75 75 142 142 142 F F F 18 360-379 73 73 73 137 137 137 F F F 19 380-399 73 73 73 137 137 137 F F F 20 400-419 74 74 74 138 138 138 F F F 21 420-439 74 74 74 138 138 138 F F F 22 440-459 69 69 69 132 132 132 F F F 23 460-479 74 74 74 140 140 140 F F F 24 480-499 69 69 69 132 132 132 F F F 25 500-519 73 73 73 137 137 137 F F F 26 520-539 74 74 74 140 140 140 F F F 27 540-559 74 74 74 140 140 140 F F F 28 560-579 69 69 69 132 132 132 F F F 29 580-599 73 73 73 137 137 137 F F F Max gen: 75, Max flush: 142 Begin loop 38 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:58.429023324Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1486,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:58.473840293Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1486,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:58.474501448Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1486} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"26e7d501b33f47379e53601720353c2983e7654b\",\n git_commit_timestamp: \"2024-12-12T20:16:40.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:58.474537595Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1486} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:58.474551984Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1486} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:58.474566203Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1486} {"msg":"Crucible 708585a0-b7f8-4476-9534-1bcf2731225c has session id: e30bb22c-5125-4369-939d-36c30ccf32a7","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:58.475004593Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1486,"session_id":"e30bb22c-5125-4369-939d-36c30ccf32a7"} {"msg":"Upstairs opts: Upstairs UUID: 708585a0-b7f8-4476-9534-1bcf2731225c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:58.475050779Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1486,"session_id":"e30bb22c-5125-4369-939d-36c30ccf32a7"} {"msg":"Crucible stats registered with UUID: 708585a0-b7f8-4476-9534-1bcf2731225c","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:58.475080517Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1486,"session_id":"e30bb22c-5125-4369-939d-36c30ccf32a7"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:58.50563912Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1486,"task":"crutest"} {"msg":"The guest has requested activation with gen:76","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:58.505664488Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1486} {"msg":"708585a0-b7f8-4476-9534-1bcf2731225c active request set","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:58.505737794Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1486,"session_id":"e30bb22c-5125-4369-939d-36c30ccf32a7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:58.505755962Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1486,"client":"0","":"downstairs","session_id":"e30bb22c-5125-4369-939d-36c30ccf32a7"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:58.505779211Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1486,"client":"0","":"downstairs","session_id":"e30bb22c-5125-4369-939d-36c30ccf32a7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:58.5057934Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1486,"client":"1","":"downstairs","session_id":"e30bb22c-5125-4369-939d-36c30ccf32a7"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:58.505807749Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1486,"client":"1","":"downstairs","session_id":"e30bb22c-5125-4369-939d-36c30ccf32a7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:58.505829887Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1486,"client":"2","":"downstairs","session_id":"e30bb22c-5125-4369-939d-36c30ccf32a7"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:58.505859115Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1486,"client":"2","":"downstairs","session_id":"e30bb22c-5125-4369-939d-36c30ccf32a7"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:58.505884173Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1486,"":"io task","client":"2","":"downstairs","session_id":"e30bb22c-5125-4369-939d-36c30ccf32a7"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:58.505909692Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1486,"":"io task","client":"0","":"downstairs","session_id":"e30bb22c-5125-4369-939d-36c30ccf32a7"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:58.50593514Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1486,"":"io task","client":"1","":"downstairs","session_id":"e30bb22c-5125-4369-939d-36c30ccf32a7"} {"msg":"ds_connection connected from Ok(127.0.0.1:61784)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:58.505961608Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1486,"":"io task","client":"2","":"downstairs","session_id":"e30bb22c-5125-4369-939d-36c30ccf32a7"} {"msg":"ds_connection connected from Ok(127.0.0.1:45894)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:58.505989646Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1486,"":"io task","client":"1","":"downstairs","session_id":"e30bb22c-5125-4369-939d-36c30ccf32a7"} {"msg":"ds_connection connected from Ok(127.0.0.1:47593)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:58.506019534Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1486,"":"io task","client":"0","":"downstairs","session_id":"e30bb22c-5125-4369-939d-36c30ccf32a7"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:58.506744984Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1486,"client":"2","":"downstairs","session_id":"e30bb22c-5125-4369-939d-36c30ccf32a7"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:58.506786641Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1486,"client":"0","":"downstairs","session_id":"e30bb22c-5125-4369-939d-36c30ccf32a7"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:58.506820059Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1486,"client":"1","":"downstairs","session_id":"e30bb22c-5125-4369-939d-36c30ccf32a7"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:58.507295486Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1486,"client":"2","":"downstairs","session_id":"e30bb22c-5125-4369-939d-36c30ccf32a7"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:58.50738743Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1486,"client":"1","":"downstairs","session_id":"e30bb22c-5125-4369-939d-36c30ccf32a7"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:58.507408608Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1486,"client":"0","":"downstairs","session_id":"e30bb22c-5125-4369-939d-36c30ccf32a7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:58.507792642Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1486,"session_id":"e30bb22c-5125-4369-939d-36c30ccf32a7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:58.50796564Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1486,"session_id":"e30bb22c-5125-4369-939d-36c30ccf32a7"} {"msg":"[0]R flush_numbers[0..12]: [131, 139, 137, 128, 131, 131, 137, 140, 140, 139, 139, 135]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:58.507999818Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1486,"":"downstairs","":"downstairs","session_id":"e30bb22c-5125-4369-939d-36c30ccf32a7"} {"msg":"[0]R generation[0..12]: [68, 74, 73, 66, 68, 68, 73, 74, 74, 74, 74, 71]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:58.508023336Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1486,"":"downstairs","":"downstairs","session_id":"e30bb22c-5125-4369-939d-36c30ccf32a7"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:58.508038905Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1486,"":"downstairs","":"downstairs","session_id":"e30bb22c-5125-4369-939d-36c30ccf32a7"} {"msg":"[1]R flush_numbers[0..12]: [131, 139, 137, 128, 131, 131, 137, 140, 140, 139, 139, 135]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:58.508053034Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1486,"":"downstairs","":"downstairs","session_id":"e30bb22c-5125-4369-939d-36c30ccf32a7"} {"msg":"[1]R generation[0..12]: [68, 74, 73, 66, 68, 68, 73, 74, 74, 74, 74, 71]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:58.508066783Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1486,"":"downstairs","":"downstairs","session_id":"e30bb22c-5125-4369-939d-36c30ccf32a7"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:58.508084252Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1486,"":"downstairs","":"downstairs","session_id":"e30bb22c-5125-4369-939d-36c30ccf32a7"} {"msg":"[2]R flush_numbers[0..12]: [131, 139, 137, 128, 131, 131, 137, 140, 140, 139, 139, 135]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:58.508098151Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1486,"":"downstairs","":"downstairs","session_id":"e30bb22c-5125-4369-939d-36c30ccf32a7"} {"msg":"[2]R generation[0..12]: [68, 74, 73, 66, 68, 68, 73, 74, 74, 74, 74, 71]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:58.50811181Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1486,"":"downstairs","":"downstairs","session_id":"e30bb22c-5125-4369-939d-36c30ccf32a7"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:58.508125819Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1486,"":"downstairs","":"downstairs","session_id":"e30bb22c-5125-4369-939d-36c30ccf32a7"} {"msg":"Max found gen is 76","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:58.508138868Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1486,"":"downstairs","":"downstairs","session_id":"e30bb22c-5125-4369-939d-36c30ccf32a7"} {"msg":"Generation requested: 76 >= found:76","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:58.508152067Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1486,"":"downstairs","":"downstairs","session_id":"e30bb22c-5125-4369-939d-36c30ccf32a7"} {"msg":"Next flush: 143","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:58.508164816Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1486,"":"downstairs","":"downstairs","session_id":"e30bb22c-5125-4369-939d-36c30ccf32a7"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:58.508177585Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1486,"":"downstairs","":"downstairs","session_id":"e30bb22c-5125-4369-939d-36c30ccf32a7"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:58.508190654Z","hostname":Wait for a query_work_queue command to finish before sending IO "w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1486,"session_id":"e30bb22c-5125-4369-939d-36c30ccf32a7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:58.508207283Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1486,"client":"0","":"downstairs","session_id":"e30bb22c-5125-4369-939d-36c30ccf32a7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:58.508220492Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1486,"client":"1","":"downstairs","session_id":"e30bb22c-5125-4369-939d-36c30ccf32a7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:58.508233411Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1486,"client":"2","":"downstairs","session_id":"e30bb22c-5125-4369-939d-36c30ccf32a7"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:58.50824639Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1486,"session_id":"e30bb22c-5125-4369-939d-36c30ccf32a7"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:58.50825895Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1486,"session_id":"e30bb22c-5125-4369-939d-36c30ccf32a7"} {"msg":"708585a0-b7f8-4476-9534-1bcf2731225c is now active with session: e30bb22c-5125-4369-939d-36c30ccf32a7","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:58.508271549Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1486,"session_id":"e30bb22c-5125-4369-939d-36c30ccf32a7"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:58.508285258Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1486,"session_id":"e30bb22c-5125-4369-939d-36c30ccf32a7"} {"msg":"The guest has finished waiting for activation with:76","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:35:58.508297617Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1486} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 458 len 32768 data: 6 7 7 7 7 7 6 6 02/30 Flush 03/30 Write block 5 len 28672 data: 5 5 6 8 9 8 7 04/30 Read block 493 len 24576 05/30 Read block 527 len 16384 06/30 Read block 12 len 32768 07/30 Read block 293 len 32768 08/30 Write block 110 len 8192 data: 9 10 09/30 Read block 130 len 4096 10/30 Write block 223 len 32768 data: 6 5 7 9 9 9 11 11 11/30 Read block 251 len 24576 12/30 Read block 476 len 12288 13/30 Write block 347 len 24576 data: 9 8 7 10 6 7 14/30 Read block 168 len 4096 15/30 Flush 16/30 Write block 33 len 12288 data: 8 8 9 17/30 Read block 300 len 20480 18/30 Read block 241 len 28672 19/30 Write block 305 len 4096 data: 2 20/30 Read block 572 len 8192 21/30 Read block 63 len 20480 22/30 Read block 542 len 36864 23/30 Read block 34 len 12288 24/30 Read block 500 len 16384 25/30 Flush 26/30 Write block 305 len 16384 data: 3 2 3 4 27/30 Read block 299 len 20480 28/30 Write block 0 len 20480 data: 4 5 5 5 5 29/30 Read block 550 len 32768 30/30 Read block 252 len 28672 ---------------------------------------------------------------- Crucible gen:76 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 4 Done Done Done false 1026 Acked Read 5 Done Done Done false 1027 Acked Write 5 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Read 7 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:35:58.528602787Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1486} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 76 76 76 144 144 144 T T T 1 020-039 76 76 76 145 145 145 F F F 2 040-059 73 73 73 137 137 137 F F F 3 060-079 66 66 66 128 128 128 F F F 4 080-099 68 68 68 131 131 131 F F F 5 100-119 76 76 76 144 144 144 F F F 6 120-139 73 73 73 137 137 137 F F F 7 140-159 74 74 74 140 140 140 F F F 8 160-179 74 74 74 140 140 140 F F F 9 180-199 74 74 74 139 139 139 F F F 10 200-219 74 74 74 139 139 139 F F F 11 220-239 76 76 76 144 144 144 F F F 12 240-259 74 74 74 140 140 140 F F F 13 260-279 74 74 74 141 141 141 F F F 14 280-299 64 64 64 125 125 125 F F F 15 300-319 76 76 76 145 145 145 T T T 16 320-339 74 74 74 140 140 140 F F F 17 340-359 76 76 76 144 144 144 F F F 18 360-379 73 73 73 137 137 137 F F F 19 380-399 73 73 73 137 137 137 F F F 20 400-419 74 74 74 138 138 138 F F F 21 420-439 74 74 74 138 138 138 F F F 22 440-459 76 76 76 143 143 143 F F F 23 460-479 76 76 76 143 143 143 F F F 24 480-499 69 69 69 132 132 132 F F F 25 500-519 73 73 73 137 137 137 F F F 26 520-539 74 74 74 140 140 140 F F F 27 540-559 74 74 74 140 140 140 F F F 28 560-579 69 69 69 132 132 132 F F F 29 580-599 73 73 73 137 137 137 F F F Max gen: 76, Max flush: 145 A Difference in extent metadata is expected here On loop 38 Verifying data now No change after verify Loop: 38 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 77 77 77 146 146 146 F F F 1 020-039 76 76 76 145 145 145 F F F 2 040-059 73 73 73 137 137 137 F F F 3 060-079 66 66 66 128 128 128 F F F 4 080-099 68 68 68 131 131 131 F F F 5 100-119 76 76 76 144 144 144 F F F 6 120-139 73 73 73 137 137 137 F F F 7 140-159 74 74 74 140 140 140 F F F 8 160-179 74 74 74 140 140 140 F F F 9 180-199 74 74 74 139 139 139 F F F 10 200-219 74 74 74 139 139 139 F F F 11 220-239 76 76 76 144 144 144 F F F 12 240-259 74 74 74 140 140 140 F F F 13 260-279 74 74 74 141 141 141 F F F 14 280-299 64 64 64 125 125 125 F F F 15 300-319 77 77 77 146 146 146 F F F 16 320-339 74 74 74 140 140 140 F F F 17 340-359 76 76 76 144 144 144 F F F 18 360-379 73 73 73 137 137 137 F F F 19 380-399 73 73 73 137 137 137 F F F 20 400-419 74 74 74 138 138 138 F F F 21 420-439 74 74 74 138 138 138 F F F 22 440-459 76 76 76 143 143 143 F F F 23 460-479 76 76 76 143 143 143 F F F 24 480-499 69 69 69 132 132 132 F F F 25 500-519 73 73 73 137 137 137 F F F 26 520-539 74 74 74 140 140 140 F F F 27 540-559 74 74 74 140 140 140 F F F 28 560-579 69 69 69 132 132 132 F F F 29 580-599 73 73 73 137 137 137 F F F Max gen: 77, Max flush: 146 Begin loop 39 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:02.78386988Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1494,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:02.828077551Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1494,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:02.828732386Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1494} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"26e7d501b33f47379e53601720353c2983e7654b\",\n git_commit_timestamp: \"2024-12-12T20:16:40.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:02.828801441Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1494} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:02.828832099Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1494} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:02.828856877Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1494} {"msg":"Crucible 23a56582-4f1a-46c6-90eb-ddd00a016b65 has session id: 33f0e6e1-5700-44a8-b8aa-598ad2f8147f","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:02.829272209Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1494,"session_id":"33f0e6e1-5700-44a8-b8aa-598ad2f8147f"} {"msg":"Upstairs opts: Upstairs UUID: 23a56582-4f1a-46c6-90eb-ddd00a016b65, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:02.829320436Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1494,"session_id":"33f0e6e1-5700-44a8-b8aa-598ad2f8147f"} {"msg":"Crucible stats registered with UUID: 23a56582-4f1a-46c6-90eb-ddd00a016b65","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:02.829344064Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1494,"session_id":"33f0e6e1-5700-44a8-b8aa-598ad2f8147f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:02.860844402Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1494,"task":"crutest"} {"msg":"The guest has requested activation with gen:78","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:02.86087352Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1494} {"msg":"23a56582-4f1a-46c6-90eb-ddd00a016b65 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:02.860946275Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1494,"session_id":"33f0e6e1-5700-44a8-b8aa-598ad2f8147f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:02.860967143Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1494,"client":"0","":"downstairs","session_id":"33f0e6e1-5700-44a8-b8aa-598ad2f8147f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:02.860995701Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1494,"client":"0","":"downstairs","session_id":"33f0e6e1-5700-44a8-b8aa-598ad2f8147f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:02.86101345Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1494,"client":"1","":"downstairs","session_id":"33f0e6e1-5700-44a8-b8aa-598ad2f8147f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:02.861031179Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1494,"client":"1","":"downstairs","session_id":"33f0e6e1-5700-44a8-b8aa-598ad2f8147f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:02.861047748Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1494,"client":"2","":"downstairs","session_id":"33f0e6e1-5700-44a8-b8aa-598ad2f8147f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:02.861065476Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1494,"client":"2","":"downstairs","session_id":"33f0e6e1-5700-44a8-b8aa-598ad2f8147f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:02.861082035Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1494,"":"io task","client":"2","":"downstairs","session_id":"33f0e6e1-5700-44a8-b8aa-598ad2f8147f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:02.861100804Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1494,"":"io task","client":"0","":"downstairs","session_id":"33f0e6e1-5700-44a8-b8aa-598ad2f8147f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:02.861137402Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1494,"":"io task","client":"1","":"downstairs","session_id":"33f0e6e1-5700-44a8-b8aa-598ad2f8147f"} {"msg":"ds_connection connected from Ok(127.0.0.1:55621)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:02.861173149Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1494,"":"io task","client":"2","":"downstairs","session_id":"33f0e6e1-5700-44a8-b8aa-598ad2f8147f"} {"msg":"ds_connection connected from Ok(127.0.0.1:50956)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:02.861246104Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1494,"":"io task","client":"1","":"downstairs","session_id":"33f0e6e1-5700-44a8-b8aa-598ad2f8147f"} {"msg":"ds_connection connected from Ok(127.0.0.1:62370)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:02.861498217Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1494,"":"io task","client":"0","":"downstairs","session_id":"33f0e6e1-5700-44a8-b8aa-598ad2f8147f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:02.8618802Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1494,"client":"0","":"downstairs","session_id":"33f0e6e1-5700-44a8-b8aa-598ad2f8147f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:02.861910258Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1494,"client":"1","":"downstairs","session_id":"33f0e6e1-5700-44a8-b8aa-598ad2f8147f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:02.862040209Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1494,"client":"2","":"downstairs","session_id":"33f0e6e1-5700-44a8-b8aa-598ad2f8147f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:02.862482519Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1494,"client":"0","":"downstairs","session_id":"33f0e6e1-5700-44a8-b8aa-598ad2f8147f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:02.862592901Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1494,"client":"1","":"downstairs","session_id":"33f0e6e1-5700-44a8-b8aa-598ad2f8147f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:02.862715893Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1494,"client":"2","":"downstairs","session_id":"33f0e6e1-5700-44a8-b8aa-598ad2f8147f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:02.863093527Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1494,"session_id":"33f0e6e1-5700-44a8-b8aa-598ad2f8147f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:02.86318634Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1494,"session_id":"33f0e6e1-5700-44a8-b8aa-598ad2f8147f"} {"msg":"[0]R flush_numbers[0..12]: [146, 145, 137, 128, 131, 144, 137, 140, 140, 139, 139, 144]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:02.863257605Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1494,"":"downstairs","":"downstairs","session_id":"33f0e6e1-5700-44a8-b8aa-598ad2f8147f"} {"msg":"[0]R generation[0..12]: [77, 76, 73, 66, 68, 76, 73, 74, 74, 74, 74, 76]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:02.863277174Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1494,"":"downstairs","":"downstairs","session_id":"33f0e6e1-5700-44a8-b8aa-598ad2f8147f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:02.863294453Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1494,"":"downstairs","":"downstairs","session_id":"33f0e6e1-5700-44a8-b8aa-598ad2f8147f"} {"msg":"[1]R flush_numbers[0..12]: [146, 145, 137, 128, 131, 144, 137, 140, 140, 139, 139, 144]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:02.863308642Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1494,"":"downstairs","":"downstairs","session_id":"33f0e6e1-5700-44a8-b8aa-598ad2f8147f"} {"msg":"[1]R generation[0..12]: [77, 76, 73, 66, 68, 76, 73, 74, 74, 74, 74, 76]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:02.863322431Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1494,"":"downstairs","":"downstairs","session_id":"33f0e6e1-5700-44a8-b8aa-598ad2f8147f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:02.8633364Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1494,"":"downstairs","":"downstairs","session_id":"33f0e6e1-5700-44a8-b8aa-598ad2f8147f"} {"msg":"[2]R flush_numbers[0..12]: [146, 145, 137, 128, 131, 144, 137, 140, 140, 139, 139, 144]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:02.863350369Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1494,"":"downstairs","":"downstairs","session_id":"33f0e6e1-5700-44a8-b8aa-598ad2f8147f"} {"msg":"[2]R generation[0..12]: [77, 76, 73, 66, 68, 76, 73, 74, 74, 74, 74, 76]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:02.863364328Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1494,"":"downstairs","":"downstairs","session_id":"33f0e6e1-5700-44a8-b8aa-598ad2f8147f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:02.863378457Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1494,"":"downstairs","":"downstairs","session_id":"33f0e6e1-5700-44a8-b8aa-598ad2f8147f"} {"msg":"Max found gen is 78","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:02.863391406Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1494,"":"downstairs","":"downstairs","session_id":"33f0e6e1-5700-44a8-b8aa-598ad2f8147f"} {"msg":"Generation requested: 78 >= found:78","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:02.863404745Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1494,"":"downstairs","":"downstairs","session_id":"33f0e6e1-5700-44a8-b8aa-598ad2f8147f"} {"msg":"Next flush: 147","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:02.863417704Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1494,"":"downstairs","":"downstairs","session_id":"33f0e6e1-5700-44a8-b8aa-598ad2f8147f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:02.863439873Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1494,"":"downstairs","":"downstairs","session_id":"33f0e6e1-5700-44a8-b8aa-598ad2f8147f"} Wait for a query_work_queue command to finish before sending IO {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:02.863470931Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1494,"session_id":"33f0e6e1-5700-44a8-b8aa-598ad2f8147f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:02.863496649Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1494,"client":"0","":"downstairs","session_id":"33f0e6e1-5700-44a8-b8aa-598ad2f8147f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:02.863520377Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1494,"client":"1","":"downstairs","session_id":"33f0e6e1-5700-44a8-b8aa-598ad2f8147f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:02.863534586Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1494,"client":"2","":"downstairs","session_id":"33f0e6e1-5700-44a8-b8aa-598ad2f8147f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:02.863556045Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1494,"session_id":"33f0e6e1-5700-44a8-b8aa-598ad2f8147f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:02.863579673Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1494,"session_id":"33f0e6e1-5700-44a8-b8aa-598ad2f8147f"} {"msg":"23a56582-4f1a-46c6-90eb-ddd00a016b65 is now active with session: 33f0e6e1-5700-44a8-b8aa-598ad2f8147f","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:02.863605701Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1494,"session_id":"33f0e6e1-5700-44a8-b8aa-598ad2f8147f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:02.86362443Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1494,"session_id":"33f0e6e1-5700-44a8-b8aa-598ad2f8147f"} {"msg":"The guest has finished waiting for activation with:78","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:02.863637419Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1494} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 167 len 28672 data: 3 3 3 3 2 3 3 02/30 Read block 556 len 16384 03/30 Read block 283 len 24576 04/30 Write block 70 len 16384 data: 5 5 6 6 05/30 Read block 464 len 32768 06/30 Read block 538 len 36864 07/30 Flush 08/30 Write block 133 len 40960 data: 4 4 4 6 6 5 4 4 3 6 09/30 Write block 270 len 40960 data: 6 6 9 7 7 6 4 4 5 5 10/30 Read block 71 len 20480 11/30 Read block 304 len 40960 12/30 Read block 376 len 40960 13/30 Write block 345 len 40960 data: 7 9 10 9 8 11 7 8 7 9 14/30 Read block 231 len 36864 15/30 Write block 149 len 20480 data: 5 5 4 4 5 16/30 Read block 93 len 28672 17/30 Write block 189 len 32768 data: 8 8 8 8 8 6 6 7 18/30 Read block 327 len 20480 19/30 Read block 124 len 16384 20/30 Write block 174 len 16384 data: 2 2 2 3 21/30 Flush 22/30 Write block 340 len 4096 data: 3 23/30 Read block 294 len 12288 24/30 Write block 329 len 12288 data: 6 6 6 25/30 Read block 538 len 16384 26/30 Write block 243 len 4096 data: 5 27/30 Write block 145 len 36864 data: 6 7 7 8 6 6 5 5 6 28/30 Write block 125 len 24576 data: 8 7 6 8 8 8 29/30 Write block 457 len 24576 data: 6 7 8 8 8 8 30/30 Read block 282 len 20480 ---------------------------------------------------------------- Crucible gen:78 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 1 Done Done Done false 1022 Acked Read 3 Done Done Done false 1023 Acked Write 3 Done Done Done false 1024 Acked Read 4 Done Done Done false 1025 Acked Write 1 Done Done Done false 1026 Acked Write 9 Done Done Done false 1027 Acked Write 6 Done Done Done false 1028 Acked Write 6 Done Done Done false 1029 Acked Read 5 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 1 2 Done 8 9 8 25 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1020 1020 1020 Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:36:02.884558716Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1494} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 77 77 77 146 146 146 F F F 1 020-039 76 76 76 145 145 145 F F F 2 040-059 73 73 73 137 137 137 F F F 3 060-079 78 78 78 147 147 147 F F F 4 080-099 68 68 68 131 131 131 F F F 5 100-119 76 76 76 144 144 144 F F F 6 120-139 78 78 78 148 148 148 T T T 7 140-159 78 78 78 148 148 148 T T T 8 160-179 78 78 78 148 148 148 F F F 9 180-199 78 78 78 148 148 148 F F F 10 200-219 74 74 74 139 139 139 F F F 11 220-239 76 76 76 144 144 144 F F F 12 240-259 74 74 74 140 140 140 T T T 13 260-279 78 78 78 148 148 148 F F F 14 280-299 64 64 64 125 125 125 F F F 15 300-319 77 77 77 146 146 146 F F F 16 320-339 74 74 74 140 140 140 T T T 17 340-359 78 78 78 148 148 148 T T T 18 360-379 73 73 73 137 137 137 F F F 19 380-399 73 73 73 137 137 137 F F F 20 400-419 74 74 74 138 138 138 F F F 21 420-439 74 74 74 138 138 138 F F F 22 440-459 76 76 76 143 143 143 T T T 23 460-479 76 76 76 143 143 143 T T T 24 480-499 69 69 69 132 132 132 F F F 25 500-519 73 73 73 137 137 137 F F F 26 520-539 74 74 74 140 140 140 F F F 27 540-559 74 74 74 140 140 140 F F F 28 560-579 69 69 69 132 132 132 F F F 29 580-599 73 73 73 137 137 137 F F F Max gen: 78, Max flush: 148 A Difference in extent metadata is expected here On loop 39 Verifying data now No change after verify Loop: 39 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 77 77 77 146 146 146 F F F 1 020-039 76 76 76 145 145 145 F F F 2 040-059 73 73 73 137 137 137 F F F 3 060-079 78 78 78 147 147 147 F F F 4 080-099 68 68 68 131 131 131 F F F 5 100-119 76 76 76 144 144 144 F F F 6 120-139 79 79 79 149 149 149 F F F 7 140-159 79 79 79 149 149 149 F F F 8 160-179 78 78 78 148 148 148 F F F 9 180-199 78 78 78 148 148 148 F F F 10 200-219 74 74 74 139 139 139 F F F 11 220-239 76 76 76 144 144 144 F F F 12 240-259 79 79 79 149 149 149 F F F 13 260-279 78 78 78 148 148 148 F F F 14 280-299 64 64 64 125 125 125 F F F 15 300-319 77 77 77 146 146 146 F F F 16 320-339 79 79 79 149 149 149 F F F 17 340-359 79 79 79 149 149 149 F F F 18 360-379 73 73 73 137 137 137 F F F 19 380-399 73 73 73 137 137 137 F F F 20 400-419 74 74 74 138 138 138 F F F 21 420-439 74 74 74 138 138 138 F F F 22 440-459 79 79 79 149 149 149 F F F 23 460-479 79 79 79 149 149 149 F F F 24 480-499 69 69 69 132 132 132 F F F 25 500-519 73 73 73 137 137 137 F F F 26 520-539 74 74 74 140 140 140 F F F 27 540-559 74 74 74 140 140 140 F F F 28 560-579 69 69 69 132 132 132 F F F 29 580-599 73 73 73 137 137 137 F F F Max gen: 79, Max flush: 149 Begin loop 40 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:07.166574754Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1502,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:07.211348466Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1502,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:07.212076006Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1502} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"26e7d501b33f47379e53601720353c2983e7654b\",\n git_commit_timestamp: \"2024-12-12T20:16:40.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:07.212112884Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1502} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:07.212127413Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1502} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:07.212142202Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1502} {"msg":"Crucible 8189eea1-c787-4b28-9920-ed14a7dc64e1 has session id: 07f22e8b-cccd-40f5-9839-4facd534002c","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:07.212554683Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1502,"session_id":"07f22e8b-cccd-40f5-9839-4facd534002c"} {"msg":"Upstairs opts: Upstairs UUID: 8189eea1-c787-4b28-9920-ed14a7dc64e1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:07.21260046Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1502,"session_id":"07f22e8b-cccd-40f5-9839-4facd534002c"} {"msg":"Crucible stats registered with UUID: 8189eea1-c787-4b28-9920-ed14a7dc64e1","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:07.212627328Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1502,"session_id":"07f22e8b-cccd-40f5-9839-4facd534002c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:07.243440203Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1502,"task":"crutest"} {"msg":"The guest has requested activation with gen:80","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:07.24348366Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1502} {"msg":"8189eea1-c787-4b28-9920-ed14a7dc64e1 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:07.243511718Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1502,"session_id":"07f22e8b-cccd-40f5-9839-4facd534002c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:07.243526957Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1502,"client":"0","":"downstairs","session_id":"07f22e8b-cccd-40f5-9839-4facd534002c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:07.243551946Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1502,"client":"0","":"downstairs","session_id":"07f22e8b-cccd-40f5-9839-4facd534002c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:07.243566345Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1502,"client":"1","":"downstairs","session_id":"07f22e8b-cccd-40f5-9839-4facd534002c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:07.243581394Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1502,"client":"1","":"downstairs","session_id":"07f22e8b-cccd-40f5-9839-4facd534002c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:07.243599063Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1502,"client":"2","":"downstairs","session_id":"07f22e8b-cccd-40f5-9839-4facd534002c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:07.243629681Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1502,"client":"2","":"downstairs","session_id":"07f22e8b-cccd-40f5-9839-4facd534002c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:07.243645109Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1502,"":"io task","client":"2","":"downstairs","session_id":"07f22e8b-cccd-40f5-9839-4facd534002c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:07.243659078Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1502,"":"io task","client":"0","":"downstairs","session_id":"07f22e8b-cccd-40f5-9839-4facd534002c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:07.243672718Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1502,"":"io task","client":"1","":"downstairs","session_id":"07f22e8b-cccd-40f5-9839-4facd534002c"} {"msg":"ds_connection connected from Ok(127.0.0.1:36901)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:07.243904842Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1502,"":"io task","client":"1","":"downstairs","session_id":"07f22e8b-cccd-40f5-9839-4facd534002c"} {"msg":"ds_connection connected from Ok(127.0.0.1:34095)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:07.244114487Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1502,"":"io task","client":"2","":"downstairs","session_id":"07f22e8b-cccd-40f5-9839-4facd534002c"} {"msg":"ds_connection connected from Ok(127.0.0.1:38857)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:07.244162264Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1502,"":"io task","client":"0","":"downstairs","session_id":"07f22e8b-cccd-40f5-9839-4facd534002c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:07.244690977Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1502,"client":"2","":"downstairs","session_id":"07f22e8b-cccd-40f5-9839-4facd534002c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:07.244757163Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1502,"client":"0","":"downstairs","session_id":"07f22e8b-cccd-40f5-9839-4facd534002c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:07.244777621Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1502,"client":"1","":"downstairs","session_id":"07f22e8b-cccd-40f5-9839-4facd534002c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:07.24537564Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1502,"client":"0","":"downstairs","session_id":"07f22e8b-cccd-40f5-9839-4facd534002c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:07.245405188Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1502,"client":"2","":"downstairs","session_id":"07f22e8b-cccd-40f5-9839-4facd534002c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:07.245422027Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1502,"client":"1","":"downstairs","session_id":"07f22e8b-cccd-40f5-9839-4facd534002c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:07.24595372Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1502,"session_id":"07f22e8b-cccd-40f5-9839-4facd534002c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:07.245976699Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1502,"session_id":"07f22e8b-cccd-40f5-9839-4facd534002c"} {"msg":"[0]R flush_numbers[0..12]: [146, 145, 137, 147, 131, 144, 149, 149, 148, 148, 139, 144]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:07.246050443Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1502,"":"downstairs","":"downstairs","session_id":"07f22e8b-cccd-40f5-9839-4facd534002c"} {"msg":"[0]R generation[0..12]: [77, 76, 73, 78, 68, 76, 79, 79, 78, 78, 74, 76]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:07.246069332Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1502,"":"downstairs","":"downstairs","session_id":"07f22e8b-cccd-40f5-9839-4facd534002c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:07.246083871Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1502,"":"downstairs","":"downstairs","session_id":"07f22e8b-cccd-40f5-9839-4facd534002c"} {"msg":"[1]R flush_numbers[0..12]: [146, 145, 137, 147, 131, 144, 149, 149, 148, 148, 139, 144]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:07.24609788Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1502,"":"downstairs","":"downstairs","session_id":"07f22e8b-cccd-40f5-9839-4facd534002c"} {"msg":"[1]R generation[0..12]: [77, 76, 73, 78, 68, 76, 79, 79, 78, 78, 74, 76]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:07.246111829Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1502,"":"downstairs","":"downstairs","session_id":"07f22e8b-cccd-40f5-9839-4facd534002c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:07.246126188Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1502,"":"downstairs","":"downstairs","session_id":"07f22e8b-cccd-40f5-9839-4facd534002c"} {"msg":"[2]R flush_numbers[0..12]: [146, 145, 137, 147, 131, 144, 149, 149, 148, 148, 139, 144]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:07.246139857Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1502,"":"downstairs","":"downstairs","session_id":"07f22e8b-cccd-40f5-9839-4facd534002c"} {"msg":"[2]R generation[0..12]: [77, 76, 73, 78, 68, 76, 79, 79, 78, 78, 74, 76]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:07.246153346Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1502,"":"downstairs","":"downstairs","session_id":"07f22e8b-cccd-40f5-9839-4facd534002c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:07.246167186Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1502,"":"downstairs","":"downstairs","session_id":"07f22e8b-cccd-40f5-9839-4facd534002c"} {"msg":"Max found gen is 80","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:07.246180285Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1502,"":"downstairs","":"downstairs","session_id":"07f22e8b-cccd-40f5-9839-4facd534002c"} {"msg":"Generation requested: 80 >= found:80","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:07.246193444Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1502,"":"downstairs","":"downstairs","session_id":"07f22e8b-cccd-40f5-9839-4facd534002c"} {"msg":"Next flush: 150","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:07.246206303Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1502,"":"downstairs","":"downstairs","session_id":"07f22e8b-cccd-40f5-9839-4facd534002c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:07.246222142Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","Wait for a query_work_queue command to finish before sending IO pid":1502,"":"downstairs","":"downstairs","session_id":"07f22e8b-cccd-40f5-9839-4facd534002c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:07.24624065Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1502,"session_id":"07f22e8b-cccd-40f5-9839-4facd534002c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:07.24625276Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1502,"client":"0","":"downstairs","session_id":"07f22e8b-cccd-40f5-9839-4facd534002c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:07.246267048Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1502,"client":"1","":"downstairs","session_id":"07f22e8b-cccd-40f5-9839-4facd534002c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:07.246280408Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1502,"client":"2","":"downstairs","session_id":"07f22e8b-cccd-40f5-9839-4facd534002c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:07.246293817Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1502,"session_id":"07f22e8b-cccd-40f5-9839-4facd534002c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:07.246306096Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1502,"session_id":"07f22e8b-cccd-40f5-9839-4facd534002c"} {"msg":"8189eea1-c787-4b28-9920-ed14a7dc64e1 is now active with session: 07f22e8b-cccd-40f5-9839-4facd534002c","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:07.246318935Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1502,"session_id":"07f22e8b-cccd-40f5-9839-4facd534002c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:07.246330874Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1502,"session_id":"07f22e8b-cccd-40f5-9839-4facd534002c"} {"msg":"The guest has finished waiting for activation with:80","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:07.246343213Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1502} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 62 len 36864 data: 5 6 5 5 5 4 5 5 6 02/30 Write block 595 len 16384 data: 4 3 3 4 03/30 Read block 92 len 8192 04/30 Write block 70 len 36864 data: 7 6 7 7 5 5 4 6 5 05/30 Write block 329 len 28672 data: 7 7 7 5 7 7 6 06/30 Read block 473 len 8192 07/30 Write block 484 len 24576 data: 7 8 8 6 5 6 08/30 Flush 09/30 Write block 106 len 12288 data: 7 10 9 10/30 Write block 164 len 4096 data: 6 11/30 Write block 365 len 24576 data: 8 9 8 5 6 5 12/30 Read block 231 len 40960 13/30 Write block 484 len 28672 data: 8 9 9 7 6 7 6 14/30 Read block 183 len 12288 15/30 Write block 503 len 24576 data: 6 6 5 5 7 6 16/30 Read block 526 len 16384 17/30 Write block 61 len 36864 data: 6 6 7 6 6 6 5 6 6 18/30 Read block 95 len 40960 19/30 Write block 138 len 4096 data: 6 20/30 Read block 443 len 20480 21/30 Write block 390 len 32768 data: 7 7 6 6 7 7 6 7 22/30 Write block 281 len 36864 data: 5 5 4 4 6 7 6 7 7 23/30 Write block 559 len 20480 data: 8 4 5 7 6 24/30 Read block 84 len 4096 25/30 Write block 80 len 4096 data: 5 26/30 Read block 259 len 40960 27/30 Write block 522 len 36864 data: 13 17 16 15 9 8 7 7 4 28/30 Read block 244 len 28672 29/30 Read block 450 len 36864 30/30 Write block 512 len 4096 data: 5 ---------------------------------------------------------------- Crucible gen:80 GIO:true work queues: Upstairs:0 downstairs:22 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1008 Acked Write 3 Done Done Done false 1009 Acked Write 1 Done Done Done false 1010 Acked Write 6 Done Done Done false 1011 Acked Read 10 Done Done Done false 1012 Acked Write 7 Done Done Done false 1013 Acked Read 3 Done Done Done false 1014 Acked Write 6 Done Done Done false 1015 Acked Read 4 Done Done Done false 1016 Acked Write 9 Done Done Done false 1017 Acked Read 10 Done Done Done false 1018 Acked Write 1 Done Done Done false 1019 Acked Read 5 Done Done Done false 1020 Acked Write 8 Done Done Done false 1021 Acked Write 9 Done Done Done false 1022 Acked Write 5 Done Done Done false 1023 Acked Read 1 Done Done Done false 1024 Acked Write 1 Done Done Done false 1025 Acked Read 10 Done Done Done false 1026 Acked Write 9 Done Done Done false 1027 Acked Read 7 Done Done Done false 1028 Acked Read 9 Done Done Done false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 21 21 21 63 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1007 1007 1007 Downstairs last five completed: 1007 1006 1005 1004 1003 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:36:07.266892306Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1502} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 77 77 77 146 146 146 F F F 1 020-039 76 76 76 145 145 145 F F F 2 040-059 73 73 73 137 137 137 F F F 3 060-079 80 80 80 150 150 150 T T T 4 080-099 68 68 68 131 131 131 T T T 5 100-119 76 76 76 144 144 144 T T T 6 120-139 79 79 79 149 149 149 T T T 7 140-159 79 79 79 149 149 149 F F F 8 160-179 78 78 78 148 148 148 T T T 9 180-199 78 78 78 148 148 148 F F F 10 200-219 74 74 74 139 139 139 F F F 11 220-239 76 76 76 144 144 144 F F F 12 240-259 79 79 79 149 149 149 F F F 13 260-279 78 78 78 148 148 148 F F F 14 280-299 64 64 64 125 125 125 T T T 15 300-319 77 77 77 146 146 146 F F F 16 320-339 80 80 80 150 150 150 F F F 17 340-359 79 79 79 149 149 149 F F F 18 360-379 73 73 73 137 137 137 T T T 19 380-399 73 73 73 137 137 137 T T T 20 400-419 74 74 74 138 138 138 F F F 21 420-439 74 74 74 138 138 138 F F F 22 440-459 79 79 79 149 149 149 F F F 23 460-479 79 79 79 149 149 149 F F F 24 480-499 80 80 80 150 150 150 T T T 25 500-519 73 73 73 137 137 137 T T T 26 520-539 74 74 74 140 140 140 T T T 27 540-559 74 74 74 140 140 140 T T T 28 560-579 69 69 69 132 132 132 T T T 29 580-599 80 80 80 150 150 150 F F F Max gen: 80, Max flush: 150 A Difference in extent metadata is expected here On loop 40 Verifying data now No change after verify Loop: 40 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 77 77 77 146 146 146 F F F 1 020-039 76 76 76 145 145 145 F F F 2 040-059 73 73 73 137 137 137 F F F 3 060-079 81 81 81 151 151 151 F F F 4 080-099 81 81 81 151 151 151 F F F 5 100-119 81 81 81 151 151 151 F F F 6 120-139 81 81 81 151 151 151 F F F 7 140-159 79 79 79 149 149 149 F F F 8 160-179 81 81 81 151 151 151 F F F 9 180-199 78 78 78 148 148 148 F F F 10 200-219 74 74 74 139 139 139 F F F 11 220-239 76 76 76 144 144 144 F F F 12 240-259 79 79 79 149 149 149 F F F 13 260-279 78 78 78 148 148 148 F F F 14 280-299 81 81 81 151 151 151 F F F 15 300-319 77 77 77 146 146 146 F F F 16 320-339 80 80 80 150 150 150 F F F 17 340-359 79 79 79 149 149 149 F F F 18 360-379 81 81 81 151 151 151 F F F 19 380-399 81 81 81 151 151 151 F F F 20 400-419 74 74 74 138 138 138 F F F 21 420-439 74 74 74 138 138 138 F F F 22 440-459 79 79 79 149 149 149 F F F 23 460-479 79 79 79 149 149 149 F F F 24 480-499 81 81 81 151 151 151 F F F 25 500-519 81 81 81 151 151 151 F F F 26 520-539 81 81 81 151 151 151 F F F 27 540-559 81 81 81 151 151 151 F F F 28 560-579 81 81 81 151 151 151 F F F 29 580-599 80 80 80 150 150 150 F F F Max gen: 81, Max flush: 151 Begin loop 41 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:11.58161412Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1510,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:36:11.625231803Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1510,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:11.625614736Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1510,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:11.657692524Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1510,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:11.658276114Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1510} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"26e7d501b33f47379e53601720353c2983e7654b\",\n git_commit_timestamp: \"2024-12-12T20:16:40.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:11.658309732Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1510} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:11.658323631Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1510} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:11.65833747Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1510} {"msg":"Crucible cbd8b06c-5256-4d92-8d64-467df2992633 has session id: e5e92b7a-0224-474a-b077-74ec861a2646","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:11.658791729Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1510,"session_id":"e5e92b7a-0224-474a-b077-74ec861a2646"} {"msg":"Upstairs opts: Upstairs UUID: cbd8b06c-5256-4d92-8d64-467df2992633, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:11.658836275Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1510,"session_id":"e5e92b7a-0224-474a-b077-74ec861a2646"} {"msg":"Crucible stats registered with UUID: cbd8b06c-5256-4d92-8d64-467df2992633","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:11.658858024Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1510,"session_id":"e5e92b7a-0224-474a-b077-74ec861a2646"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:11.689524589Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1510,"task":"crutest"} {"msg":"The guest has requested activation with gen:82","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:11.689556107Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1510} {"msg":"cbd8b06c-5256-4d92-8d64-467df2992633 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:11.689621983Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1510,"session_id":"e5e92b7a-0224-474a-b077-74ec861a2646"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:11.689644251Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1510,"client":"0","":"downstairs","session_id":"e5e92b7a-0224-474a-b077-74ec861a2646"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:11.689671759Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1510,"client":"0","":"downstairs","session_id":"e5e92b7a-0224-474a-b077-74ec861a2646"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:11.689689168Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1510,"client":"1","":"downstairs","session_id":"e5e92b7a-0224-474a-b077-74ec861a2646"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:11.689706937Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1510,"client":"1","":"downstairs","session_id":"e5e92b7a-0224-474a-b077-74ec861a2646"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:11.689723206Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1510,"client":"2","":"downstairs","session_id":"e5e92b7a-0224-474a-b077-74ec861a2646"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:11.689740475Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1510,"client":"2","":"downstairs","session_id":"e5e92b7a-0224-474a-b077-74ec861a2646"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:11.689757743Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1510,"":"io task","client":"2","":"downstairs","session_id":"e5e92b7a-0224-474a-b077-74ec861a2646"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:11.689772872Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1510,"":"io task","client":"0","":"downstairs","session_id":"e5e92b7a-0224-474a-b077-74ec861a2646"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:11.689786251Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1510,"":"io task","client":"1","":"downstairs","session_id":"e5e92b7a-0224-474a-b077-74ec861a2646"} {"msg":"ds_connection connected from Ok(127.0.0.1:36870)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:11.689876205Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1510,"":"io task","client":"2","":"downstairs","session_id":"e5e92b7a-0224-474a-b077-74ec861a2646"} {"msg":"ds_connection connected from Ok(127.0.0.1:58182)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:11.68994449Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1510,"":"io task","client":"0","":"downstairs","session_id":"e5e92b7a-0224-474a-b077-74ec861a2646"} {"msg":"ds_connection connected from Ok(127.0.0.1:48934)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:11.690057293Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1510,"":"io task","client":"1","":"downstairs","session_id":"e5e92b7a-0224-474a-b077-74ec861a2646"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:11.690771733Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1510,"client":"2","":"downstairs","session_id":"e5e92b7a-0224-474a-b077-74ec861a2646"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:11.690800391Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1510,"client":"0","":"downstairs","session_id":"e5e92b7a-0224-474a-b077-74ec861a2646"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:11.69081525Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1510,"client":"1","":"downstairs","session_id":"e5e92b7a-0224-474a-b077-74ec861a2646"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:11.69140579Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1510,"client":"2","":"downstairs","session_id":"e5e92b7a-0224-474a-b077-74ec861a2646"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:11.69153981Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1510,"client":"1","":"downstairs","session_id":"e5e92b7a-0224-474a-b077-74ec861a2646"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:11.691655752Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1510,"client":"0","":"downstairs","session_id":"e5e92b7a-0224-474a-b077-74ec861a2646"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:11.691885536Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1510,"session_id":"e5e92b7a-0224-474a-b077-74ec861a2646"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:11.691996329Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1510,"session_id":"e5e92b7a-0224-474a-b077-74ec861a2646"} {"msg":"[0]R flush_numbers[0..12]: [146, 145, 137, 151, 151, 151, 151, 149, 151, 148, 139, 144]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:11.692257601Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1510,"":"downstairs","":"downstairs","session_id":"e5e92b7a-0224-474a-b077-74ec861a2646"} {"msg":"[0]R generation[0..12]: [77, 76, 73, 81, 81, 81, 81, 79, 81, 78, 74, 76]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:11.692277489Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1510,"":"downstairs","":"downstairs","session_id":"e5e92b7a-0224-474a-b077-74ec861a2646"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:11.692291879Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1510,"":"downstairs","":"downstairs","session_id":"e5e92b7a-0224-474a-b077-74ec861a2646"} {"msg":"[1]R flush_numbers[0..12]: [146, 145, 137, 151, 151, 151, 151, 149, 151, 148, 139, 144]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:11.692309277Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1510,"":"downstairs","":"downstairs","session_id":"e5e92b7a-0224-474a-b077-74ec861a2646"} {"msg":"[1]R generation[0..12]: [77, 76, 73, 81, 81, 81, 81, 79, 81, 78, 74, 76]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:11.692323236Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1510,"":"downstairs","":"downstairs","session_id":"e5e92b7a-0224-474a-b077-74ec861a2646"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:11.692337385Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1510,"":"downstairs","":"downstairs","session_id":"e5e92b7a-0224-474a-b077-74ec861a2646"} {"msg":"[2]R flush_numbers[0..12]: [146, 145, 137, 151, 151, 151, 151, 149, 151, 148, 139, 144]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:11.692351035Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1510,"":"downstairs","":"downstairs","session_id":"e5e92b7a-0224-474a-b077-74ec861a2646"} {"msg":"[2]R generation[0..12]: [77, 76, 73, 81, 81, 81, 81, 79, 81, 78, 74, 76]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:11.692364563Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1510,"":"downstairs","":"downstairs","session_id":"e5e92b7a-0224-474a-b077-74ec861a2646"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:11.692378582Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1510,"":"downstairs","":"downstairs","session_id":"e5e92b7a-0224-474a-b077-74ec861a2646"} {"msg":"Max found gen is 82","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:11.692391652Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1510,"":"downstairs","":"downstairs","session_id":"e5e92b7a-0224-474a-b077-74ec861a2646"} {"msg":"Generation requested: 82 >= found:82","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:11.692404731Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1510,"":"downstairs","":"downstairs","session_id":"e5e92b7a-0224-474a-b077-74ec861a2646"} {"msg":"Next flush: 152","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:11.69241744Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1510,"":"downstairs","":"downstairs","session_id":"e5e92b7a-0224-474a-b077-74ec861a2646"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:11.692430519Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1510,"":"downstairs","":"downstairs","session_id":"e5e92b7a-0224-474a-b077-74ec861a2646"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:11.692443888Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1510,"session_id":"e5e92b7a-0224-474a-b077-74ec861a2646"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:11.692455917Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","Wait for a query_work_queue command to finish before sending IO pid":1510,"client":"0","":"downstairs","session_id":"e5e92b7a-0224-474a-b077-74ec861a2646"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:11.692476166Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1510,"client":"1","":"downstairs","session_id":"e5e92b7a-0224-474a-b077-74ec861a2646"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:11.692489695Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1510,"client":"2","":"downstairs","session_id":"e5e92b7a-0224-474a-b077-74ec861a2646"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:11.692503054Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1510,"session_id":"e5e92b7a-0224-474a-b077-74ec861a2646"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:11.692517173Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1510,"session_id":"e5e92b7a-0224-474a-b077-74ec861a2646"} {"msg":"cbd8b06c-5256-4d92-8d64-467df2992633 is now active with session: e5e92b7a-0224-474a-b077-74ec861a2646","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:11.692530072Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1510,"session_id":"e5e92b7a-0224-474a-b077-74ec861a2646"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:11.692541861Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1510,"session_id":"e5e92b7a-0224-474a-b077-74ec861a2646"} {"msg":"The guest has finished waiting for activation with:82","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:11.692553881Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1510} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 277 len 36864 data: 5 6 6 5 6 6 5 5 7 02/30 Flush 03/30 Write block 88 len 4096 data: 11 04/30 Read block 534 len 32768 05/30 Read block 132 len 4096 06/30 Read block 259 len 4096 07/30 Write block 185 len 40960 data: 10 8 8 9 9 9 9 9 9 7 08/30 Write block 431 len 4096 data: 7 09/30 Read block 470 len 20480 10/30 Write block 204 len 12288 data: 9 8 9 11/30 Read block 583 len 8192 12/30 Read block 453 len 40960 13/30 Write block 388 len 24576 data: 6 6 8 8 7 7 14/30 Read block 202 len 8192 15/30 Read block 220 len 32768 16/30 Write block 479 len 28672 data: 5 4 5 4 5 9 10 17/30 Write block 427 len 40960 data: 6 6 7 5 8 7 7 5 6 5 18/30 Read block 292 len 16384 19/30 Read block 505 len 8192 20/30 Write block 69 len 20480 data: 7 8 7 8 8 21/30 Flush 22/30 Write block 276 len 24576 data: 5 6 7 7 6 7 23/30 Write block 446 len 36864 data: 8 9 7 9 9 10 10 9 9 24/30 Read block 226 len 20480 25/30 Write block 164 len 12288 data: 7 5 5 26/30 Read block 334 len 32768 27/30 Read block 110 len 4096 28/30 Read block 160 len 4096 29/30 Write block 74 len 32768 data: 6 6 5 7 6 5 6 6 30/30 Read block 147 len 32768 ---------------------------------------------------------------- Crucible gen:82 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 6 Done Done Done false 1022 Acked Write 9 Done Done Done false 1023 Acked Read 5 Done Done Done false 1024 Acked Write 3 Done Done Done false 1025 Acked Read 8 Done Done Done false 1026 Acked Read 1 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Write 8 Sent Done Done false 1029 Acked Read 8 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 1 0 3 Done 7 8 9 24 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1020 1020 1020 Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:36:11.712687482Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1510} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 77 77 77 146 146 146 F F F 1 020-039 76 76 76 145 145 145 F F F 2 040-059 73 73 73 137 137 137 F F F 3 060-079 82 82 82 153 153 153 T T T 4 080-099 82 82 82 153 153 153 T T T 5 100-119 81 81 81 151 151 151 F F F 6 120-139 81 81 81 151 151 151 F F F 7 140-159 79 79 79 149 149 149 F F F 8 160-179 81 81 81 151 151 151 T T T 9 180-199 82 82 82 153 153 153 F F F 10 200-219 82 82 82 153 153 153 F F F 11 220-239 76 76 76 144 144 144 F F F 12 240-259 79 79 79 149 149 149 F F F 13 260-279 82 82 82 152 152 152 T T T 14 280-299 82 82 82 152 152 152 T T T 15 300-319 77 77 77 146 146 146 F F F 16 320-339 80 80 80 150 150 150 F F F 17 340-359 79 79 79 149 149 149 F F F 18 360-379 81 81 81 151 151 151 F F F 19 380-399 82 82 82 153 153 153 F F F 20 400-419 74 74 74 138 138 138 F F F 21 420-439 82 82 82 153 153 153 F F F 22 440-459 79 79 79 149 149 149 T T T 23 460-479 82 82 82 153 153 153 F F F 24 480-499 82 82 82 153 153 153 F F F 25 500-519 81 81 81 151 151 151 F F F 26 520-539 81 81 81 151 151 151 F F F 27 540-559 81 81 81 151 151 151 F F F 28 560-579 81 81 81 151 151 151 F F F 29 580-599 80 80 80 150 150 150 F F F Max gen: 82, Max flush: 153 A Difference in extent metadata is expected here On loop 41 Verifying data now No change after verify Loop: 41 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 77 77 77 146 146 146 F F F 1 020-039 76 76 76 145 145 145 F F F 2 040-059 73 73 73 137 137 137 F F F 3 060-079 83 83 83 154 154 154 F F F 4 080-099 83 83 83 154 154 154 F F F 5 100-119 81 81 81 151 151 151 F F F 6 120-139 81 81 81 151 151 151 F F F 7 140-159 79 79 79 149 149 149 F F F 8 160-179 83 83 83 154 154 154 F F F 9 180-199 82 82 82 153 153 153 F F F 10 200-219 82 82 82 153 153 153 F F F 11 220-239 76 76 76 144 144 144 F F F 12 240-259 79 79 79 149 149 149 F F F 13 260-279 83 83 83 154 154 154 F F F 14 280-299 83 83 83 154 154 154 F F F 15 300-319 77 77 77 146 146 146 F F F 16 320-339 80 80 80 150 150 150 F F F 17 340-359 79 79 79 149 149 149 F F F 18 360-379 81 81 81 151 151 151 F F F 19 380-399 82 82 82 153 153 153 F F F 20 400-419 74 74 74 138 138 138 F F F 21 420-439 82 82 82 153 153 153 F F F 22 440-459 83 83 83 154 154 154 F F F 23 460-479 82 82 82 153 153 153 F F F 24 480-499 82 82 82 153 153 153 F F F 25 500-519 81 81 81 151 151 151 F F F 26 520-539 81 81 81 151 151 151 F F F 27 540-559 81 81 81 151 151 151 F F F 28 560-579 81 81 81 151 151 151 F F F 29 580-599 80 80 80 150 150 150 F F F Max gen: 83, Max flush: 154 Begin loop 42 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:16.022377803Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1518,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:36:16.065267465Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1518,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:16.065673157Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1518,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:16.097168496Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1518,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:16.097735327Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1518} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"26e7d501b33f47379e53601720353c2983e7654b\",\n git_commit_timestamp: \"2024-12-12T20:16:40.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:16.097771284Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1518} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:16.097785423Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1518} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:16.097799652Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1518} {"msg":"Crucible ba45c17d-0b8c-4caf-af53-57b15caab670 has session id: 2b7dfb18-3f2e-4d2e-a3f0-ae076d77a821","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:16.098296878Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1518,"session_id":"2b7dfb18-3f2e-4d2e-a3f0-ae076d77a821"} {"msg":"Upstairs opts: Upstairs UUID: ba45c17d-0b8c-4caf-af53-57b15caab670, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:16.098343355Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1518,"session_id":"2b7dfb18-3f2e-4d2e-a3f0-ae076d77a821"} {"msg":"Crucible stats registered with UUID: ba45c17d-0b8c-4caf-af53-57b15caab670","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:16.098369113Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1518,"session_id":"2b7dfb18-3f2e-4d2e-a3f0-ae076d77a821"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:16.128896638Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1518,"task":"crutest"} {"msg":"The guest has requested activation with gen:84","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:16.128924306Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1518} {"msg":"ba45c17d-0b8c-4caf-af53-57b15caab670 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:16.129020969Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1518,"session_id":"2b7dfb18-3f2e-4d2e-a3f0-ae076d77a821"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:16.129040768Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1518,"client":"0","":"downstairs","session_id":"2b7dfb18-3f2e-4d2e-a3f0-ae076d77a821"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:16.129063546Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1518,"client":"0","":"downstairs","session_id":"2b7dfb18-3f2e-4d2e-a3f0-ae076d77a821"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:16.129078855Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1518,"client":"1","":"downstairs","session_id":"2b7dfb18-3f2e-4d2e-a3f0-ae076d77a821"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:16.129097144Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1518,"client":"1","":"downstairs","session_id":"2b7dfb18-3f2e-4d2e-a3f0-ae076d77a821"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:16.129110713Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1518,"client":"2","":"downstairs","session_id":"2b7dfb18-3f2e-4d2e-a3f0-ae076d77a821"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:16.129124812Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1518,"client":"2","":"downstairs","session_id":"2b7dfb18-3f2e-4d2e-a3f0-ae076d77a821"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:16.129137881Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1518,"":"io task","client":"2","":"downstairs","session_id":"2b7dfb18-3f2e-4d2e-a3f0-ae076d77a821"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:16.12915145Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1518,"":"io task","client":"0","":"downstairs","session_id":"2b7dfb18-3f2e-4d2e-a3f0-ae076d77a821"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:16.129171179Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1518,"":"io task","client":"1","":"downstairs","session_id":"2b7dfb18-3f2e-4d2e-a3f0-ae076d77a821"} {"msg":"ds_connection connected from Ok(127.0.0.1:34005)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:16.129267892Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1518,"":"io task","client":"2","":"downstairs","session_id":"2b7dfb18-3f2e-4d2e-a3f0-ae076d77a821"} {"msg":"ds_connection connected from Ok(127.0.0.1:42285)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:16.129308549Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1518,"":"io task","client":"0","":"downstairs","session_id":"2b7dfb18-3f2e-4d2e-a3f0-ae076d77a821"} {"msg":"ds_connection connected from Ok(127.0.0.1:57932)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:16.129341557Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1518,"":"io task","client":"1","":"downstairs","session_id":"2b7dfb18-3f2e-4d2e-a3f0-ae076d77a821"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:16.130096455Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1518,"client":"1","":"downstairs","session_id":"2b7dfb18-3f2e-4d2e-a3f0-ae076d77a821"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:16.130125963Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1518,"client":"2","":"downstairs","session_id":"2b7dfb18-3f2e-4d2e-a3f0-ae076d77a821"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:16.130159151Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1518,"client":"0","":"downstairs","session_id":"2b7dfb18-3f2e-4d2e-a3f0-ae076d77a821"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:16.130672095Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1518,"client":"1","":"downstairs","session_id":"2b7dfb18-3f2e-4d2e-a3f0-ae076d77a821"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:16.130701904Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1518,"client":"0","":"downstairs","session_id":"2b7dfb18-3f2e-4d2e-a3f0-ae076d77a821"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:16.130941127Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1518,"client":"2","":"downstairs","session_id":"2b7dfb18-3f2e-4d2e-a3f0-ae076d77a821"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:16.131282333Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1518,"session_id":"2b7dfb18-3f2e-4d2e-a3f0-ae076d77a821"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:16.131320891Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1518,"session_id":"2b7dfb18-3f2e-4d2e-a3f0-ae076d77a821"} {"msg":"[0]R flush_numbers[0..12]: [146, 145, 137, 154, 154, 151, 151, 149, 154, 153, 153, 144]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:16.131377547Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1518,"":"downstairs","":"downstairs","session_id":"2b7dfb18-3f2e-4d2e-a3f0-ae076d77a821"} {"msg":"[0]R generation[0..12]: [77, 76, 73, 83, 83, 81, 81, 79, 83, 82, 82, 76]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:16.131405015Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1518,"":"downstairs","":"downstairs","session_id":"2b7dfb18-3f2e-4d2e-a3f0-ae076d77a821"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:16.131420354Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1518,"":"downstairs","":"downstairs","session_id":"2b7dfb18-3f2e-4d2e-a3f0-ae076d77a821"} {"msg":"[1]R flush_numbers[0..12]: [146, 145, 137, 154, 154, 151, 151, 149, 154, 153, 153, 144]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:16.131434743Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1518,"":"downstairs","":"downstairs","session_id":"2b7dfb18-3f2e-4d2e-a3f0-ae076d77a821"} {"msg":"[1]R generation[0..12]: [77, 76, 73, 83, 83, 81, 81, 79, 83, 82, 82, 76]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:16.131448382Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1518,"":"downstairs","":"downstairs","session_id":"2b7dfb18-3f2e-4d2e-a3f0-ae076d77a821"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:16.131462111Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1518,"":"downstairs","":"downstairs","session_id":"2b7dfb18-3f2e-4d2e-a3f0-ae076d77a821"} {"msg":"[2]R flush_numbers[0..12]: [146, 145, 137, 154, 154, 151, 151, 149, 154, 153, 153, 144]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:16.13147576Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1518,"":"downstairs","":"downstairs","session_id":"2b7dfb18-3f2e-4d2e-a3f0-ae076d77a821"} {"msg":"[2]R generation[0..12]: [77, 76, 73, 83, 83, 81, 81, 79, 83, 82, 82, 76]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:16.131489209Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1518,"":"downstairs","":"downstairs","session_id":"2b7dfb18-3f2e-4d2e-a3f0-ae076d77a821"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:16.131503988Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1518,"":"downstairs","":"downstairs","session_id":"2b7dfb18-3f2e-4d2e-a3f0-ae076d77a821"} {"msg":"Max found gen is 84","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:16.131517057Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1518,"":"downstairs","":"downstairs","session_id":"2b7dfb18-3f2e-4d2e-a3f0-ae076d77a821"} {"msg":"Generation requested: 84 >= found:84","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:16.131530446Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1518,"":"downstairs","":"downstairs","session_id":"2b7dfb18-3f2e-4d2e-a3f0-ae076d77a821"} {"msg":"Next flush: 155","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:16.131546555Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1518,"":"downstairs","":"downstairs","session_id":"2b7dfb18-3f2e-4d2e-a3f0-ae076d77a821"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:16.131559894Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1518,"":"downstairs","":"downstairs","session_id":"2b7dfb18-3f2e-4d2e-a3f0-ae076d77a821"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:16.131573393Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1518,"session_id":"2b7dfb18-3f2e-4d2e-a3f0-ae076d77a821"} Wait for a query_work_queue command to finish before sending IO {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:16.131593742Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1518,"client":"0","":"downstairs","session_id":"2b7dfb18-3f2e-4d2e-a3f0-ae076d77a821"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:16.13161987Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1518,"client":"1","":"downstairs","session_id":"2b7dfb18-3f2e-4d2e-a3f0-ae076d77a821"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:16.131635119Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1518,"client":"2","":"downstairs","session_id":"2b7dfb18-3f2e-4d2e-a3f0-ae076d77a821"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:16.131648388Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1518,"session_id":"2b7dfb18-3f2e-4d2e-a3f0-ae076d77a821"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:16.131660467Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1518,"session_id":"2b7dfb18-3f2e-4d2e-a3f0-ae076d77a821"} {"msg":"ba45c17d-0b8c-4caf-af53-57b15caab670 is now active with session: 2b7dfb18-3f2e-4d2e-a3f0-ae076d77a821","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:16.131673137Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1518,"session_id":"2b7dfb18-3f2e-4d2e-a3f0-ae076d77a821"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:16.131684936Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1518,"session_id":"2b7dfb18-3f2e-4d2e-a3f0-ae076d77a821"} {"msg":"The guest has finished waiting for activation with:84","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:16.131706524Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1518} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 152 len 28672 data: 6 7 6 6 7 7 7 02/30 Read block 32 len 32768 03/30 Flush 04/30 Write block 35 len 16384 data: 10 9 9 11 05/30 Read block 257 len 24576 06/30 Read block 515 len 4096 07/30 Read block 466 len 8192 08/30 Write block 337 len 28672 data: 5 6 5 4 3 4 4 09/30 Flush 10/30 Write block 95 len 40960 data: 6 4 3 2 4 6 7 9 9 9 11/30 Read block 147 len 4096 12/30 Read block 559 len 16384 13/30 Flush 14/30 Write block 343 len 28672 data: 5 7 8 10 11 10 9 15/30 Read block 411 len 32768 16/30 Read block 99 len 24576 17/30 Read block 293 len 40960 18/30 Write block 401 len 28672 data: 9 10 8 7 9 10 10 19/30 Read block 238 len 8192 20/30 Read block 278 len 24576 21/30 Read block 397 len 12288 22/30 Read block 160 len 20480 23/30 Flush 24/30 Write block 210 len 4096 data: 8 25/30 Flush 26/30 Flush 27/30 Write block 435 len 28672 data: 7 6 5 5 5 6 6 28/30 Write block 77 len 20480 data: 8 7 6 7 7 29/30 Read block 62 len 20480 30/30 Write block 163 len 16384 data: 7 8 6 6 ---------------------------------------------------------------- Crucible gen:84 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 7 Done Done Done false 1027 Acked Write 5 Done Done Done false 1028 Acked Read 5 Done Done Done false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:36:16.15164148Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1518} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 77 77 77 146 146 146 F F F 1 020-039 84 84 84 156 156 156 F F F 2 040-059 73 73 73 137 137 137 F F F 3 060-079 83 83 83 154 154 154 T T T 4 080-099 84 84 84 157 157 157 T T T 5 100-119 84 84 84 157 157 157 F F F 6 120-139 81 81 81 151 151 151 F F F 7 140-159 84 84 84 155 155 155 F F F 8 160-179 83 83 83 154 154 154 T T T 9 180-199 82 82 82 153 153 153 F F F 10 200-219 84 84 84 159 159 159 F F F 11 220-239 76 76 76 144 144 144 F F F 12 240-259 79 79 79 149 149 149 F F F 13 260-279 83 83 83 154 154 154 F F F 14 280-299 83 83 83 154 154 154 F F F 15 300-319 77 77 77 146 146 146 F F F 16 320-339 84 84 84 156 156 156 F F F 17 340-359 84 84 84 158 158 158 F F F 18 360-379 81 81 81 151 151 151 F F F 19 380-399 82 82 82 153 153 153 F F F 20 400-419 84 84 84 158 158 158 F F F 21 420-439 82 82 82 153 153 153 T T T 22 440-459 83 83 83 154 154 154 T T T 23 460-479 82 82 82 153 153 153 F F F 24 480-499 82 82 82 153 153 153 F F F 25 500-519 81 81 81 151 151 151 F F F 26 520-539 81 81 81 151 151 151 F F F 27 540-559 81 81 81 151 151 151 F F F 28 560-579 81 81 81 151 151 151 F F F 29 580-599 80 80 80 150 150 150 F F F Max gen: 84, Max flush: 159 A Difference in extent metadata is expected here On loop 42 Verifying data now No change after verify Loop: 42 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 77 77 77 146 146 146 F F F 1 020-039 84 84 84 156 156 156 F F F 2 040-059 73 73 73 137 137 137 F F F 3 060-079 85 85 85 160 160 160 F F F 4 080-099 85 85 85 160 160 160 F F F 5 100-119 84 84 84 157 157 157 F F F 6 120-139 81 81 81 151 151 151 F F F 7 140-159 84 84 84 155 155 155 F F F 8 160-179 85 85 85 160 160 160 F F F 9 180-199 82 82 82 153 153 153 F F F 10 200-219 84 84 84 159 159 159 F F F 11 220-239 76 76 76 144 144 144 F F F 12 240-259 79 79 79 149 149 149 F F F 13 260-279 83 83 83 154 154 154 F F F 14 280-299 83 83 83 154 154 154 F F F 15 300-319 77 77 77 146 146 146 F F F 16 320-339 84 84 84 156 156 156 F F F 17 340-359 84 84 84 158 158 158 F F F 18 360-379 81 81 81 151 151 151 F F F 19 380-399 82 82 82 153 153 153 F F F 20 400-419 84 84 84 158 158 158 F F F 21 420-439 85 85 85 160 160 160 F F F 22 440-459 85 85 85 160 160 160 F F F 23 460-479 82 82 82 153 153 153 F F F 24 480-499 82 82 82 153 153 153 F F F 25 500-519 81 81 81 151 151 151 F F F 26 520-539 81 81 81 151 151 151 F F F 27 540-559 81 81 81 151 151 151 F F F 28 560-579 81 81 81 151 151 151 F F F 29 580-599 80 80 80 150 150 150 F F F Max gen: 85, Max flush: 160 Begin loop 43 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:20.421003351Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1526,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:36:20.464158035Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1526,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:20.464559128Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1526,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:20.496183277Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1526,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:20.496772506Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1526} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"26e7d501b33f47379e53601720353c2983e7654b\",\n git_commit_timestamp: \"2024-12-12T20:16:40.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:20.496807804Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1526} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:20.496821653Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1526} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:20.496835932Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1526} {"msg":"Crucible 335d6219-e3ab-4518-8ce8-7706a9b9c097 has session id: d63fafa2-fb8e-4268-be20-ac948ad7bd4c","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:20.49730372Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1526,"session_id":"d63fafa2-fb8e-4268-be20-ac948ad7bd4c"} {"msg":"Upstairs opts: Upstairs UUID: 335d6219-e3ab-4518-8ce8-7706a9b9c097, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:20.497351627Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1526,"session_id":"d63fafa2-fb8e-4268-be20-ac948ad7bd4c"} {"msg":"Crucible stats registered with UUID: 335d6219-e3ab-4518-8ce8-7706a9b9c097","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:20.497373255Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1526,"session_id":"d63fafa2-fb8e-4268-be20-ac948ad7bd4c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:20.528680526Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1526,"task":"crutest"} {"msg":"The guest has requested activation with gen:86","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:20.528711394Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1526} {"msg":"335d6219-e3ab-4518-8ce8-7706a9b9c097 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:20.528788639Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1526,"session_id":"d63fafa2-fb8e-4268-be20-ac948ad7bd4c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:20.528809997Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1526,"client":"0","":"downstairs","session_id":"d63fafa2-fb8e-4268-be20-ac948ad7bd4c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:20.528838126Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1526,"client":"0","":"downstairs","session_id":"d63fafa2-fb8e-4268-be20-ac948ad7bd4c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:20.528855874Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1526,"client":"1","":"downstairs","session_id":"d63fafa2-fb8e-4268-be20-ac948ad7bd4c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:20.528874083Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1526,"client":"1","":"downstairs","session_id":"d63fafa2-fb8e-4268-be20-ac948ad7bd4c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:20.528890352Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1526,"client":"2","":"downstairs","session_id":"d63fafa2-fb8e-4268-be20-ac948ad7bd4c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:20.52891425Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1526,"client":"2","":"downstairs","session_id":"d63fafa2-fb8e-4268-be20-ac948ad7bd4c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:20.528943778Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1526,"":"io task","client":"2","":"downstairs","session_id":"d63fafa2-fb8e-4268-be20-ac948ad7bd4c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:20.528975046Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1526,"":"io task","client":"0","":"downstairs","session_id":"d63fafa2-fb8e-4268-be20-ac948ad7bd4c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:20.529006684Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1526,"":"io task","client":"1","":"downstairs","session_id":"d63fafa2-fb8e-4268-be20-ac948ad7bd4c"} {"msg":"ds_connection connected from Ok(127.0.0.1:42007)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:20.529040392Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1526,"":"io task","client":"2","":"downstairs","session_id":"d63fafa2-fb8e-4268-be20-ac948ad7bd4c"} {"msg":"ds_connection connected from Ok(127.0.0.1:45020)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:20.529071989Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1526,"":"io task","client":"0","":"downstairs","session_id":"d63fafa2-fb8e-4268-be20-ac948ad7bd4c"} {"msg":"ds_connection connected from Ok(127.0.0.1:48969)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:20.529275055Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1526,"":"io task","client":"1","":"downstairs","session_id":"d63fafa2-fb8e-4268-be20-ac948ad7bd4c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:20.529703776Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1526,"client":"2","":"downstairs","session_id":"d63fafa2-fb8e-4268-be20-ac948ad7bd4c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:20.529831487Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1526,"client":"1","":"downstairs","session_id":"d63fafa2-fb8e-4268-be20-ac948ad7bd4c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:20.530014634Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1526,"client":"0","":"downstairs","session_id":"d63fafa2-fb8e-4268-be20-ac948ad7bd4c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:20.530343462Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1526,"client":"2","":"downstairs","session_id":"d63fafa2-fb8e-4268-be20-ac948ad7bd4c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:20.530524779Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1526,"client":"1","":"downstairs","session_id":"d63fafa2-fb8e-4268-be20-ac948ad7bd4c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:20.530827958Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1526,"client":"0","":"downstairs","session_id":"d63fafa2-fb8e-4268-be20-ac948ad7bd4c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:20.530989457Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1526,"session_id":"d63fafa2-fb8e-4268-be20-ac948ad7bd4c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:20.53109042Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1526,"session_id":"d63fafa2-fb8e-4268-be20-ac948ad7bd4c"} {"msg":"[0]R flush_numbers[0..12]: [146, 156, 137, 160, 160, 157, 151, 155, 160, 153, 159, 144]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:20.531342063Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1526,"":"downstairs","":"downstairs","session_id":"d63fafa2-fb8e-4268-be20-ac948ad7bd4c"} {"msg":"[0]R generation[0..12]: [77, 84, 73, 85, 85, 84, 81, 84, 85, 82, 84, 76]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:20.531361112Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1526,"":"downstairs","":"downstairs","session_id":"d63fafa2-fb8e-4268-be20-ac948ad7bd4c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:20.531375611Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1526,"":"downstairs","":"downstairs","session_id":"d63fafa2-fb8e-4268-be20-ac948ad7bd4c"} {"msg":"[1]R flush_numbers[0..12]: [146, 156, 137, 160, 160, 157, 151, 155, 160, 153, 159, 144]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:20.53138945Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1526,"":"downstairs","":"downstairs","session_id":"d63fafa2-fb8e-4268-be20-ac948ad7bd4c"} {"msg":"[1]R generation[0..12]: [77, 84, 73, 85, 85, 84, 81, 84, 85, 82, 84, 76]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:20.531403138Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1526,"":"downstairs","":"downstairs","session_id":"d63fafa2-fb8e-4268-be20-ac948ad7bd4c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:20.531416908Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1526,"":"downstairs","":"downstairs","session_id":"d63fafa2-fb8e-4268-be20-ac948ad7bd4c"} {"msg":"[2]R flush_numbers[0..12]: [146, 156, 137, 160, 160, 157, 151, 155, 160, 153, 159, 144]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:20.531433636Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1526,"":"downstairs","":"downstairs","session_id":"d63fafa2-fb8e-4268-be20-ac948ad7bd4c"} {"msg":"[2]R generation[0..12]: [77, 84, 73, 85, 85, 84, 81, 84, 85, 82, 84, 76]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:20.531447495Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1526,"":"downstairs","":"downstairs","session_id":"d63fafa2-fb8e-4268-be20-ac948ad7bd4c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:20.531461474Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1526,"":"downstairs","":"downstairs","session_id":"d63fafa2-fb8e-4268-be20-ac948ad7bd4c"} {"msg":"Max found gen is 86","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:20.531474394Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1526,"":"downstairs","":"downstairs","session_id":"d63fafa2-fb8e-4268-be20-ac948ad7bd4c"} {"msg":"Generation requested: 86 >= found:86","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:20.531487373Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1526,"":"downstairs","":"downstairs","session_id":"d63fafa2-fb8e-4268-be20-ac948ad7bd4c"} {"msg":"Next flush: 161","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:20.531500032Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1526,"":"downstairs","":"downstairs","session_id":"d63fafa2-fb8e-4268-be20-ac948ad7bd4c"} {"msg":"All extents match","v":0,"name":"crucible"Wait for a query_work_queue command to finish before sending IO ,"level":30,"time":"2024-12-12T20:36:20.531517531Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1526,"":"downstairs","":"downstairs","session_id":"d63fafa2-fb8e-4268-be20-ac948ad7bd4c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:20.53153084Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1526,"session_id":"d63fafa2-fb8e-4268-be20-ac948ad7bd4c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:20.531542739Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1526,"client":"0","":"downstairs","session_id":"d63fafa2-fb8e-4268-be20-ac948ad7bd4c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:20.531555748Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1526,"client":"1","":"downstairs","session_id":"d63fafa2-fb8e-4268-be20-ac948ad7bd4c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:20.531568797Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1526,"client":"2","":"downstairs","session_id":"d63fafa2-fb8e-4268-be20-ac948ad7bd4c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:20.531582286Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1526,"session_id":"d63fafa2-fb8e-4268-be20-ac948ad7bd4c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:20.531594635Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1526,"session_id":"d63fafa2-fb8e-4268-be20-ac948ad7bd4c"} {"msg":"335d6219-e3ab-4518-8ce8-7706a9b9c097 is now active with session: d63fafa2-fb8e-4268-be20-ac948ad7bd4c","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:20.531607204Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1526,"session_id":"d63fafa2-fb8e-4268-be20-ac948ad7bd4c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:20.531619184Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1526,"session_id":"d63fafa2-fb8e-4268-be20-ac948ad7bd4c"} {"msg":"The guest has finished waiting for activation with:86","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:20.531631283Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1526} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 242 len 4096 data: 5 02/30 Read block 142 len 32768 03/30 Read block 491 len 40960 04/30 Read block 517 len 16384 05/30 Write block 447 len 12288 data: 10 8 10 06/30 Write block 175 len 24576 data: 3 3 4 4 7 5 07/30 Flush 08/30 Write block 579 len 40960 data: 9 7 8 9 9 10 9 9 7 6 09/30 Read block 30 len 4096 10/30 Write block 469 len 36864 data: 5 5 5 4 5 5 6 6 5 11/30 Write block 587 len 32768 data: 8 7 6 8 8 8 6 5 12/30 Write block 461 len 16384 data: 9 9 8 7 13/30 Read block 475 len 36864 14/30 Flush 15/30 Write block 219 len 8192 data: 8 8 16/30 Read block 27 len 20480 17/30 Read block 240 len 16384 18/30 Read block 26 len 36864 19/30 Write block 142 len 32768 data: 7 5 5 7 8 8 9 7 20/30 Read block 562 len 28672 21/30 Read block 171 len 4096 22/30 Flush 23/30 Flush 24/30 Write block 344 len 16384 data: 8 9 11 12 25/30 Write block 393 len 28672 data: 8 8 8 7 8 9 10 26/30 Read block 496 len 24576 27/30 Write block 343 len 16384 data: 6 9 10 12 28/30 Write block 419 len 32768 data: 8 8 8 9 7 8 8 7 29/30 Read block 433 len 24576 30/30 Read block 74 len 8192 ---------------------------------------------------------------- Crucible gen:86 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 4 Done Done Done false 1024 Acked Write 7 Done Done Done false 1025 Acked Read 6 Done Done Done false 1026 Acked Write 4 Done Done Done false 1027 Acked Write 8 Done Done Done false 1028 Acked Read 6 Done Done Done false 1029 Acked Read 2 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 0 2 Done 6 6 7 19 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:36:20.550742415Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1526} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 77 77 77 146 146 146 F F F 1 020-039 84 84 84 156 156 156 F F F 2 040-059 73 73 73 137 137 137 F F F 3 060-079 85 85 85 160 160 160 F F F 4 080-099 85 85 85 160 160 160 F F F 5 100-119 84 84 84 157 157 157 F F F 6 120-139 81 81 81 151 151 151 F F F 7 140-159 86 86 86 163 163 163 F F F 8 160-179 86 86 86 161 161 161 F F F 9 180-199 86 86 86 161 161 161 F F F 10 200-219 86 86 86 163 163 163 F F F 11 220-239 86 86 86 163 163 163 F F F 12 240-259 86 86 86 161 161 161 F F F 13 260-279 83 83 83 154 154 154 F F F 14 280-299 83 83 83 154 154 154 F F F 15 300-319 77 77 77 146 146 146 F F F 16 320-339 84 84 84 156 156 156 F F F 17 340-359 84 84 84 158 158 158 T T T 18 360-379 81 81 81 151 151 151 F F F 19 380-399 82 82 82 153 153 153 T T T 20 400-419 84 84 84 158 158 158 T T T 21 420-439 85 85 85 160 160 160 T T T 22 440-459 86 86 86 161 161 161 F F F 23 460-479 86 86 86 162 162 162 F F F 24 480-499 82 82 82 153 153 153 F F F 25 500-519 81 81 81 151 151 151 F F F 26 520-539 81 81 81 151 151 151 F F F 27 540-559 81 81 81 151 151 151 F F F 28 560-579 86 86 86 162 162 162 F F F 29 580-599 86 86 86 162 162 162 F F F Max gen: 86, Max flush: 163 A Difference in extent metadata is expected here On loop 43 Verifying data now No change after verify Loop: 43 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 77 77 77 146 146 146 F F F 1 020-039 84 84 84 156 156 156 F F F 2 040-059 73 73 73 137 137 137 F F F 3 060-079 85 85 85 160 160 160 F F F 4 080-099 85 85 85 160 160 160 F F F 5 100-119 84 84 84 157 157 157 F F F 6 120-139 81 81 81 151 151 151 F F F 7 140-159 86 86 86 163 163 163 F F F 8 160-179 86 86 86 161 161 161 F F F 9 180-199 86 86 86 161 161 161 F F F 10 200-219 86 86 86 163 163 163 F F F 11 220-239 86 86 86 163 163 163 F F F 12 240-259 86 86 86 161 161 161 F F F 13 260-279 83 83 83 154 154 154 F F F 14 280-299 83 83 83 154 154 154 F F F 15 300-319 77 77 77 146 146 146 F F F 16 320-339 84 84 84 156 156 156 F F F 17 340-359 87 87 87 164 164 164 F F F 18 360-379 81 81 81 151 151 151 F F F 19 380-399 87 87 87 164 164 164 F F F 20 400-419 87 87 87 164 164 164 F F F 21 420-439 87 87 87 164 164 164 F F F 22 440-459 86 86 86 161 161 161 F F F 23 460-479 86 86 86 162 162 162 F F F 24 480-499 82 82 82 153 153 153 F F F 25 500-519 81 81 81 151 151 151 F F F 26 520-539 81 81 81 151 151 151 F F F 27 540-559 81 81 81 151 151 151 F F F 28 560-579 86 86 86 162 162 162 F F F 29 580-599 86 86 86 162 162 162 F F F Max gen: 87, Max flush: 164 Begin loop 44 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:24.844333966Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1534,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:36:24.88691341Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1534,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:24.887326271Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1534,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:24.918686489Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1534,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:24.91925224Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1534} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"26e7d501b33f47379e53601720353c2983e7654b\",\n git_commit_timestamp: \"2024-12-12T20:16:40.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:24.919285828Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1534} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:24.919303486Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1534} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:24.919317856Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1534} {"msg":"Crucible d0092fbe-59fb-4e9d-963f-68ba64889875 has session id: 2f9374c4-7768-4bd3-b1f3-db62aa546994","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:24.919743076Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1534,"session_id":"2f9374c4-7768-4bd3-b1f3-db62aa546994"} {"msg":"Upstairs opts: Upstairs UUID: d0092fbe-59fb-4e9d-963f-68ba64889875, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:24.919790333Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1534,"session_id":"2f9374c4-7768-4bd3-b1f3-db62aa546994"} {"msg":"Crucible stats registered with UUID: d0092fbe-59fb-4e9d-963f-68ba64889875","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:24.919810512Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1534,"session_id":"2f9374c4-7768-4bd3-b1f3-db62aa546994"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:24.950277091Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1534,"task":"crutest"} {"msg":"The guest has requested activation with gen:88","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:24.950306819Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1534} {"msg":"d0092fbe-59fb-4e9d-963f-68ba64889875 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:24.950376384Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1534,"session_id":"2f9374c4-7768-4bd3-b1f3-db62aa546994"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:24.950397613Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1534,"client":"0","":"downstairs","session_id":"2f9374c4-7768-4bd3-b1f3-db62aa546994"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:24.950425411Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1534,"client":"0","":"downstairs","session_id":"2f9374c4-7768-4bd3-b1f3-db62aa546994"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:24.950442779Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1534,"client":"1","":"downstairs","session_id":"2f9374c4-7768-4bd3-b1f3-db62aa546994"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:24.950460488Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1534,"client":"1","":"downstairs","session_id":"2f9374c4-7768-4bd3-b1f3-db62aa546994"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:24.950476857Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1534,"client":"2","":"downstairs","session_id":"2f9374c4-7768-4bd3-b1f3-db62aa546994"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:24.950494326Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1534,"client":"2","":"downstairs","session_id":"2f9374c4-7768-4bd3-b1f3-db62aa546994"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:24.950554102Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1534,"":"io task","client":"2","":"downstairs","session_id":"2f9374c4-7768-4bd3-b1f3-db62aa546994"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:24.950591709Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1534,"":"io task","client":"0","":"downstairs","session_id":"2f9374c4-7768-4bd3-b1f3-db62aa546994"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:24.950627207Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1534,"":"io task","client":"1","":"downstairs","session_id":"2f9374c4-7768-4bd3-b1f3-db62aa546994"} {"msg":"ds_connection connected from Ok(127.0.0.1:56580)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:24.950663304Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1534,"":"io task","client":"2","":"downstairs","session_id":"2f9374c4-7768-4bd3-b1f3-db62aa546994"} {"msg":"ds_connection connected from Ok(127.0.0.1:39834)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:24.950700941Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1534,"":"io task","client":"1","":"downstairs","session_id":"2f9374c4-7768-4bd3-b1f3-db62aa546994"} {"msg":"ds_connection connected from Ok(127.0.0.1:37319)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:24.950736929Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1534,"":"io task","client":"0","":"downstairs","session_id":"2f9374c4-7768-4bd3-b1f3-db62aa546994"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:24.951274762Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1534,"client":"2","":"downstairs","session_id":"2f9374c4-7768-4bd3-b1f3-db62aa546994"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:24.95130623Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1534,"client":"1","":"downstairs","session_id":"2f9374c4-7768-4bd3-b1f3-db62aa546994"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:24.95144598Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1534,"client":"0","":"downstairs","session_id":"2f9374c4-7768-4bd3-b1f3-db62aa546994"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:24.952006671Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1534,"client":"1","":"downstairs","session_id":"2f9374c4-7768-4bd3-b1f3-db62aa546994"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:24.952036469Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1534,"client":"2","":"downstairs","session_id":"2f9374c4-7768-4bd3-b1f3-db62aa546994"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:24.952051728Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1534,"client":"0","":"downstairs","session_id":"2f9374c4-7768-4bd3-b1f3-db62aa546994"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:24.952514756Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1534,"session_id":"2f9374c4-7768-4bd3-b1f3-db62aa546994"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:24.952639628Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1534,"session_id":"2f9374c4-7768-4bd3-b1f3-db62aa546994"} {"msg":"[0]R flush_numbers[0..12]: [146, 156, 137, 160, 160, 157, 151, 163, 161, 161, 163, 163]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:24.952676655Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1534,"":"downstairs","":"downstairs","session_id":"2f9374c4-7768-4bd3-b1f3-db62aa546994"} {"msg":"[0]R generation[0..12]: [77, 84, 73, 85, 85, 84, 81, 86, 86, 86, 86, 86]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:24.952697734Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1534,"":"downstairs","":"downstairs","session_id":"2f9374c4-7768-4bd3-b1f3-db62aa546994"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:24.952713313Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1534,"":"downstairs","":"downstairs","session_id":"2f9374c4-7768-4bd3-b1f3-db62aa546994"} {"msg":"[1]R flush_numbers[0..12]: [146, 156, 137, 160, 160, 157, 151, 163, 161, 161, 163, 163]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:24.952727472Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1534,"":"downstairs","":"downstairs","session_id":"2f9374c4-7768-4bd3-b1f3-db62aa546994"} {"msg":"[1]R generation[0..12]: [77, 84, 73, 85, 85, 84, 81, 86, 86, 86, 86, 86]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:24.952741201Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1534,"":"downstairs","":"downstairs","session_id":"2f9374c4-7768-4bd3-b1f3-db62aa546994"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:24.95275504Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1534,"":"downstairs","":"downstairs","session_id":"2f9374c4-7768-4bd3-b1f3-db62aa546994"} {"msg":"[2]R flush_numbers[0..12]: [146, 156, 137, 160, 160, 157, 151, 163, 161, 161, 163, 163]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:24.952768699Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1534,"":"downstairs","":"downstairs","session_id":"2f9374c4-7768-4bd3-b1f3-db62aa546994"} {"msg":"[2]R generation[0..12]: [77, 84, 73, 85, 85, 84, 81, 86, 86, 86, 86, 86]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:24.952782228Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1534,"":"downstairs","":"downstairs","session_id":"2f9374c4-7768-4bd3-b1f3-db62aa546994"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:24.952795907Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1534,"":"downstairs","":"downstairs","session_id":"2f9374c4-7768-4bd3-b1f3-db62aa546994"} {"msg":"Max found gen is 88","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:24.952808946Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1534,"":"downstairs","":"downstairs","session_id":"2f9374c4-7768-4bd3-b1f3-db62aa546994"} {"msg":"Generation requested: 88 >= found:88","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:24.952822085Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1534,"":"downstairs","":"downstairs","session_id":"2f9374c4-7768-4bd3-b1f3-db62aa546994"} {"msg":"Next flush: 165","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:24.952834964Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1534,"":"downstairs","":"downstairs","session_id":"2f9374c4-7768-4bd3-b1f3-db62aa546994"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:24.952847813Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1534,"":"downstairs","":"downstairs","session_id":"2f9374c4-7768-4bd3-b1f3-db62aa546994"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-12-12T20:36:24.952860763Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1534,"session_id":"2f9374c4-7768-4bd3-b1f3-db62aa546994"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:24.952875132Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1534,"client":"0","":"downstairs","session_id":"2f9374c4-7768-4bd3-b1f3-db62aa546994"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:24.952888531Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1534,"client":"1","":"downstairs","session_id":"2f9374c4-7768-4bd3-b1f3-db62aa546994"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:24.95290319Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1534,"client":"2","":"downstairs","session_id":"2f9374c4-7768-4bd3-b1f3-db62aa546994"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:24.952916449Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1534,"session_id":"2f9374c4-7768-4bd3-b1f3-db62aa546994"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:24.952928648Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1534,"session_id":"2f9374c4-7768-4bd3-b1f3-db62aa546994"} {"msg":"d0092fbe-59fb-4e9d-963f-68ba64889875 is now active with session: 2f9374c4-7768-4bd3-b1f3-db62aa546994","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:24.952941397Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1534,"session_id":"2f9374c4-7768-4bd3-b1f3-db62aa546994"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:24.952953186Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1534,"session_id":"2f9374c4-7768-4bd3-b1f3-db62aa546994"} {"msg":"The guest has finished waiting for activation with:88","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:24.952965295Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1534} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 542 len 32768 data: 7 6 7 6 6 6 6 7 02/30 Flush 03/30 Flush 04/30 Write block 306 len 24576 data: 3 4 5 4 4 8 05/30 Write block 334 len 8192 data: 8 7 06/30 Write block 259 len 8192 data: 7 7 07/30 Read block 424 len 32768 08/30 Write block 585 len 24576 data: 10 10 9 8 7 9 09/30 Write block 456 len 28672 data: 6 7 8 9 9 10 10 10/30 Write block 78 len 12288 data: 8 7 8 11/30 Read block 120 len 28672 12/30 Read block 437 len 28672 13/30 Read block 239 len 20480 14/30 Write block 509 len 20480 data: 6 6 6 6 6 15/30 Write block 352 len 32768 data: 9 8 10 9 8 9 6 5 16/30 Read block 76 len 12288 17/30 Write block 331 len 8192 data: 8 6 18/30 Read block 543 len 32768 19/30 Read block 541 len 20480 20/30 Write block 134 len 24576 data: 5 5 7 7 7 5 21/30 Flush 22/30 Write block 43 len 4096 data: 6 23/30 Read block 22 len 28672 24/30 Write block 62 len 32768 data: 7 8 7 7 7 6 7 8 25/30 Flush 26/30 Write block 421 len 36864 data: 9 10 8 9 9 8 7 7 8 27/30 Read block 517 len 32768 28/30 Write block 396 len 24576 data: 8 9 10 11 9 10 29/30 Write block 488 len 8192 data: 7 8 30/30 Write block 586 len 32768 data: 11 10 9 8 10 9 9 7 ---------------------------------------------------------------- Crucible gen:88 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 9 Done Done Done false 1026 Acked Read 8 Done Done Done false 1027 Acked Write 6 Done Done Done false 1028 Acked Write 2 Done Done Done false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:36:24.972120135Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1534} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 77 77 77 146 146 146 F F F 1 020-039 84 84 84 156 156 156 F F F 2 040-059 88 88 88 168 168 168 F F F 3 060-079 88 88 88 168 168 168 F F F 4 080-099 88 88 88 167 167 167 F F F 5 100-119 84 84 84 157 157 157 F F F 6 120-139 88 88 88 167 167 167 F F F 7 140-159 86 86 86 163 163 163 F F F 8 160-179 86 86 86 161 161 161 F F F 9 180-199 86 86 86 161 161 161 F F F 10 200-219 86 86 86 163 163 163 F F F 11 220-239 86 86 86 163 163 163 F F F 12 240-259 88 88 88 167 167 167 F F F 13 260-279 88 88 88 167 167 167 F F F 14 280-299 83 83 83 154 154 154 F F F 15 300-319 88 88 88 167 167 167 F F F 16 320-339 88 88 88 167 167 167 F F F 17 340-359 88 88 88 167 167 167 F F F 18 360-379 81 81 81 151 151 151 F F F 19 380-399 87 87 87 164 164 164 T T T 20 400-419 87 87 87 164 164 164 T T T 21 420-439 87 87 87 164 164 164 T T T 22 440-459 88 88 88 167 167 167 F F F 23 460-479 88 88 88 167 167 167 F F F 24 480-499 82 82 82 153 153 153 T T T 25 500-519 88 88 88 167 167 167 F F F 26 520-539 81 81 81 151 151 151 F F F 27 540-559 88 88 88 165 165 165 F F F 28 560-579 86 86 86 162 162 162 F F F 29 580-599 88 88 88 167 167 167 T T T Max gen: 88, Max flush: 168 A Difference in extent metadata is expected here On loop 44 Verifying data now No change after verify Loop: 44 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 77 77 77 146 146 146 F F F 1 020-039 84 84 84 156 156 156 F F F 2 040-059 88 88 88 168 168 168 F F F 3 060-079 88 88 88 168 168 168 F F F 4 080-099 88 88 88 167 167 167 F F F 5 100-119 84 84 84 157 157 157 F F F 6 120-139 88 88 88 167 167 167 F F F 7 140-159 86 86 86 163 163 163 F F F 8 160-179 86 86 86 161 161 161 F F F 9 180-199 86 86 86 161 161 161 F F F 10 200-219 86 86 86 163 163 163 F F F 11 220-239 86 86 86 163 163 163 F F F 12 240-259 88 88 88 167 167 167 F F F 13 260-279 88 88 88 167 167 167 F F F 14 280-299 83 83 83 154 154 154 F F F 15 300-319 88 88 88 167 167 167 F F F 16 320-339 88 88 88 167 167 167 F F F 17 340-359 88 88 88 167 167 167 F F F 18 360-379 81 81 81 151 151 151 F F F 19 380-399 89 89 89 169 169 169 F F F 20 400-419 89 89 89 169 169 169 F F F 21 420-439 89 89 89 169 169 169 F F F 22 440-459 88 88 88 167 167 167 F F F 23 460-479 88 88 88 167 167 167 F F F 24 480-499 89 89 89 169 169 169 F F F 25 500-519 88 88 88 167 167 167 F F F 26 520-539 81 81 81 151 151 151 F F F 27 540-559 88 88 88 165 165 165 F F F 28 560-579 86 86 86 162 162 162 F F F 29 580-599 89 89 89 169 169 169 F F F Max gen: 89, Max flush: 169 Begin loop 45 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:29.27361696Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1542,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:36:29.316206704Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1542,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:29.316610046Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1542,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:29.348185219Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1542,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:29.348819065Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1542} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"26e7d501b33f47379e53601720353c2983e7654b\",\n git_commit_timestamp: \"2024-12-12T20:16:40.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:29.348848683Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1542} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:29.348862112Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1542} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:29.348875991Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1542} {"msg":"Crucible d4a59135-9398-48d7-9bdf-819b38fa2001 has session id: cedefae2-625e-467a-b681-cbf206e12468","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:29.349346438Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1542,"session_id":"cedefae2-625e-467a-b681-cbf206e12468"} {"msg":"Upstairs opts: Upstairs UUID: d4a59135-9398-48d7-9bdf-819b38fa2001, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:29.349393505Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1542,"session_id":"cedefae2-625e-467a-b681-cbf206e12468"} {"msg":"Crucible stats registered with UUID: d4a59135-9398-48d7-9bdf-819b38fa2001","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:29.349415154Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1542,"session_id":"cedefae2-625e-467a-b681-cbf206e12468"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:29.381711167Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1542,"task":"crutest"} {"msg":"The guest has requested activation with gen:90","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:29.381736045Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1542} {"msg":"d4a59135-9398-48d7-9bdf-819b38fa2001 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:29.381823479Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1542,"session_id":"cedefae2-625e-467a-b681-cbf206e12468"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:29.381844698Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1542,"client":"0","":"downstairs","session_id":"cedefae2-625e-467a-b681-cbf206e12468"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:29.381871716Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1542,"client":"0","":"downstairs","session_id":"cedefae2-625e-467a-b681-cbf206e12468"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:29.381892544Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1542,"client":"1","":"downstairs","session_id":"cedefae2-625e-467a-b681-cbf206e12468"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:29.381911763Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1542,"client":"1","":"downstairs","session_id":"cedefae2-625e-467a-b681-cbf206e12468"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:29.381928462Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1542,"client":"2","":"downstairs","session_id":"cedefae2-625e-467a-b681-cbf206e12468"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:29.382002347Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1542,"client":"2","":"downstairs","session_id":"cedefae2-625e-467a-b681-cbf206e12468"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:29.382030435Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1542,"":"io task","client":"2","":"downstairs","session_id":"cedefae2-625e-467a-b681-cbf206e12468"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:29.382046544Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1542,"":"io task","client":"0","":"downstairs","session_id":"cedefae2-625e-467a-b681-cbf206e12468"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:29.382060563Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1542,"":"io task","client":"1","":"downstairs","session_id":"cedefae2-625e-467a-b681-cbf206e12468"} {"msg":"ds_connection connected from Ok(127.0.0.1:48873)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:29.382074502Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1542,"":"io task","client":"1","":"downstairs","session_id":"cedefae2-625e-467a-b681-cbf206e12468"} {"msg":"ds_connection connected from Ok(127.0.0.1:37190)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:29.382157276Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1542,"":"io task","client":"0","":"downstairs","session_id":"cedefae2-625e-467a-b681-cbf206e12468"} {"msg":"ds_connection connected from Ok(127.0.0.1:35636)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:29.382197043Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1542,"":"io task","client":"2","":"downstairs","session_id":"cedefae2-625e-467a-b681-cbf206e12468"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:29.382898495Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1542,"client":"2","":"downstairs","session_id":"cedefae2-625e-467a-b681-cbf206e12468"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:29.382934482Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1542,"client":"1","":"downstairs","session_id":"cedefae2-625e-467a-b681-cbf206e12468"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:29.38296491Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1542,"client":"0","":"downstairs","session_id":"cedefae2-625e-467a-b681-cbf206e12468"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:29.383592427Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1542,"client":"1","":"downstairs","session_id":"cedefae2-625e-467a-b681-cbf206e12468"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:29.383625505Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1542,"client":"0","":"downstairs","session_id":"cedefae2-625e-467a-b681-cbf206e12468"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:29.383709859Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1542,"client":"2","":"downstairs","session_id":"cedefae2-625e-467a-b681-cbf206e12468"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:29.384174557Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1542,"session_id":"cedefae2-625e-467a-b681-cbf206e12468"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:29.384209374Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1542,"session_id":"cedefae2-625e-467a-b681-cbf206e12468"} {"msg":"[0]R flush_numbers[0..12]: [146, 156, 168, 168, 167, 157, 167, 163, 161, 161, 163, 163]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:29.384227203Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1542,"":"downstairs","":"downstairs","session_id":"cedefae2-625e-467a-b681-cbf206e12468"} {"msg":"[0]R generation[0..12]: [77, 84, 88, 88, 88, 84, 88, 86, 86, 86, 86, 86]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:29.384242012Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1542,"":"downstairs","":"downstairs","session_id":"cedefae2-625e-467a-b681-cbf206e12468"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:29.384256521Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1542,"":"downstairs","":"downstairs","session_id":"cedefae2-625e-467a-b681-cbf206e12468"} {"msg":"[1]R flush_numbers[0..12]: [146, 156, 168, 168, 167, 157, 167, 163, 161, 161, 163, 163]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:29.38427027Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1542,"":"downstairs","":"downstairs","session_id":"cedefae2-625e-467a-b681-cbf206e12468"} {"msg":"[1]R generation[0..12]: [77, 84, 88, 88, 88, 84, 88, 86, 86, 86, 86, 86]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:29.384283839Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1542,"":"downstairs","":"downstairs","session_id":"cedefae2-625e-467a-b681-cbf206e12468"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:29.384297538Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1542,"":"downstairs","":"downstairs","session_id":"cedefae2-625e-467a-b681-cbf206e12468"} {"msg":"[2]R flush_numbers[0..12]: [146, 156, 168, 168, 167, 157, 167, 163, 161, 161, 163, 163]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:29.384310987Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1542,"":"downstairs","":"downstairs","session_id":"cedefae2-625e-467a-b681-cbf206e12468"} {"msg":"[2]R generation[0..12]: [77, 84, 88, 88, 88, 84, 88, 86, 86, 86, 86, 86]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:29.384324367Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1542,"":"downstairs","":"downstairs","session_id":"cedefae2-625e-467a-b681-cbf206e12468"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:29.384337986Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1542,"":"downstairs","":"downstairs","session_id":"cedefae2-625e-467a-b681-cbf206e12468"} {"msg":"Max found gen is 90","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:29.384350935Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1542,"":"downstairs","":"downstairs","session_id":"cedefae2-625e-467a-b681-cbf206e12468"} {"msg":"Generation requested: 90 >= found:90","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:29.384366864Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1542,"":"downstairs","":"downstairs","session_id":"cedefae2-625e-467a-b681-cbf206e12468"} {"msg":"Next flush: 170","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:29.384379893Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1542,"":"downstairs","":"downstairs","session_id":"cedefae2-625e-467a-b681-cbf206e12468"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-12-12T20:36:29.384392542Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1542,"":"downstairs","":"downstairs","session_id":"cedefae2-625e-467a-b681-cbf206e12468"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:29.38441559Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1542,"session_id":"cedefae2-625e-467a-b681-cbf206e12468"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:29.384438179Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1542,"client":"0","":"downstairs","session_id":"cedefae2-625e-467a-b681-cbf206e12468"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:29.384463597Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1542,"client":"1","":"downstairs","session_id":"cedefae2-625e-467a-b681-cbf206e12468"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:29.384490645Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1542,"client":"2","":"downstairs","session_id":"cedefae2-625e-467a-b681-cbf206e12468"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:29.384518573Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1542,"session_id":"cedefae2-625e-467a-b681-cbf206e12468"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:29.384544371Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1542,"session_id":"cedefae2-625e-467a-b681-cbf206e12468"} {"msg":"d4a59135-9398-48d7-9bdf-819b38fa2001 is now active with session: cedefae2-625e-467a-b681-cbf206e12468","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2024-12-12T20:36:29.384571279Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1542,"session_id":"cedefae2-625e-467a-b681-cbf206e12468"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:29.384598568Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1542,"session_id":"cedefae2-625e-467a-b681-cbf206e12468"} {"msg":"The guest has finished waiting for activation with:90","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:29.384619986Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1542} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 143 len 28672 data: 6 6 8 9 9 10 8 02/30 Flush 03/30 Write block 473 len 28672 data: 6 6 7 7 6 7 6 04/30 Read block 546 len 4096 05/30 Read block 407 len 36864 06/30 Read block 364 len 40960 07/30 Read block 125 len 16384 08/30 Read block 14 len 4096 09/30 Read block 473 len 40960 10/30 Flush 11/30 Write block 571 len 36864 data: 5 5 5 5 5 5 5 7 10 12/30 Read block 440 len 40960 13/30 Read block 328 len 8192 14/30 Read block 538 len 32768 15/30 Read block 249 len 28672 16/30 Write block 236 len 32768 data: 9 8 7 8 7 6 6 6 17/30 Flush 18/30 Write block 518 len 24576 data: 12 15 16 15 14 18 19/30 Write block 291 len 28672 data: 8 8 9 9 9 8 8 20/30 Read block 491 len 20480 21/30 Write block 252 len 16384 data: 5 8 8 8 22/30 Write block 115 len 24576 data: 7 7 5 5 4 7 23/30 Read block 406 len 12288 24/30 Read block 472 len 32768 25/30 Read block 315 len 20480 26/30 Write block 360 len 12288 data: 5 6 8 27/30 Read block 323 len 28672 28/30 Write block 264 len 16384 data: 7 10 11 11 29/30 Write block 487 len 8192 data: 8 8 30/30 Write block 568 len 32768 data: 6 7 5 6 6 6 6 6 ---------------------------------------------------------------- Crucible gen:90 GIO:true work queues: Upstairs:0 downstairs:13 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1017 Acked Write 6 Done Done Done false 1018 Acked Write 7 Done Done Done false 1019 Acked Read 5 Done Done Done false 1020 Acked Write 4 Done Done Done false 1021 Acked Write 6 Done Done Done false 1022 Acked Read 3 Done Done Done false 1023 Acked Read 8 Done Done Done false 1024 Acked Read 5 Done Done Done false 1025 Acked Write 3 Done Done Done false 1026 Acked Read 7 Done Done Done false 1027 Acked Write 4 Done Done Done false 1028 Acked Write 2 Done Done Done false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 12 12 12 36 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1016 1016 1016 Downstairs last five completed: 1016 1015 1014 1013 1012 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:36:29.404838852Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1542} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 77 77 77 146 146 146 F F F 1 020-039 84 84 84 156 156 156 F F F 2 040-059 88 88 88 168 168 168 F F F 3 060-079 88 88 88 168 168 168 F F F 4 080-099 88 88 88 167 167 167 F F F 5 100-119 84 84 84 157 157 157 T T T 6 120-139 88 88 88 167 167 167 T T T 7 140-159 90 90 90 170 170 170 F F F 8 160-179 86 86 86 161 161 161 F F F 9 180-199 86 86 86 161 161 161 F F F 10 200-219 86 86 86 163 163 163 F F F 11 220-239 90 90 90 172 172 172 F F F 12 240-259 90 90 90 172 172 172 T T T 13 260-279 88 88 88 167 167 167 T T T 14 280-299 83 83 83 154 154 154 T T T 15 300-319 88 88 88 167 167 167 F F F 16 320-339 88 88 88 167 167 167 F F F 17 340-359 88 88 88 167 167 167 F F F 18 360-379 81 81 81 151 151 151 T T T 19 380-399 89 89 89 169 169 169 F F F 20 400-419 89 89 89 169 169 169 F F F 21 420-439 89 89 89 169 169 169 F F F 22 440-459 88 88 88 167 167 167 F F F 23 460-479 90 90 90 171 171 171 F F F 24 480-499 89 89 89 169 169 169 T T T 25 500-519 88 88 88 167 167 167 T T T 26 520-539 81 81 81 151 151 151 T T T 27 540-559 88 88 88 165 165 165 F F F 28 560-579 90 90 90 172 172 172 T T T 29 580-599 89 89 89 169 169 169 F F F Max gen: 90, Max flush: 172 A Difference in extent metadata is expected here On loop 45 Verifying data now No change after verify Loop: 45 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 77 77 77 146 146 146 F F F 1 020-039 84 84 84 156 156 156 F F F 2 040-059 88 88 88 168 168 168 F F F 3 060-079 88 88 88 168 168 168 F F F 4 080-099 88 88 88 167 167 167 F F F 5 100-119 91 91 91 173 173 173 F F F 6 120-139 91 91 91 173 173 173 F F F 7 140-159 90 90 90 170 170 170 F F F 8 160-179 86 86 86 161 161 161 F F F 9 180-199 86 86 86 161 161 161 F F F 10 200-219 86 86 86 163 163 163 F F F 11 220-239 90 90 90 172 172 172 F F F 12 240-259 91 91 91 173 173 173 F F F 13 260-279 91 91 91 173 173 173 F F F 14 280-299 91 91 91 173 173 173 F F F 15 300-319 88 88 88 167 167 167 F F F 16 320-339 88 88 88 167 167 167 F F F 17 340-359 88 88 88 167 167 167 F F F 18 360-379 91 91 91 173 173 173 F F F 19 380-399 89 89 89 169 169 169 F F F 20 400-419 89 89 89 169 169 169 F F F 21 420-439 89 89 89 169 169 169 F F F 22 440-459 88 88 88 167 167 167 F F F 23 460-479 90 90 90 171 171 171 F F F 24 480-499 91 91 91 173 173 173 F F F 25 500-519 91 91 91 173 173 173 F F F 26 520-539 91 91 91 173 173 173 F F F 27 540-559 88 88 88 165 165 165 F F F 28 560-579 91 91 91 173 173 173 F F F 29 580-599 89 89 89 169 169 169 F F F Max gen: 91, Max flush: 173 Begin loop 46 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:33.739032503Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1550,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:33.783354447Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1550,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:33.784079307Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1550} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"26e7d501b33f47379e53601720353c2983e7654b\",\n git_commit_timestamp: \"2024-12-12T20:16:40.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:33.784117135Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1550} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:33.784131503Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1550} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:33.784145982Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1550} {"msg":"Crucible 18d6eafa-6ee8-4099-a100-cf50e6e55288 has session id: 01268075-5d4d-4806-8a43-ad2725f52213","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:33.784554874Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1550,"session_id":"01268075-5d4d-4806-8a43-ad2725f52213"} {"msg":"Upstairs opts: Upstairs UUID: 18d6eafa-6ee8-4099-a100-cf50e6e55288, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:33.784602521Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1550,"session_id":"01268075-5d4d-4806-8a43-ad2725f52213"} {"msg":"Crucible stats registered with UUID: 18d6eafa-6ee8-4099-a100-cf50e6e55288","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:33.784632129Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1550,"session_id":"01268075-5d4d-4806-8a43-ad2725f52213"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:33.815414146Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1550,"task":"crutest"} {"msg":"The guest has requested activation with gen:92","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:33.815445414Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1550} {"msg":"18d6eafa-6ee8-4099-a100-cf50e6e55288 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:33.81550302Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1550,"session_id":"01268075-5d4d-4806-8a43-ad2725f52213"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:33.815519619Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1550,"client":"0","":"downstairs","session_id":"01268075-5d4d-4806-8a43-ad2725f52213"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:33.815546707Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1550,"client":"0","":"downstairs","session_id":"01268075-5d4d-4806-8a43-ad2725f52213"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:33.815563956Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1550,"client":"1","":"downstairs","session_id":"01268075-5d4d-4806-8a43-ad2725f52213"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:33.815581485Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1550,"client":"1","":"downstairs","session_id":"01268075-5d4d-4806-8a43-ad2725f52213"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:33.815598204Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1550,"client":"2","":"downstairs","session_id":"01268075-5d4d-4806-8a43-ad2725f52213"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:33.815615863Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1550,"client":"2","":"downstairs","session_id":"01268075-5d4d-4806-8a43-ad2725f52213"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:33.815632281Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1550,"":"io task","client":"2","":"downstairs","session_id":"01268075-5d4d-4806-8a43-ad2725f52213"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:33.8156494Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1550,"":"io task","client":"0","":"downstairs","session_id":"01268075-5d4d-4806-8a43-ad2725f52213"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:33.815666809Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1550,"":"io task","client":"1","":"downstairs","session_id":"01268075-5d4d-4806-8a43-ad2725f52213"} {"msg":"ds_connection connected from Ok(127.0.0.1:45943)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:33.815783111Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1550,"":"io task","client":"1","":"downstairs","session_id":"01268075-5d4d-4806-8a43-ad2725f52213"} {"msg":"ds_connection connected from Ok(127.0.0.1:51874)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:33.816043903Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1550,"":"io task","client":"2","":"downstairs","session_id":"01268075-5d4d-4806-8a43-ad2725f52213"} {"msg":"ds_connection connected from Ok(127.0.0.1:39442)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:33.816101109Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1550,"":"io task","client":"0","":"downstairs","session_id":"01268075-5d4d-4806-8a43-ad2725f52213"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:33.816603254Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1550,"client":"0","":"downstairs","session_id":"01268075-5d4d-4806-8a43-ad2725f52213"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:33.816633092Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1550,"client":"1","":"downstairs","session_id":"01268075-5d4d-4806-8a43-ad2725f52213"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:33.81666247Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1550,"client":"2","":"downstairs","session_id":"01268075-5d4d-4806-8a43-ad2725f52213"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:33.817155296Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1550,"client":"0","":"downstairs","session_id":"01268075-5d4d-4806-8a43-ad2725f52213"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:33.817255429Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1550,"client":"1","":"downstairs","session_id":"01268075-5d4d-4806-8a43-ad2725f52213"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:33.817472434Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1550,"client":"2","":"downstairs","session_id":"01268075-5d4d-4806-8a43-ad2725f52213"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:33.817776143Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1550,"session_id":"01268075-5d4d-4806-8a43-ad2725f52213"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:33.817888146Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1550,"session_id":"01268075-5d4d-4806-8a43-ad2725f52213"} {"msg":"[0]R flush_numbers[0..12]: [146, 156, 168, 168, 167, 173, 173, 170, 161, 161, 163, 172]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:33.818041285Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1550,"":"downstairs","":"downstairs","session_id":"01268075-5d4d-4806-8a43-ad2725f52213"} {"msg":"[0]R generation[0..12]: [77, 84, 88, 88, 88, 91, 91, 90, 86, 86, 86, 90]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:33.818061074Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1550,"":"downstairs","":"downstairs","session_id":"01268075-5d4d-4806-8a43-ad2725f52213"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:33.818075673Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1550,"":"downstairs","":"downstairs","session_id":"01268075-5d4d-4806-8a43-ad2725f52213"} {"msg":"[1]R flush_numbers[0..12]: [146, 156, 168, 168, 167, 173, 173, 170, 161, 161, 163, 172]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:33.818089682Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1550,"":"downstairs","":"downstairs","session_id":"01268075-5d4d-4806-8a43-ad2725f52213"} {"msg":"[1]R generation[0..12]: [77, 84, 88, 88, 88, 91, 91, 90, 86, 86, 86, 90]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:33.818103181Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1550,"":"downstairs","":"downstairs","session_id":"01268075-5d4d-4806-8a43-ad2725f52213"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:33.81812015Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1550,"":"downstairs","":"downstairs","session_id":"01268075-5d4d-4806-8a43-ad2725f52213"} {"msg":"[2]R flush_numbers[0..12]: [146, 156, 168, 168, 167, 173, 173, 170, 161, 161, 163, 172]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:33.818134309Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1550,"":"downstairs","":"downstairs","session_id":"01268075-5d4d-4806-8a43-ad2725f52213"} {"msg":"[2]R generation[0..12]: [77, 84, 88, 88, 88, 91, 91, 90, 86, 86, 86, 90]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:33.818154057Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1550,"":"downstairs","":"downstairs","session_id":"01268075-5d4d-4806-8a43-ad2725f52213"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:33.818180066Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1550,"":"downstairs","":"downstairs","session_id":"01268075-5d4d-4806-8a43-ad2725f52213"} {"msg":"Max found gen is 92","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:33.818206954Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1550,"":"downstairs","":"downstairs","session_id":"01268075-5d4d-4806-8a43-ad2725f52213"} {"msg":"Generation requested: 92 >= found:92","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:33.818223463Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1550,"":"downstairs","":"downstairs","session_id":"01268075-5d4d-4806-8a43-ad2725f52213"} {"msg":"Next flush: 174","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-12-12T20:36:33.818246871Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1550,"":"downstairs","":"downstairs","session_id":"01268075-5d4d-4806-8a43-ad2725f52213"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:33.818272089Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1550,"":"downstairs","":"downstairs","session_id":"01268075-5d4d-4806-8a43-ad2725f52213"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:33.818299527Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1550,"session_id":"01268075-5d4d-4806-8a43-ad2725f52213"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:33.818313906Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1550,"client":"0","":"downstairs","session_id":"01268075-5d4d-4806-8a43-ad2725f52213"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:33.818327365Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1550,"client":"1","":"downstairs","session_id":"01268075-5d4d-4806-8a43-ad2725f52213"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:33.818349014Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1550,"client":"2","":"downstairs","session_id":"01268075-5d4d-4806-8a43-ad2725f52213"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:33.818374542Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1550,"session_id":"01268075-5d4d-4806-8a43-ad2725f52213"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:33.818399351Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1550,"session_id":"01268075-5d4d-4806-8a43-ad2725f52213"} {"msg":"18d6eafa-6ee8-4099-a100-cf50e6e55288 is now active with session: 01268075-5d4d-4806-8a43-ad2725f52213","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:33.818415389Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1550,"session_id":"01268075-5d4d-4806-8a43-ad2725f52213"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:33.818432478Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1550,"session_id":"01268075-5d4d-4806-8a43-ad2725f52213"} {"msg":"The guest has finished waiting for activation with:92","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:33.818455037Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1550} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 412 len 32768 data: 5 6 8 7 7 7 9 9 02/30 Write block 416 len 4096 data: 8 03/30 Write block 204 len 12288 data: 10 9 10 04/30 Write block 489 len 16384 data: 9 7 5 5 05/30 Read block 405 len 24576 06/30 Read block 431 len 32768 07/30 Read block 51 len 8192 08/30 Write block 584 len 32768 data: 11 11 12 11 10 9 11 10 09/30 Read block 103 len 40960 10/30 Read block 500 len 4096 11/30 Read block 322 len 4096 12/30 Read block 91 len 32768 13/30 Read block 175 len 20480 14/30 Write block 423 len 8192 data: 9 10 15/30 Write block 49 len 40960 data: 8 8 7 4 6 7 9 12 12 12 16/30 Flush 17/30 Write block 241 len 24576 data: 7 7 7 4 3 5 18/30 Read block 304 len 28672 19/30 Write block 174 len 32768 data: 3 4 4 5 5 8 6 6 20/30 Write block 559 len 8192 data: 9 5 21/30 Write block 373 len 40960 data: 6 8 5 7 6 8 9 10 10 10 22/30 Read block 4 len 12288 23/30 Write block 313 len 36864 data: 10 11 11 11 11 10 7 7 7 24/30 Write block 364 len 24576 data: 9 9 10 9 6 7 25/30 Read block 429 len 36864 26/30 Read block 452 len 8192 27/30 Read block 81 len 16384 28/30 Write block 278 len 4096 data: 8 29/30 Write block 174 len 16384 data: 4 5 5 6 30/30 Write block 175 len 28672 data: 6 6 7 6 9 7 7 ---------------------------------------------------------------- Crucible gen:92 GIO:true work queues: Upstairs:0 downstairs:14 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1016 Acked Write 6 Done Done Done false 1017 Acked Read 7 Done Done Done false 1018 Acked Write 8 Done Done Done false 1019 Acked Write 2 Done Done Done false 1020 Acked Write 10 Done Done Done false 1021 Acked Read 3 Done Done Done false 1022 Acked Write 9 Done Done Done false 1023 Acked Write 6 Done Done Done false 1024 Acked Read 9 Done Done Done false 1025 Acked Read 2 Done Done Done false 1026 Acked Read 4 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Write 4 Done Sent Sent false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 2 2 5 Done 13 12 12 37 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1015 1015 1015 Downstairs last five completed: 1015 1014 1013 1012 1011 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:36:33.837725208Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1550} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 77 77 77 146 146 146 F F F 1 020-039 84 84 84 156 156 156 F F F 2 040-059 92 92 92 174 174 174 F F F 3 060-079 88 88 88 168 168 168 F F F 4 080-099 88 88 88 167 167 167 F F F 5 100-119 91 91 91 173 173 173 F F F 6 120-139 91 91 91 173 173 173 F F F 7 140-159 90 90 90 170 170 170 F F F 8 160-179 86 86 86 161 161 161 T T T 9 180-199 86 86 86 161 161 161 T T T 10 200-219 92 92 92 174 174 174 F F F 11 220-239 90 90 90 172 172 172 F F F 12 240-259 91 91 91 173 173 173 T T T 13 260-279 91 91 91 173 173 173 T T T 14 280-299 91 91 91 173 173 173 F F F 15 300-319 88 88 88 167 167 167 T T T 16 320-339 88 88 88 167 167 167 T T T 17 340-359 88 88 88 167 167 167 F F F 18 360-379 91 91 91 173 173 173 T T T 19 380-399 89 89 89 169 169 169 T T T 20 400-419 92 92 92 174 174 174 F F F 21 420-439 92 92 92 174 174 174 F F F 22 440-459 88 88 88 167 167 167 F F F 23 460-479 90 90 90 171 171 171 F F F 24 480-499 92 92 92 174 174 174 F F F 25 500-519 91 91 91 173 173 173 F F F 26 520-539 91 91 91 173 173 173 F F F 27 540-559 88 88 88 165 165 165 T T T 28 560-579 91 91 91 173 173 173 T T T 29 580-599 92 92 92 174 174 174 F F F Max gen: 92, Max flush: 174 A Difference in extent metadata is expected here On loop 46 Verifying data now No change after verify Loop: 46 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 77 77 77 146 146 146 F F F 1 020-039 84 84 84 156 156 156 F F F 2 040-059 92 92 92 174 174 174 F F F 3 060-079 88 88 88 168 168 168 F F F 4 080-099 88 88 88 167 167 167 F F F 5 100-119 91 91 91 173 173 173 F F F 6 120-139 91 91 91 173 173 173 F F F 7 140-159 90 90 90 170 170 170 F F F 8 160-179 93 93 93 175 175 175 F F F 9 180-199 93 93 93 175 175 175 F F F 10 200-219 92 92 92 174 174 174 F F F 11 220-239 90 90 90 172 172 172 F F F 12 240-259 93 93 93 175 175 175 F F F 13 260-279 93 93 93 175 175 175 F F F 14 280-299 91 91 91 173 173 173 F F F 15 300-319 93 93 93 175 175 175 F F F 16 320-339 93 93 93 175 175 175 F F F 17 340-359 88 88 88 167 167 167 F F F 18 360-379 93 93 93 175 175 175 F F F 19 380-399 93 93 93 175 175 175 F F F 20 400-419 92 92 92 174 174 174 F F F 21 420-439 92 92 92 174 174 174 F F F 22 440-459 88 88 88 167 167 167 F F F 23 460-479 90 90 90 171 171 171 F F F 24 480-499 92 92 92 174 174 174 F F F 25 500-519 91 91 91 173 173 173 F F F 26 520-539 91 91 91 173 173 173 F F F 27 540-559 93 93 93 175 175 175 F F F 28 560-579 93 93 93 175 175 175 F F F 29 580-599 92 92 92 174 174 174 F F F Max gen: 93, Max flush: 175 Begin loop 47 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:38.132283522Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1558,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:36:38.174975768Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1558,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:38.175339753Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1558,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:38.206894797Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1558,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:38.207487926Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1558} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"26e7d501b33f47379e53601720353c2983e7654b\",\n git_commit_timestamp: \"2024-12-12T20:16:40.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:38.207518244Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1558} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:38.207531733Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1558} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:38.207545872Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1558} {"msg":"Crucible a85d102e-01a4-4cce-80e3-b1f17b2dc73f has session id: 37d2723d-cbc8-484e-b295-a61b4cc2e497","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:38.207959634Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1558,"session_id":"37d2723d-cbc8-484e-b295-a61b4cc2e497"} {"msg":"Upstairs opts: Upstairs UUID: a85d102e-01a4-4cce-80e3-b1f17b2dc73f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:38.208002161Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1558,"session_id":"37d2723d-cbc8-484e-b295-a61b4cc2e497"} {"msg":"Crucible stats registered with UUID: a85d102e-01a4-4cce-80e3-b1f17b2dc73f","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:38.208027559Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1558,"session_id":"37d2723d-cbc8-484e-b295-a61b4cc2e497"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:38.240521488Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1558,"task":"crutest"} {"msg":"The guest has requested activation with gen:94","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:38.240566815Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1558} {"msg":"a85d102e-01a4-4cce-80e3-b1f17b2dc73f active request set","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:38.240631301Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1558,"session_id":"37d2723d-cbc8-484e-b295-a61b4cc2e497"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:38.240659429Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1558,"client":"0","":"downstairs","session_id":"37d2723d-cbc8-484e-b295-a61b4cc2e497"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:38.240683927Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1558,"client":"0","":"downstairs","session_id":"37d2723d-cbc8-484e-b295-a61b4cc2e497"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:38.240698816Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1558,"client":"1","":"downstairs","session_id":"37d2723d-cbc8-484e-b295-a61b4cc2e497"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:38.240713295Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1558,"client":"1","":"downstairs","session_id":"37d2723d-cbc8-484e-b295-a61b4cc2e497"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:38.240731534Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1558,"client":"2","":"downstairs","session_id":"37d2723d-cbc8-484e-b295-a61b4cc2e497"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:38.240746063Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1558,"client":"2","":"downstairs","session_id":"37d2723d-cbc8-484e-b295-a61b4cc2e497"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:38.240759682Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1558,"":"io task","client":"2","":"downstairs","session_id":"37d2723d-cbc8-484e-b295-a61b4cc2e497"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:38.240810648Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1558,"":"io task","client":"0","":"downstairs","session_id":"37d2723d-cbc8-484e-b295-a61b4cc2e497"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:38.240848716Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1558,"":"io task","client":"1","":"downstairs","session_id":"37d2723d-cbc8-484e-b295-a61b4cc2e497"} {"msg":"ds_connection connected from Ok(127.0.0.1:62383)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:38.240877744Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1558,"":"io task","client":"2","":"downstairs","session_id":"37d2723d-cbc8-484e-b295-a61b4cc2e497"} {"msg":"ds_connection connected from Ok(127.0.0.1:63261)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:38.240923071Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1558,"":"io task","client":"1","":"downstairs","session_id":"37d2723d-cbc8-484e-b295-a61b4cc2e497"} {"msg":"ds_connection connected from Ok(127.0.0.1:46433)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:38.240953789Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1558,"":"io task","client":"0","":"downstairs","session_id":"37d2723d-cbc8-484e-b295-a61b4cc2e497"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:38.241645861Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1558,"client":"1","":"downstairs","session_id":"37d2723d-cbc8-484e-b295-a61b4cc2e497"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:38.241674859Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1558,"client":"2","":"downstairs","session_id":"37d2723d-cbc8-484e-b295-a61b4cc2e497"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:38.241893284Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1558,"client":"0","":"downstairs","session_id":"37d2723d-cbc8-484e-b295-a61b4cc2e497"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:38.242342883Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1558,"client":"1","":"downstairs","session_id":"37d2723d-cbc8-484e-b295-a61b4cc2e497"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:38.242457385Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1558,"client":"2","":"downstairs","session_id":"37d2723d-cbc8-484e-b295-a61b4cc2e497"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:38.242483953Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1558,"client":"0","":"downstairs","session_id":"37d2723d-cbc8-484e-b295-a61b4cc2e497"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:38.243036535Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1558,"session_id":"37d2723d-cbc8-484e-b295-a61b4cc2e497"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:38.243074732Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1558,"session_id":"37d2723d-cbc8-484e-b295-a61b4cc2e497"} {"msg":"[0]R flush_numbers[0..12]: [146, 156, 174, 168, 167, 173, 173, 170, 175, 175, 174, 172]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:38.243173626Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1558,"":"downstairs","":"downstairs","session_id":"37d2723d-cbc8-484e-b295-a61b4cc2e497"} {"msg":"[0]R generation[0..12]: [77, 84, 92, 88, 88, 91, 91, 90, 93, 93, 92, 90]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:38.243194684Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1558,"":"downstairs","":"downstairs","session_id":"37d2723d-cbc8-484e-b295-a61b4cc2e497"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:38.243210133Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1558,"":"downstairs","":"downstairs","session_id":"37d2723d-cbc8-484e-b295-a61b4cc2e497"} {"msg":"[1]R flush_numbers[0..12]: [146, 156, 174, 168, 167, 173, 173, 170, 175, 175, 174, 172]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:38.243224162Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1558,"":"downstairs","":"downstairs","session_id":"37d2723d-cbc8-484e-b295-a61b4cc2e497"} {"msg":"[1]R generation[0..12]: [77, 84, 92, 88, 88, 91, 91, 90, 93, 93, 92, 90]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:38.243237831Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1558,"":"downstairs","":"downstairs","session_id":"37d2723d-cbc8-484e-b295-a61b4cc2e497"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:38.24325199Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1558,"":"downstairs","":"downstairs","session_id":"37d2723d-cbc8-484e-b295-a61b4cc2e497"} {"msg":"[2]R flush_numbers[0..12]: [146, 156, 174, 168, 167, 173, 173, 170, 175, 175, 174, 172]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:38.243265779Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1558,"":"downstairs","":"downstairs","session_id":"37d2723d-cbc8-484e-b295-a61b4cc2e497"} {"msg":"[2]R generation[0..12]: [77, 84, 92, 88, 88, 91, 91, 90, 93, 93, 92, 90]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:38.243279298Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1558,"":"downstairs","":"downstairs","session_id":"37d2723d-cbc8-484e-b295-a61b4cc2e497"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:38.243293297Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1558,"":"downstairs","":"downstairs","session_id":"37d2723d-cbc8-484e-b295-a61b4cc2e497"} {"msg":"Max found gen is 94","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:38.243306467Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1558,"":"downstairs","":"downstairs","session_id":"37d2723d-cbc8-484e-b295-a61b4cc2e497"} {"msg":"Generation requested: 94 >= found:94","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:38.243319655Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1558,"":"downstairs","":"downstairs","session_id":"37d2723d-cbc8-484e-b295-a61b4cc2e497"} {"msg":"Next flush: 176","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:38.243332445Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1558,"":"downstairs","":"downstairs","session_id":"37d2723d-cbc8-484e-b295-a61b4cc2e497"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:38.243348433Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":Wait for a query_work_queue command to finish before sending IO 1558,"":"downstairs","":"downstairs","session_id":"37d2723d-cbc8-484e-b295-a61b4cc2e497"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:38.243366842Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1558,"session_id":"37d2723d-cbc8-484e-b295-a61b4cc2e497"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:38.243378942Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1558,"client":"0","":"downstairs","session_id":"37d2723d-cbc8-484e-b295-a61b4cc2e497"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:38.243392241Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1558,"client":"1","":"downstairs","session_id":"37d2723d-cbc8-484e-b295-a61b4cc2e497"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:38.24340528Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1558,"client":"2","":"downstairs","session_id":"37d2723d-cbc8-484e-b295-a61b4cc2e497"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:38.243418429Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1558,"session_id":"37d2723d-cbc8-484e-b295-a61b4cc2e497"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:38.243430868Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1558,"session_id":"37d2723d-cbc8-484e-b295-a61b4cc2e497"} {"msg":"a85d102e-01a4-4cce-80e3-b1f17b2dc73f is now active with session: 37d2723d-cbc8-484e-b295-a61b4cc2e497","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:38.243443587Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1558,"session_id":"37d2723d-cbc8-484e-b295-a61b4cc2e497"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:38.243455516Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1558,"session_id":"37d2723d-cbc8-484e-b295-a61b4cc2e497"} {"msg":"The guest has finished waiting for activation with:94","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:38.243467555Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1558} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 75 len 4096 data: 7 02/30 Write block 461 len 12288 data: 11 11 9 03/30 Write block 330 len 8192 data: 8 9 04/30 Write block 385 len 32768 data: 7 8 7 7 7 9 9 8 05/30 Write block 57 len 20480 data: 13 13 13 12 7 06/30 Read block 384 len 8192 07/30 Read block 191 len 36864 08/30 Read block 354 len 12288 09/30 Read block 593 len 28672 10/30 Write block 87 len 24576 data: 11 12 10 10 8 5 11/30 Read block 185 len 36864 12/30 Read block 35 len 24576 13/30 Write block 137 len 32768 data: 8 8 6 5 4 8 7 7 14/30 Write block 434 len 28672 data: 6 8 7 6 6 6 7 15/30 Write block 11 len 16384 data: 8 6 5 6 16/30 Write block 324 len 12288 data: 5 4 4 17/30 Read block 589 len 8192 18/30 Read block 63 len 28672 19/30 Write block 270 len 8192 data: 7 7 20/30 Read block 400 len 8192 21/30 Write block 42 len 8192 data: 7 7 22/30 Flush 23/30 Write block 441 len 16384 data: 7 7 7 7 24/30 Write block 503 len 36864 data: 7 7 6 6 8 7 7 7 7 25/30 Read block 437 len 4096 26/30 Read block 57 len 24576 27/30 Write block 147 len 12288 data: 10 11 9 28/30 Read block 165 len 36864 29/30 Read block 132 len 20480 30/30 Write block 228 len 36864 data: 10 12 12 11 10 10 9 9 10 ---------------------------------------------------------------- Crucible gen:94 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 4 Done Done Done false 1023 Acked Write 9 Done Done Done false 1024 Acked Read 1 Done Done Done false 1025 Acked Read 6 Done Done Done false 1026 Acked Write 3 Done Done Done false 1027 Acked Read 9 Done Done Done false 1028 Acked Read 5 Done Done Done false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 7 7 7 21 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1021 1021 1021 Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:36:38.26181524Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1558} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 94 94 94 176 176 176 F F F 1 020-039 84 84 84 156 156 156 F F F 2 040-059 94 94 94 176 176 176 F F F 3 060-079 94 94 94 176 176 176 F F F 4 080-099 94 94 94 176 176 176 F F F 5 100-119 91 91 91 173 173 173 F F F 6 120-139 94 94 94 176 176 176 F F F 7 140-159 94 94 94 176 176 176 T T T 8 160-179 93 93 93 175 175 175 F F F 9 180-199 93 93 93 175 175 175 F F F 10 200-219 92 92 92 174 174 174 F F F 11 220-239 90 90 90 172 172 172 T T T 12 240-259 93 93 93 175 175 175 F F F 13 260-279 94 94 94 176 176 176 F F F 14 280-299 91 91 91 173 173 173 F F F 15 300-319 93 93 93 175 175 175 F F F 16 320-339 94 94 94 176 176 176 F F F 17 340-359 88 88 88 167 167 167 F F F 18 360-379 93 93 93 175 175 175 F F F 19 380-399 94 94 94 176 176 176 F F F 20 400-419 92 92 92 174 174 174 F F F 21 420-439 94 94 94 176 176 176 F F F 22 440-459 94 94 94 176 176 176 T T T 23 460-479 94 94 94 176 176 176 F F F 24 480-499 92 92 92 174 174 174 F F F 25 500-519 91 91 91 173 173 173 T T T 26 520-539 91 91 91 173 173 173 F F F 27 540-559 93 93 93 175 175 175 F F F 28 560-579 93 93 93 175 175 175 F F F 29 580-599 92 92 92 174 174 174 F F F Max gen: 94, Max flush: 176 A Difference in extent metadata is expected here On loop 47 Verifying data now No change after verify Loop: 47 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 94 94 94 176 176 176 F F F 1 020-039 84 84 84 156 156 156 F F F 2 040-059 94 94 94 176 176 176 F F F 3 060-079 94 94 94 176 176 176 F F F 4 080-099 94 94 94 176 176 176 F F F 5 100-119 91 91 91 173 173 173 F F F 6 120-139 94 94 94 176 176 176 F F F 7 140-159 95 95 95 177 177 177 F F F 8 160-179 93 93 93 175 175 175 F F F 9 180-199 93 93 93 175 175 175 F F F 10 200-219 92 92 92 174 174 174 F F F 11 220-239 95 95 95 177 177 177 F F F 12 240-259 93 93 93 175 175 175 F F F 13 260-279 94 94 94 176 176 176 F F F 14 280-299 91 91 91 173 173 173 F F F 15 300-319 93 93 93 175 175 175 F F F 16 320-339 94 94 94 176 176 176 F F F 17 340-359 88 88 88 167 167 167 F F F 18 360-379 93 93 93 175 175 175 F F F 19 380-399 94 94 94 176 176 176 F F F 20 400-419 92 92 92 174 174 174 F F F 21 420-439 94 94 94 176 176 176 F F F 22 440-459 95 95 95 177 177 177 F F F 23 460-479 94 94 94 176 176 176 F F F 24 480-499 92 92 92 174 174 174 F F F 25 500-519 95 95 95 177 177 177 F F F 26 520-539 91 91 91 173 173 173 F F F 27 540-559 93 93 93 175 175 175 F F F 28 560-579 93 93 93 175 175 175 F F F 29 580-599 92 92 92 174 174 174 F F F Max gen: 95, Max flush: 177 Begin loop 48 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:42.557023999Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1566,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:42.600924552Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1566,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:42.601579137Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1566} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"26e7d501b33f47379e53601720353c2983e7654b\",\n git_commit_timestamp: \"2024-12-12T20:16:40.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:42.601615014Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1566} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:42.601629283Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1566} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:42.601643522Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1566} {"msg":"Crucible ea829a78-87fa-43b4-bae0-af28689dbce5 has session id: dfbd444f-da22-4f23-a7a2-76508aba5f3a","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:42.60210951Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1566,"session_id":"dfbd444f-da22-4f23-a7a2-76508aba5f3a"} {"msg":"Upstairs opts: Upstairs UUID: ea829a78-87fa-43b4-bae0-af28689dbce5, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:42.602157867Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1566,"session_id":"dfbd444f-da22-4f23-a7a2-76508aba5f3a"} {"msg":"Crucible stats registered with UUID: ea829a78-87fa-43b4-bae0-af28689dbce5","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:42.602181165Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1566,"session_id":"dfbd444f-da22-4f23-a7a2-76508aba5f3a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:42.633191727Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1566,"task":"crutest"} {"msg":"The guest has requested activation with gen:96","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:42.633223695Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1566} {"msg":"ea829a78-87fa-43b4-bae0-af28689dbce5 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:42.63329839Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1566,"session_id":"dfbd444f-da22-4f23-a7a2-76508aba5f3a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:42.633320098Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1566,"client":"0","":"downstairs","session_id":"dfbd444f-da22-4f23-a7a2-76508aba5f3a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:42.633347716Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1566,"client":"0","":"downstairs","session_id":"dfbd444f-da22-4f23-a7a2-76508aba5f3a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:42.633364915Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1566,"client":"1","":"downstairs","session_id":"dfbd444f-da22-4f23-a7a2-76508aba5f3a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:42.633382614Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1566,"client":"1","":"downstairs","session_id":"dfbd444f-da22-4f23-a7a2-76508aba5f3a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:42.633398972Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1566,"client":"2","":"downstairs","session_id":"dfbd444f-da22-4f23-a7a2-76508aba5f3a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:42.633416171Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1566,"client":"2","":"downstairs","session_id":"dfbd444f-da22-4f23-a7a2-76508aba5f3a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:42.63343212Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1566,"":"io task","client":"2","":"downstairs","session_id":"dfbd444f-da22-4f23-a7a2-76508aba5f3a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:42.633449269Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1566,"":"io task","client":"0","":"downstairs","session_id":"dfbd444f-da22-4f23-a7a2-76508aba5f3a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:42.633466098Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1566,"":"io task","client":"1","":"downstairs","session_id":"dfbd444f-da22-4f23-a7a2-76508aba5f3a"} {"msg":"ds_connection connected from Ok(127.0.0.1:45664)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:42.633562671Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1566,"":"io task","client":"0","":"downstairs","session_id":"dfbd444f-da22-4f23-a7a2-76508aba5f3a"} {"msg":"ds_connection connected from Ok(127.0.0.1:61890)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:42.633611298Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1566,"":"io task","client":"1","":"downstairs","session_id":"dfbd444f-da22-4f23-a7a2-76508aba5f3a"} {"msg":"ds_connection connected from Ok(127.0.0.1:36874)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:42.633736049Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1566,"":"io task","client":"2","":"downstairs","session_id":"dfbd444f-da22-4f23-a7a2-76508aba5f3a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:42.634328918Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1566,"client":"0","":"downstairs","session_id":"dfbd444f-da22-4f23-a7a2-76508aba5f3a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:42.634487968Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1566,"client":"1","":"downstairs","session_id":"dfbd444f-da22-4f23-a7a2-76508aba5f3a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:42.634536584Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1566,"client":"2","":"downstairs","session_id":"dfbd444f-da22-4f23-a7a2-76508aba5f3a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:42.634946036Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1566,"client":"0","":"downstairs","session_id":"dfbd444f-da22-4f23-a7a2-76508aba5f3a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:42.635106175Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1566,"client":"1","":"downstairs","session_id":"dfbd444f-da22-4f23-a7a2-76508aba5f3a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:42.635191219Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1566,"client":"2","":"downstairs","session_id":"dfbd444f-da22-4f23-a7a2-76508aba5f3a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:42.635486999Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1566,"session_id":"dfbd444f-da22-4f23-a7a2-76508aba5f3a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:42.635643828Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1566,"session_id":"dfbd444f-da22-4f23-a7a2-76508aba5f3a"} {"msg":"[0]R flush_numbers[0..12]: [176, 156, 176, 176, 176, 173, 176, 177, 175, 175, 174, 177]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:42.635768169Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1566,"":"downstairs","":"downstairs","session_id":"dfbd444f-da22-4f23-a7a2-76508aba5f3a"} {"msg":"[0]R generation[0..12]: [94, 84, 94, 94, 94, 91, 94, 95, 93, 93, 92, 95]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:42.635788518Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1566,"":"downstairs","":"downstairs","session_id":"dfbd444f-da22-4f23-a7a2-76508aba5f3a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:42.635803547Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1566,"":"downstairs","":"downstairs","session_id":"dfbd444f-da22-4f23-a7a2-76508aba5f3a"} {"msg":"[1]R flush_numbers[0..12]: [176, 156, 176, 176, 176, 173, 176, 177, 175, 175, 174, 177]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:42.635817766Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1566,"":"downstairs","":"downstairs","session_id":"dfbd444f-da22-4f23-a7a2-76508aba5f3a"} {"msg":"[1]R generation[0..12]: [94, 84, 94, 94, 94, 91, 94, 95, 93, 93, 92, 95]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:42.635831715Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1566,"":"downstairs","":"downstairs","session_id":"dfbd444f-da22-4f23-a7a2-76508aba5f3a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:42.635845514Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1566,"":"downstairs","":"downstairs","session_id":"dfbd444f-da22-4f23-a7a2-76508aba5f3a"} {"msg":"[2]R flush_numbers[0..12]: [176, 156, 176, 176, 176, 173, 176, 177, 175, 175, 174, 177]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:42.635859433Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1566,"":"downstairs","":"downstairs","session_id":"dfbd444f-da22-4f23-a7a2-76508aba5f3a"} {"msg":"[2]R generation[0..12]: [94, 84, 94, 94, 94, 91, 94, 95, 93, 93, 92, 95]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:42.635872892Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1566,"":"downstairs","":"downstairs","session_id":"dfbd444f-da22-4f23-a7a2-76508aba5f3a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:42.635890251Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1566,"":"downstairs","":"downstairs","session_id":"dfbd444f-da22-4f23-a7a2-76508aba5f3a"} {"msg":"Max found gen is 96","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:42.63590372Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1566,"":"downstairs","":"downstairs","session_id":"dfbd444f-da22-4f23-a7a2-76508aba5f3a"} {"msg":"Generation requested: 96 >= found:96","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:42.635916989Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1566,"":"downstairs","":"downstairs","session_id":"dfbd444f-da22-4f23-a7a2-76508aba5f3a"} Wait for a query_work_queue command to finish before sending IO {"msg":"Next flush: 178","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:42.635935158Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1566,"":"downstairs","":"downstairs","session_id":"dfbd444f-da22-4f23-a7a2-76508aba5f3a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:42.635958886Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1566,"":"downstairs","":"downstairs","session_id":"dfbd444f-da22-4f23-a7a2-76508aba5f3a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:42.635985304Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1566,"session_id":"dfbd444f-da22-4f23-a7a2-76508aba5f3a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:42.63603809Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1566,"client":"0","":"downstairs","session_id":"dfbd444f-da22-4f23-a7a2-76508aba5f3a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:42.636067299Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1566,"client":"1","":"downstairs","session_id":"dfbd444f-da22-4f23-a7a2-76508aba5f3a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:42.636094717Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1566,"client":"2","":"downstairs","session_id":"dfbd444f-da22-4f23-a7a2-76508aba5f3a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:42.636123855Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1566,"session_id":"dfbd444f-da22-4f23-a7a2-76508aba5f3a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:42.636149993Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1566,"session_id":"dfbd444f-da22-4f23-a7a2-76508aba5f3a"} {"msg":"ea829a78-87fa-43b4-bae0-af28689dbce5 is now active with session: dfbd444f-da22-4f23-a7a2-76508aba5f3a","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:42.636176711Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1566,"session_id":"dfbd444f-da22-4f23-a7a2-76508aba5f3a"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 } {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:42.636207119Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1566,"session_id":"dfbd444f-da22-4f23-a7a2-76508aba5f3a"} {"msg":"The guest has finished waiting for activation with:96","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:42.636226428Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1566} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 145 len 8192 data: 9 10 02/30 Write block 524 len 4096 data: 17 03/30 Write block 226 len 4096 data: 10 04/30 Read block 64 len 8192 05/30 Write block 506 len 28672 data: 7 9 8 8 8 8 7 06/30 Write block 497 len 36864 data: 6 3 3 4 6 8 8 8 7 07/30 Flush 08/30 Write block 579 len 36864 data: 11 8 9 10 10 12 12 13 12 09/30 Read block 110 len 36864 10/30 Read block 13 len 36864 11/30 Write block 438 len 20480 data: 7 7 8 8 8 12/30 Write block 204 len 12288 data: 11 10 11 13/30 Read block 534 len 16384 14/30 Read block 470 len 16384 15/30 Flush 16/30 Write block 359 len 8192 data: 6 6 17/30 Read block 358 len 28672 18/30 Read block 99 len 4096 19/30 Flush 20/30 Write block 207 len 24576 data: 7 7 6 9 10 10 21/30 Write block 236 len 40960 data: 11 9 8 9 8 8 8 8 5 4 22/30 Flush 23/30 Flush 24/30 Write block 450 len 40960 data: 10 11 11 10 10 7 7 8 9 10 25/30 Write block 11 len 8192 data: 9 7 26/30 Read block 57 len 12288 27/30 Write block 186 len 40960 data: 9 9 10 10 10 10 10 10 8 7 28/30 Write block 574 len 28672 data: 7 7 6 6 8 12 9 29/30 Write block 344 len 8192 data: 10 11 30/30 Read block 534 len 16384 ---------------------------------------------------------------- Crucible gen:96 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 10 Done Done Done false 1024 Acked Write 2 Done Done Done false 1025 Acked Read 3 Done Done Done false 1026 Acked Write 10 Done Done Done false 1027 Acked Write 7 Done Done Done false 1028 Acked Write 2 Done Done Done false 1029 Acked Read 4 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 7 7 6 20 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:36:42.654957056Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1566} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 94 94 94 176 176 176 T T T 1 020-039 84 84 84 156 156 156 F F F 2 040-059 94 94 94 176 176 176 F F F 3 060-079 94 94 94 176 176 176 F F F 4 080-099 94 94 94 176 176 176 F F F 5 100-119 91 91 91 173 173 173 F F F 6 120-139 94 94 94 176 176 176 F F F 7 140-159 96 96 96 178 178 178 F F F 8 160-179 93 93 93 175 175 175 F F F 9 180-199 93 93 93 175 175 175 T T T 10 200-219 96 96 96 181 181 181 F F F 11 220-239 96 96 96 181 181 181 F F F 12 240-259 96 96 96 181 181 181 F F F 13 260-279 94 94 94 176 176 176 F F F 14 280-299 91 91 91 173 173 173 F F F 15 300-319 93 93 93 175 175 175 F F F 16 320-339 94 94 94 176 176 176 F F F 17 340-359 96 96 96 180 180 180 T T T 18 360-379 96 96 96 180 180 180 F F F 19 380-399 94 94 94 176 176 176 F F F 20 400-419 92 92 92 174 174 174 F F F 21 420-439 96 96 96 179 179 179 F F F 22 440-459 96 96 96 179 179 179 T T T 23 460-479 94 94 94 176 176 176 F F F 24 480-499 96 96 96 178 178 178 F F F 25 500-519 96 96 96 178 178 178 F F F 26 520-539 96 96 96 178 178 178 F F F 27 540-559 93 93 93 175 175 175 F F F 28 560-579 96 96 96 179 179 179 T T T 29 580-599 96 96 96 179 179 179 T T T Max gen: 96, Max flush: 181 A Difference in extent metadata is expected here On loop 48 Verifying data now No change after verify Loop: 48 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 97 97 97 182 182 182 F F F 1 020-039 84 84 84 156 156 156 F F F 2 040-059 94 94 94 176 176 176 F F F 3 060-079 94 94 94 176 176 176 F F F 4 080-099 94 94 94 176 176 176 F F F 5 100-119 91 91 91 173 173 173 F F F 6 120-139 94 94 94 176 176 176 F F F 7 140-159 96 96 96 178 178 178 F F F 8 160-179 93 93 93 175 175 175 F F F 9 180-199 97 97 97 182 182 182 F F F 10 200-219 96 96 96 181 181 181 F F F 11 220-239 96 96 96 181 181 181 F F F 12 240-259 96 96 96 181 181 181 F F F 13 260-279 94 94 94 176 176 176 F F F 14 280-299 91 91 91 173 173 173 F F F 15 300-319 93 93 93 175 175 175 F F F 16 320-339 94 94 94 176 176 176 F F F 17 340-359 97 97 97 182 182 182 F F F 18 360-379 96 96 96 180 180 180 F F F 19 380-399 94 94 94 176 176 176 F F F 20 400-419 92 92 92 174 174 174 F F F 21 420-439 96 96 96 179 179 179 F F F 22 440-459 97 97 97 182 182 182 F F F 23 460-479 94 94 94 176 176 176 F F F 24 480-499 96 96 96 178 178 178 F F F 25 500-519 96 96 96 178 178 178 F F F 26 520-539 96 96 96 178 178 178 F F F 27 540-559 93 93 93 175 175 175 F F F 28 560-579 97 97 97 182 182 182 F F F 29 580-599 97 97 97 182 182 182 F F F Max gen: 97, Max flush: 182 Begin loop 49 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:46.933624456Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1574,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:36:46.976376738Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1574,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:46.97678098Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1574,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:47.008545979Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1574,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:47.009142888Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1574} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"26e7d501b33f47379e53601720353c2983e7654b\",\n git_commit_timestamp: \"2024-12-12T20:16:40.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:47.009179276Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1574} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:47.009193575Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1574} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:47.009207964Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1574} {"msg":"Crucible 34a700ca-8cfb-45d2-a078-3957170ba235 has session id: 7c991c87-bfe1-45f4-949b-79781d18b039","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:47.009627515Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1574,"session_id":"7c991c87-bfe1-45f4-949b-79781d18b039"} {"msg":"Upstairs opts: Upstairs UUID: 34a700ca-8cfb-45d2-a078-3957170ba235, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:47.009672292Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1574,"session_id":"7c991c87-bfe1-45f4-949b-79781d18b039"} {"msg":"Crucible stats registered with UUID: 34a700ca-8cfb-45d2-a078-3957170ba235","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:47.00969692Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1574,"session_id":"7c991c87-bfe1-45f4-949b-79781d18b039"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:47.040415442Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1574,"task":"crutest"} {"msg":"The guest has requested activation with gen:98","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:47.0404419Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1574} {"msg":"34a700ca-8cfb-45d2-a078-3957170ba235 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:47.040525704Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1574,"session_id":"7c991c87-bfe1-45f4-949b-79781d18b039"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:47.040546583Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1574,"client":"0","":"downstairs","session_id":"7c991c87-bfe1-45f4-949b-79781d18b039"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:47.040578091Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1574,"client":"0","":"downstairs","session_id":"7c991c87-bfe1-45f4-949b-79781d18b039"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:47.040606409Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1574,"client":"1","":"downstairs","session_id":"7c991c87-bfe1-45f4-949b-79781d18b039"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:47.040635717Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1574,"client":"1","":"downstairs","session_id":"7c991c87-bfe1-45f4-949b-79781d18b039"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:47.040720501Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1574,"client":"2","":"downstairs","session_id":"7c991c87-bfe1-45f4-949b-79781d18b039"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:47.040753549Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1574,"client":"2","":"downstairs","session_id":"7c991c87-bfe1-45f4-949b-79781d18b039"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:47.040780057Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1574,"":"io task","client":"2","":"downstairs","session_id":"7c991c87-bfe1-45f4-949b-79781d18b039"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:47.040807655Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1574,"":"io task","client":"0","":"downstairs","session_id":"7c991c87-bfe1-45f4-949b-79781d18b039"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:47.040840743Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1574,"":"io task","client":"1","":"downstairs","session_id":"7c991c87-bfe1-45f4-949b-79781d18b039"} {"msg":"ds_connection connected from Ok(127.0.0.1:55843)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:47.04087203Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1574,"":"io task","client":"2","":"downstairs","session_id":"7c991c87-bfe1-45f4-949b-79781d18b039"} {"msg":"ds_connection connected from Ok(127.0.0.1:38916)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:47.040945715Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1574,"":"io task","client":"0","":"downstairs","session_id":"7c991c87-bfe1-45f4-949b-79781d18b039"} {"msg":"ds_connection connected from Ok(127.0.0.1:48440)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:47.041055798Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1574,"":"io task","client":"1","":"downstairs","session_id":"7c991c87-bfe1-45f4-949b-79781d18b039"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:47.041706943Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1574,"client":"2","":"downstairs","session_id":"7c991c87-bfe1-45f4-949b-79781d18b039"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:47.04174792Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1574,"client":"0","":"downstairs","session_id":"7c991c87-bfe1-45f4-949b-79781d18b039"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:47.041780518Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1574,"client":"1","":"downstairs","session_id":"7c991c87-bfe1-45f4-949b-79781d18b039"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:47.042380046Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1574,"client":"0","":"downstairs","session_id":"7c991c87-bfe1-45f4-949b-79781d18b039"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:47.042409735Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1574,"client":"2","":"downstairs","session_id":"7c991c87-bfe1-45f4-949b-79781d18b039"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:47.042432843Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1574,"client":"1","":"downstairs","session_id":"7c991c87-bfe1-45f4-949b-79781d18b039"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:47.042962936Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1574,"session_id":"7c991c87-bfe1-45f4-949b-79781d18b039"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:47.043002374Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1574,"session_id":"7c991c87-bfe1-45f4-949b-79781d18b039"} {"msg":"[0]R flush_numbers[0..12]: [182, 156, 176, 176, 176, 173, 176, 178, 175, 182, 181, 181]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:47.043019782Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1574,"":"downstairs","":"downstairs","session_id":"7c991c87-bfe1-45f4-949b-79781d18b039"} {"msg":"[0]R generation[0..12]: [97, 84, 94, 94, 94, 91, 94, 96, 93, 97, 96, 96]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:47.043034561Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1574,"":"downstairs","":"downstairs","session_id":"7c991c87-bfe1-45f4-949b-79781d18b039"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:47.04305281Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1574,"":"downstairs","":"downstairs","session_id":"7c991c87-bfe1-45f4-949b-79781d18b039"} {"msg":"[1]R flush_numbers[0..12]: [182, 156, 176, 176, 176, 173, 176, 178, 175, 182, 181, 181]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:47.043067349Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1574,"":"downstairs","":"downstairs","session_id":"7c991c87-bfe1-45f4-949b-79781d18b039"} {"msg":"[1]R generation[0..12]: [97, 84, 94, 94, 94, 91, 94, 96, 93, 97, 96, 96]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:47.043080958Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1574,"":"downstairs","":"downstairs","session_id":"7c991c87-bfe1-45f4-949b-79781d18b039"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:47.043095847Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1574,"":"downstairs","":"downstairs","session_id":"7c991c87-bfe1-45f4-949b-79781d18b039"} {"msg":"[2]R flush_numbers[0..12]: [182, 156, 176, 176, 176, 173, 176, 178, 175, 182, 181, 181]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:47.043109846Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1574,"":"downstairs","":"downstairs","session_id":"7c991c87-bfe1-45f4-949b-79781d18b039"} {"msg":"[2]R generation[0..12]: [97, 84, 94, 94, 94, 91, 94, 96, 93, 97, 96, 96]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:47.043123415Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1574,"":"downstairs","":"downstairs","session_id":"7c991c87-bfe1-45f4-949b-79781d18b039"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:47.043137294Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1574,"":"downstairs","":"downstairs","session_id":"7c991c87-bfe1-45f4-949b-79781d18b039"} {"msg":"Max found gen is 98","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:47.043150313Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1574,"":"downstairs","":"downstairs","session_id":"7c991c87-bfe1-45f4-949b-79781d18b039"} {"msg":"Generation requested: 98 >= found:98","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:47.043163732Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1574,"":"downstairs","":"downstairs","session_id":"7c991c87-bfe1-45f4-949b-79781d18b039"} {"msg":"Next flush: 183","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:47.043176622Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1574,"":"downstairs","":"downstairs","session_id":"7c991c87-bfe1-45f4-949b-79781d18b039"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:47.043189651Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1574,"":"downstairs","":"downstairs","session_id":"7c991c87-bfe1-45f4-949b-79781d18b039"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:47.04320283Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1574,"session_id":"7c991c87-bfe1-45f4-949b-79781d18b039"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:47.043214909Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1574,"client":"0","":"downstairs","session_id":"7c991c87-bfe1-45f4-949b-79781d18b039"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:47.043228138Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1574,"client":"1","":"downstairs","session_id":"7c991c87-bfe1-45f4-949b-79781d18b039"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO 2024-12-12T20:36:47.043241187Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1574,"client":"2","":"downstairs","session_id":"7c991c87-bfe1-45f4-949b-79781d18b039"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:47.043262286Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1574,"session_id":"7c991c87-bfe1-45f4-949b-79781d18b039"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:47.043274555Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1574,"session_id":"7c991c87-bfe1-45f4-949b-79781d18b039"} {"msg":"34a700ca-8cfb-45d2-a078-3957170ba235 is now active with session: 7c991c87-bfe1-45f4-949b-79781d18b039","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:47.043287384Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1574,"session_id":"7c991c87-bfe1-45f4-949b-79781d18b039"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:47.043299363Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1574,"session_id":"7c991c87-bfe1-45f4-949b-79781d18b039"} {"msg":"The guest has finished waiting for activation with:98","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:47.043311432Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1574} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 185 len 20480 data: 11 10 10 11 11 02/30 Read block 390 len 16384 03/30 Flush 04/30 Flush 05/30 Write block 149 len 36864 data: 10 7 6 7 8 7 7 8 8 06/30 Write block 329 len 40960 data: 8 9 10 7 8 9 8 6 6 7 07/30 Read block 98 len 24576 08/30 Write block 22 len 32768 data: 7 6 7 5 5 6 6 5 09/30 Read block 134 len 20480 10/30 Read block 558 len 28672 11/30 Write block 536 len 36864 data: 8 8 6 6 10 6 8 7 8 12/30 Write block 138 len 16384 data: 9 7 6 5 13/30 Write block 29 len 36864 data: 6 6 6 7 9 9 11 10 10 14/30 Read block 90 len 24576 15/30 Write block 254 len 28672 data: 9 9 8 6 7 8 8 16/30 Read block 544 len 12288 17/30 Write block 420 len 24576 data: 9 10 11 10 11 10 18/30 Read block 358 len 4096 19/30 Read block 19 len 4096 20/30 Read block 322 len 4096 21/30 Write block 562 len 20480 data: 8 7 7 7 8 22/30 Flush 23/30 Write block 453 len 36864 data: 11 11 8 8 9 10 11 10 12 24/30 Write block 395 len 28672 data: 9 9 10 11 12 10 11 25/30 Write block 249 len 28672 data: 4 4 5 6 9 10 10 26/30 Write block 365 len 12288 data: 10 11 10 27/30 Read block 338 len 4096 28/30 Write block 395 len 28672 data: 10 10 11 12 13 11 12 29/30 Write block 75 len 16384 data: 8 6 9 9 30/30 Write block 448 len 20480 data: 9 11 11 12 12 ---------------------------------------------------------------- Crucible gen:98 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 9 Done Done Done false 1023 Acked Write 7 Done Done Done false 1024 Acked Write 7 Done Done Done false 1025 Acked Write 3 Done Done Done false 1026 Acked Read 1 Done Done Done false 1027 Acked Write 7 Sent Sent Sent false 1028 Acked Write 4 Sent Sent Sent false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 3 3 9 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1021 1021 1021 Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:36:47.06276894Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1574} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 97 97 97 182 182 182 F F F 1 020-039 98 98 98 185 185 185 F F F 2 040-059 94 94 94 176 176 176 F F F 3 060-079 94 94 94 176 176 176 T T T 4 080-099 94 94 94 176 176 176 F F F 5 100-119 91 91 91 173 173 173 F F F 6 120-139 98 98 98 185 185 185 F F F 7 140-159 98 98 98 185 185 185 F F F 8 160-179 93 93 93 175 175 175 F F F 9 180-199 98 98 98 183 183 183 F F F 10 200-219 96 96 96 181 181 181 F F F 11 220-239 96 96 96 181 181 181 F F F 12 240-259 98 98 98 185 185 185 T T T 13 260-279 98 98 98 185 185 185 F F F 14 280-299 91 91 91 173 173 173 F F F 15 300-319 93 93 93 175 175 175 F F F 16 320-339 98 98 98 185 185 185 F F F 17 340-359 97 97 97 182 182 182 F F F 18 360-379 96 96 96 180 180 180 T T T 19 380-399 94 94 94 176 176 176 T T T 20 400-419 92 92 92 174 174 174 T T T 21 420-439 98 98 98 185 185 185 F F F 22 440-459 97 97 97 182 182 182 T T T 23 460-479 94 94 94 176 176 176 T T T 24 480-499 96 96 96 178 178 178 F F F 25 500-519 96 96 96 178 178 178 F F F 26 520-539 98 98 98 185 185 185 F F F 27 540-559 98 98 98 185 185 185 F F F 28 560-579 98 98 98 185 185 185 F F F 29 580-599 97 97 97 182 182 182 F F F Max gen: 98, Max flush: 185 A Difference in extent metadata is expected here On loop 49 Verifying data now No change after verify Loop: 49 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 97 97 97 182 182 182 F F F 1 020-039 98 98 98 185 185 185 F F F 2 040-059 94 94 94 176 176 176 F F F 3 060-079 99 99 99 186 186 186 F F F 4 080-099 94 94 94 176 176 176 F F F 5 100-119 91 91 91 173 173 173 F F F 6 120-139 98 98 98 185 185 185 F F F 7 140-159 98 98 98 185 185 185 F F F 8 160-179 93 93 93 175 175 175 F F F 9 180-199 98 98 98 183 183 183 F F F 10 200-219 96 96 96 181 181 181 F F F 11 220-239 96 96 96 181 181 181 F F F 12 240-259 99 99 99 186 186 186 F F F 13 260-279 98 98 98 185 185 185 F F F 14 280-299 91 91 91 173 173 173 F F F 15 300-319 93 93 93 175 175 175 F F F 16 320-339 98 98 98 185 185 185 F F F 17 340-359 97 97 97 182 182 182 F F F 18 360-379 99 99 99 186 186 186 F F F 19 380-399 99 99 99 186 186 186 F F F 20 400-419 99 99 99 186 186 186 F F F 21 420-439 98 98 98 185 185 185 F F F 22 440-459 99 99 99 186 186 186 F F F 23 460-479 99 99 99 186 186 186 F F F 24 480-499 96 96 96 178 178 178 F F F 25 500-519 96 96 96 178 178 178 F F F 26 520-539 98 98 98 185 185 185 F F F 27 540-559 98 98 98 185 185 185 F F F 28 560-579 98 98 98 185 185 185 F F F 29 580-599 97 97 97 182 182 182 F F F Max gen: 99, Max flush: 186 Begin loop 50 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:51.37188226Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1582,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:36:51.415635434Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1582,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:51.416055374Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1582,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:51.448272673Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1582,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:51.448860312Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1582} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"26e7d501b33f47379e53601720353c2983e7654b\",\n git_commit_timestamp: \"2024-12-12T20:16:40.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:51.44889668Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1582} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:51.448910849Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1582} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:51.448925038Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1582} {"msg":"Crucible 6e5da188-9265-4d1d-bbc2-6c43473c3b22 has session id: 35f889d0-1fd7-40fe-b227-422c2986cc1b","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:51.449366748Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1582,"session_id":"35f889d0-1fd7-40fe-b227-422c2986cc1b"} {"msg":"Upstairs opts: Upstairs UUID: 6e5da188-9265-4d1d-bbc2-6c43473c3b22, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:51.449415684Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1582,"session_id":"35f889d0-1fd7-40fe-b227-422c2986cc1b"} {"msg":"Crucible stats registered with UUID: 6e5da188-9265-4d1d-bbc2-6c43473c3b22","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:51.449442672Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1582,"session_id":"35f889d0-1fd7-40fe-b227-422c2986cc1b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:51.480740704Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1582,"task":"crutest"} {"msg":"The guest has requested activation with gen:100","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:51.480784311Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1582} {"msg":"6e5da188-9265-4d1d-bbc2-6c43473c3b22 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:51.480851806Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1582,"session_id":"35f889d0-1fd7-40fe-b227-422c2986cc1b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:51.480890434Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1582,"client":"0","":"downstairs","session_id":"35f889d0-1fd7-40fe-b227-422c2986cc1b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:51.480938431Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1582,"client":"0","":"downstairs","session_id":"35f889d0-1fd7-40fe-b227-422c2986cc1b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:51.480978678Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1582,"client":"1","":"downstairs","session_id":"35f889d0-1fd7-40fe-b227-422c2986cc1b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:51.481017625Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1582,"client":"1","":"downstairs","session_id":"35f889d0-1fd7-40fe-b227-422c2986cc1b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:51.481052303Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1582,"client":"2","":"downstairs","session_id":"35f889d0-1fd7-40fe-b227-422c2986cc1b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:51.48108886Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1582,"client":"2","":"downstairs","session_id":"35f889d0-1fd7-40fe-b227-422c2986cc1b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:51.481122818Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1582,"":"io task","client":"2","":"downstairs","session_id":"35f889d0-1fd7-40fe-b227-422c2986cc1b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:51.481153066Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1582,"":"io task","client":"0","":"downstairs","session_id":"35f889d0-1fd7-40fe-b227-422c2986cc1b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:51.481290136Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1582,"":"io task","client":"1","":"downstairs","session_id":"35f889d0-1fd7-40fe-b227-422c2986cc1b"} {"msg":"ds_connection connected from Ok(127.0.0.1:42470)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:51.481330324Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1582,"":"io task","client":"2","":"downstairs","session_id":"35f889d0-1fd7-40fe-b227-422c2986cc1b"} {"msg":"ds_connection connected from Ok(127.0.0.1:48110)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:51.481365421Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1582,"":"io task","client":"0","":"downstairs","session_id":"35f889d0-1fd7-40fe-b227-422c2986cc1b"} {"msg":"ds_connection connected from Ok(127.0.0.1:41318)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:51.481399769Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1582,"":"io task","client":"1","":"downstairs","session_id":"35f889d0-1fd7-40fe-b227-422c2986cc1b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:51.481854187Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1582,"client":"1","":"downstairs","session_id":"35f889d0-1fd7-40fe-b227-422c2986cc1b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:51.481889725Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1582,"client":"2","":"downstairs","session_id":"35f889d0-1fd7-40fe-b227-422c2986cc1b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:51.482013026Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1582,"client":"0","":"downstairs","session_id":"35f889d0-1fd7-40fe-b227-422c2986cc1b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:51.482460155Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1582,"client":"1","":"downstairs","session_id":"35f889d0-1fd7-40fe-b227-422c2986cc1b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:51.482488304Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1582,"client":"2","":"downstairs","session_id":"35f889d0-1fd7-40fe-b227-422c2986cc1b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:51.482580367Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1582,"client":"0","":"downstairs","session_id":"35f889d0-1fd7-40fe-b227-422c2986cc1b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:51.482995059Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1582,"session_id":"35f889d0-1fd7-40fe-b227-422c2986cc1b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:51.483104991Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1582,"session_id":"35f889d0-1fd7-40fe-b227-422c2986cc1b"} {"msg":"[0]R flush_numbers[0..12]: [182, 185, 176, 186, 176, 173, 185, 185, 175, 183, 181, 181]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:51.483227882Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1582,"":"downstairs","":"downstairs","session_id":"35f889d0-1fd7-40fe-b227-422c2986cc1b"} {"msg":"[0]R generation[0..12]: [97, 98, 94, 99, 94, 91, 98, 98, 93, 98, 96, 96]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:51.483246931Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1582,"":"downstairs","":"downstairs","session_id":"35f889d0-1fd7-40fe-b227-422c2986cc1b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:51.48326155Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1582,"":"downstairs","":"downstairs","session_id":"35f889d0-1fd7-40fe-b227-422c2986cc1b"} {"msg":"[1]R flush_numbers[0..12]: [182, 185, 176, 186, 176, 173, 185, 185, 175, 183, 181, 181]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:51.483275429Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1582,"":"downstairs","":"downstairs","session_id":"35f889d0-1fd7-40fe-b227-422c2986cc1b"} {"msg":"[1]R generation[0..12]: [97, 98, 94, 99, 94, 91, 98, 98, 93, 98, 96, 96]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:51.483288968Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1582,"":"downstairs","":"downstairs","session_id":"35f889d0-1fd7-40fe-b227-422c2986cc1b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:51.483303067Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1582,"":"downstairs","":"downstairs","session_id":"35f889d0-1fd7-40fe-b227-422c2986cc1b"} {"msg":"[2]R flush_numbers[0..12]: [182, 185, 176, 186, 176, 173, 185, 185, 175, 183, 181, 181]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:51.483316996Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1582,"":"downstairs","":"downstairs","session_id":"35f889d0-1fd7-40fe-b227-422c2986cc1b"} {"msg":"[2]R generation[0..12]: [97, 98, 94, 99, 94, 91, 98, 98, 93, 98, 96, 96]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:51.483330706Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1582,"":"downstairs","":"downstairs","session_id":"35f889d0-1fd7-40fe-b227-422c2986cc1b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:51.483344414Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1582,"":"downstairs","":"downstairs","session_id":"35f889d0-1fd7-40fe-b227-422c2986cc1b"} {"msg":"Max found gen is 100","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:51.483357424Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1582,"":"downstairs","":"downstairs","session_id":"35f889d0-1fd7-40fe-b227-422c2986cc1b"} {"msg":"Generation requested: 100 >= found:100","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:51.483374292Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1582,"":"downstairs","":"downstairs","session_id":"35f889d0-1fd7-40fe-b227-422c2986cc1b"} {"msg":"Next flush: 187","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:51.483387591Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1582,"":"downstairs","":"downstairs","session_id":"35f889d0-1fd7-40fe-b227-422c2986cc1b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:51.483400511Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1582,"":"downstairs","":"downstairs","session_id":"35f889d0-1fd7-40fe-b227-422c2986cc1b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:51.48341373Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1582,"session_id":"35f889d0-1fd7-40fe-b227-422c2986cc1b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-12-12T20:36:51.483426459Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1582,"client":"0","":"downstairs","session_id":"35f889d0-1fd7-40fe-b227-422c2986cc1b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:51.483448127Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1582,"client":"1","":"downstairs","session_id":"35f889d0-1fd7-40fe-b227-422c2986cc1b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:51.483461577Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1582,"client":"2","":"downstairs","session_id":"35f889d0-1fd7-40fe-b227-422c2986cc1b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:51.483475025Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1582,"session_id":"35f889d0-1fd7-40fe-b227-422c2986cc1b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:51.483487565Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1582,"session_id":"35f889d0-1fd7-40fe-b227-422c2986cc1b"} {"msg":"6e5da188-9265-4d1d-bbc2-6c43473c3b22 is now active with session: 35f889d0-1fd7-40fe-b227-422c2986cc1b","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:51.483500174Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1582,"session_id":"35f889d0-1fd7-40fe-b227-422c2986cc1b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:51.483512063Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1582,"session_id":"35f889d0-1fd7-40fe-b227-422c2986cc1b"} {"msg":"The guest has finished waiting for activation with:100","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:51.483524322Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1582} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 99 len 8192 data: 5 7 02/30 Read block 248 len 16384 03/30 Write block 458 len 32768 data: 11 12 11 13 12 10 8 7 04/30 Read block 223 len 40960 05/30 Read block 356 len 32768 06/30 Write block 56 len 36864 data: 13 14 14 14 13 8 8 9 8 07/30 Read block 303 len 4096 08/30 Read block 215 len 36864 09/30 Flush 10/30 Write block 475 len 32768 data: 8 8 7 8 7 5 6 5 11/30 Read block 368 len 36864 12/30 Read block 47 len 20480 13/30 Flush 14/30 Write block 205 len 4096 data: 11 15/30 Write block 264 len 20480 data: 8 11 12 12 11 16/30 Write block 241 len 8192 data: 9 9 17/30 Read block 385 len 4096 18/30 Write block 462 len 20480 data: 13 11 9 8 7 19/30 Read block 535 len 16384 20/30 Write block 479 len 36864 data: 8 6 7 6 6 10 11 10 9 21/30 Read block 194 len 40960 22/30 Write block 274 len 28672 data: 8 7 6 7 9 8 7 23/30 Read block 496 len 28672 24/30 Read block 304 len 24576 25/30 Read block 368 len 8192 26/30 Read block 300 len 4096 27/30 Write block 143 len 32768 data: 8 8 10 11 11 12 11 8 28/30 Write block 446 len 16384 data: 9 11 10 12 29/30 Write block 340 len 20480 data: 5 4 5 7 11 30/30 Write block 138 len 4096 data: 10 ---------------------------------------------------------------- Crucible gen:100 GIO:true work queues: Upstairs:0 downstairs:17 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1013 Acked Write 1 Done Done Done false 1014 Acked Write 5 Done Done Done false 1015 Acked Write 2 Done Done Done false 1016 Acked Read 1 Done Done Done false 1017 Acked Write 5 Done Done Done false 1018 Acked Read 4 Done Done Done false 1019 Acked Write 9 Done Done Done false 1020 Acked Read 10 Done Done Done false 1021 Acked Write 7 Done Done Done false 1022 Acked Read 7 Done Done Done false 1023 Acked Read 6 Done Done Done false 1024 Acked Read 2 Done Done Done false 1025 Acked Read 1 Done Done Done false 1026 Acked Write 8 Done Done Done false 1027 Acked Write 4 Sent Sent Sent false 1028 Acked Write 5 Sent Sent Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 3 3 9 Done 14 14 14 42 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1012 1012 1012 Downstairs last five completed: 1012 1011 1010 1009 1008 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:36:51.502864408Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1582} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 97 97 97 182 182 182 F F F 1 020-039 98 98 98 185 185 185 F F F 2 040-059 100 100 100 187 187 187 F F F 3 060-079 100 100 100 187 187 187 F F F 4 080-099 100 100 100 187 187 187 F F F 5 100-119 100 100 100 187 187 187 F F F 6 120-139 98 98 98 185 185 185 T T T 7 140-159 98 98 98 185 185 185 T T T 8 160-179 93 93 93 175 175 175 F F F 9 180-199 98 98 98 183 183 183 F F F 10 200-219 96 96 96 181 181 181 T T T 11 220-239 96 96 96 181 181 181 F F F 12 240-259 99 99 99 186 186 186 T T T 13 260-279 98 98 98 185 185 185 T T T 14 280-299 91 91 91 173 173 173 T T T 15 300-319 93 93 93 175 175 175 F F F 16 320-339 98 98 98 185 185 185 F F F 17 340-359 97 97 97 182 182 182 T T T 18 360-379 99 99 99 186 186 186 F F F 19 380-399 99 99 99 186 186 186 F F F 20 400-419 99 99 99 186 186 186 F F F 21 420-439 98 98 98 185 185 185 F F F 22 440-459 100 100 100 187 187 187 T T T 23 460-479 100 100 100 188 188 188 T T T 24 480-499 100 100 100 188 188 188 T T T 25 500-519 96 96 96 178 178 178 F F F 26 520-539 98 98 98 185 185 185 F F F 27 540-559 98 98 98 185 185 185 F F F 28 560-579 98 98 98 185 185 185 F F F 29 580-599 97 97 97 182 182 182 F F F Max gen: 100, Max flush: 188 A Difference in extent metadata is expected here On loop 50 Verifying data now No change after verify Loop: 50 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 97 97 97 182 182 182 F F F 1 020-039 98 98 98 185 185 185 F F F 2 040-059 100 100 100 187 187 187 F F F 3 060-079 100 100 100 187 187 187 F F F 4 080-099 100 100 100 187 187 187 F F F 5 100-119 100 100 100 187 187 187 F F F 6 120-139 101 101 101 189 189 189 F F F 7 140-159 101 101 101 189 189 189 F F F 8 160-179 93 93 93 175 175 175 F F F 9 180-199 98 98 98 183 183 183 F F F 10 200-219 101 101 101 189 189 189 F F F 11 220-239 96 96 96 181 181 181 F F F 12 240-259 101 101 101 189 189 189 F F F 13 260-279 101 101 101 189 189 189 F F F 14 280-299 101 101 101 189 189 189 F F F 15 300-319 93 93 93 175 175 175 F F F 16 320-339 98 98 98 185 185 185 F F F 17 340-359 101 101 101 189 189 189 F F F 18 360-379 99 99 99 186 186 186 F F F 19 380-399 99 99 99 186 186 186 F F F 20 400-419 99 99 99 186 186 186 F F F 21 420-439 98 98 98 185 185 185 F F F 22 440-459 101 101 101 189 189 189 F F F 23 460-479 101 101 101 189 189 189 F F F 24 480-499 101 101 101 189 189 189 F F F 25 500-519 96 96 96 178 178 178 F F F 26 520-539 98 98 98 185 185 185 F F F 27 540-559 98 98 98 185 185 185 F F F 28 560-579 98 98 98 185 185 185 F F F 29 580-599 97 97 97 182 182 182 F F F Max gen: 101, Max flush: 189 Begin loop 51 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:55.832173567Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1590,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:55.876668008Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1590,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:55.877420147Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1590} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"26e7d501b33f47379e53601720353c2983e7654b\",\n git_commit_timestamp: \"2024-12-12T20:16:40.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:55.877456784Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1590} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:55.877471583Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1590} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:55.877486032Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1590} {"msg":"Crucible 331aa93f-e1b6-4d2a-9036-692590a15f1b has session id: c7cb75e6-6a08-4972-a4d8-f31257f41d55","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:55.877935581Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1590,"session_id":"c7cb75e6-6a08-4972-a4d8-f31257f41d55"} {"msg":"Upstairs opts: Upstairs UUID: 331aa93f-e1b6-4d2a-9036-692590a15f1b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:55.877974749Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1590,"session_id":"c7cb75e6-6a08-4972-a4d8-f31257f41d55"} {"msg":"Crucible stats registered with UUID: 331aa93f-e1b6-4d2a-9036-692590a15f1b","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:55.877995527Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1590,"session_id":"c7cb75e6-6a08-4972-a4d8-f31257f41d55"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:55.909226614Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1590,"task":"crutest"} {"msg":"The guest has requested activation with gen:102","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:55.90927104Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1590} {"msg":"331aa93f-e1b6-4d2a-9036-692590a15f1b active request set","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:55.909334346Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1590,"session_id":"c7cb75e6-6a08-4972-a4d8-f31257f41d55"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:55.909362514Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1590,"client":"0","":"downstairs","session_id":"c7cb75e6-6a08-4972-a4d8-f31257f41d55"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:55.909385912Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1590,"client":"0","":"downstairs","session_id":"c7cb75e6-6a08-4972-a4d8-f31257f41d55"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:55.909400482Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1590,"client":"1","":"downstairs","session_id":"c7cb75e6-6a08-4972-a4d8-f31257f41d55"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:55.909415041Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1590,"client":"1","":"downstairs","session_id":"c7cb75e6-6a08-4972-a4d8-f31257f41d55"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:55.90942836Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1590,"client":"2","":"downstairs","session_id":"c7cb75e6-6a08-4972-a4d8-f31257f41d55"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:55.909442408Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1590,"client":"2","":"downstairs","session_id":"c7cb75e6-6a08-4972-a4d8-f31257f41d55"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:55.909455508Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1590,"":"io task","client":"0","":"downstairs","session_id":"c7cb75e6-6a08-4972-a4d8-f31257f41d55"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:55.909470337Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1590,"":"io task","client":"2","":"downstairs","session_id":"c7cb75e6-6a08-4972-a4d8-f31257f41d55"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:55.909484026Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1590,"":"io task","client":"1","":"downstairs","session_id":"c7cb75e6-6a08-4972-a4d8-f31257f41d55"} {"msg":"ds_connection connected from Ok(127.0.0.1:34011)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:55.909621116Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1590,"":"io task","client":"0","":"downstairs","session_id":"c7cb75e6-6a08-4972-a4d8-f31257f41d55"} {"msg":"ds_connection connected from Ok(127.0.0.1:40833)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:55.909878578Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1590,"":"io task","client":"2","":"downstairs","session_id":"c7cb75e6-6a08-4972-a4d8-f31257f41d55"} {"msg":"ds_connection connected from Ok(127.0.0.1:39903)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:55.909916606Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1590,"":"io task","client":"1","":"downstairs","session_id":"c7cb75e6-6a08-4972-a4d8-f31257f41d55"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:55.910406192Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1590,"client":"1","":"downstairs","session_id":"c7cb75e6-6a08-4972-a4d8-f31257f41d55"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:55.910448459Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1590,"client":"0","":"downstairs","session_id":"c7cb75e6-6a08-4972-a4d8-f31257f41d55"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:55.910522024Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1590,"client":"2","":"downstairs","session_id":"c7cb75e6-6a08-4972-a4d8-f31257f41d55"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:55.911117183Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1590,"client":"1","":"downstairs","session_id":"c7cb75e6-6a08-4972-a4d8-f31257f41d55"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:55.911147871Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1590,"client":"2","":"downstairs","session_id":"c7cb75e6-6a08-4972-a4d8-f31257f41d55"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:55.91116498Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1590,"client":"0","":"downstairs","session_id":"c7cb75e6-6a08-4972-a4d8-f31257f41d55"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:55.911705183Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1590,"session_id":"c7cb75e6-6a08-4972-a4d8-f31257f41d55"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:55.911804176Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1590,"session_id":"c7cb75e6-6a08-4972-a4d8-f31257f41d55"} {"msg":"[0]R flush_numbers[0..12]: [182, 185, 187, 187, 187, 187, 189, 189, 175, 183, 189, 181]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:55.911876631Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1590,"":"downstairs","":"downstairs","session_id":"c7cb75e6-6a08-4972-a4d8-f31257f41d55"} {"msg":"[0]R generation[0..12]: [97, 98, 100, 100, 100, 100, 101, 101, 93, 98, 101, 96]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:55.911896299Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1590,"":"downstairs","":"downstairs","session_id":"c7cb75e6-6a08-4972-a4d8-f31257f41d55"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:55.911911178Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1590,"":"downstairs","":"downstairs","session_id":"c7cb75e6-6a08-4972-a4d8-f31257f41d55"} {"msg":"[1]R flush_numbers[0..12]: [182, 185, 187, 187, 187, 187, 189, 189, 175, 183, 189, 181]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:55.911925267Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1590,"":"downstairs","":"downstairs","session_id":"c7cb75e6-6a08-4972-a4d8-f31257f41d55"} {"msg":"[1]R generation[0..12]: [97, 98, 100, 100, 100, 100, 101, 101, 93, 98, 101, 96]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:55.911943056Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1590,"":"downstairs","":"downstairs","session_id":"c7cb75e6-6a08-4972-a4d8-f31257f41d55"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:55.911957415Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1590,"":"downstairs","":"downstairs","session_id":"c7cb75e6-6a08-4972-a4d8-f31257f41d55"} {"msg":"[2]R flush_numbers[0..12]: [182, 185, 187, 187, 187, 187, 189, 189, 175, 183, 189, 181]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:55.911971064Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1590,"":"downstairs","":"downstairs","session_id":"c7cb75e6-6a08-4972-a4d8-f31257f41d55"} {"msg":"[2]R generation[0..12]: [97, 98, 100, 100, 100, 100, 101, 101, 93, 98, 101, 96]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:55.911984653Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1590,"":"downstairs","":"downstairs","session_id":"c7cb75e6-6a08-4972-a4d8-f31257f41d55"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:55.911998562Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1590,"":"downstairs","":"downstairs","session_id":"c7cb75e6-6a08-4972-a4d8-f31257f41d55"} {"msg":"Max found gen is 102","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:55.912011621Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1590,"":"downstairs","":"downstairs","session_id":"c7cb75e6-6a08-4972-a4d8-f31257f41d55"} {"msg":"Generation requested: 102 >= found:102","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:55.912025061Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1590,"":"downstairs","":"downstairs","session_id":"c7cb75e6-6a08-4972-a4d8-f31257f41d55"} {"msg":"Next flush: 190","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:55.912038129Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1590,"":"downstairs","":"downstairs","session_id":"c7cb75e6-6a08-4972-a4d8-f31257f41d55"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:55.912051029Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1590,"":"downstairs","":"downstairs","session_id":"c7cb75e6-6a08-4972-a4d8-f31257f41d55"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:55.912064308Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1590,"session_id":"c7cb75e6-6a08-4972-a4d8-f31257f41d55"} Wait for a query_work_queue command to finish before sending IO {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:55.912081337Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1590,"client":"0","":"downstairs","session_id":"c7cb75e6-6a08-4972-a4d8-f31257f41d55"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:55.912094666Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1590,"client":"1","":"downstairs","session_id":"c7cb75e6-6a08-4972-a4d8-f31257f41d55"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:55.912107745Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1590,"client":"2","":"downstairs","session_id":"c7cb75e6-6a08-4972-a4d8-f31257f41d55"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:55.912120794Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1590,"session_id":"c7cb75e6-6a08-4972-a4d8-f31257f41d55"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:55.912132873Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1590,"session_id":"c7cb75e6-6a08-4972-a4d8-f31257f41d55"} {"msg":"331aa93f-e1b6-4d2a-9036-692590a15f1b is now active with session: c7cb75e6-6a08-4972-a4d8-f31257f41d55","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:55.912147262Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1590,"session_id":"c7cb75e6-6a08-4972-a4d8-f31257f41d55"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:55.912159371Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1590,"session_id":"c7cb75e6-6a08-4972-a4d8-f31257f41d55"} {"msg":"The guest has finished waiting for activation with:102","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:36:55.91217155Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1590} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 365 len 16384 data: 11 12 11 7 02/30 Read block 360 len 24576 03/30 Write block 571 len 24576 data: 7 7 7 8 8 7 04/30 Flush 05/30 Write block 435 len 16384 data: 9 8 7 8 06/30 Read block 43 len 8192 07/30 Write block 340 len 20480 data: 6 5 6 8 12 08/30 Write block 566 len 24576 data: 9 6 7 8 6 8 09/30 Read block 140 len 12288 10/30 Read block 311 len 4096 11/30 Read block 146 len 12288 12/30 Read block 452 len 4096 13/30 Write block 131 len 32768 data: 7 5 5 6 6 8 9 11 14/30 Read block 169 len 28672 15/30 Read block 63 len 12288 16/30 Read block 86 len 24576 17/30 Read block 387 len 12288 18/30 Write block 170 len 8192 data: 4 3 19/30 Write block 218 len 12288 data: 7 9 9 20/30 Flush 21/30 Write block 283 len 8192 data: 6 6 22/30 Write block 512 len 40960 data: 8 7 6 6 7 9 13 16 17 16 23/30 Read block 88 len 20480 24/30 Read block 72 len 12288 25/30 Read block 364 len 8192 26/30 Read block 439 len 4096 27/30 Read block 312 len 40960 28/30 Read block 169 len 32768 29/30 Write block 264 len 24576 data: 9 12 13 13 12 6 30/30 Read block 254 len 12288 ---------------------------------------------------------------- Crucible gen:102 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1020 Acked Write 2 Done Done Done false 1021 Acked Write 10 Done Done Done false 1022 Acked Read 5 Done Done Done false 1023 Acked Read 3 Done Done Done false 1024 Acked Read 2 Done Done Done false 1025 Acked Read 1 Done Done Done false 1026 Acked Read 10 Done Done Done false 1027 Acked Read 8 Done Done Done false 1028 Acked Write 6 Done Done Sent false 1029 Acked Read 3 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 2 3 Done 10 9 8 27 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1019 1019 1019 Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:36:55.931805317Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1590} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 97 97 97 182 182 182 F F F 1 020-039 98 98 98 185 185 185 F F F 2 040-059 100 100 100 187 187 187 F F F 3 060-079 100 100 100 187 187 187 F F F 4 080-099 100 100 100 187 187 187 F F F 5 100-119 100 100 100 187 187 187 F F F 6 120-139 102 102 102 191 191 191 F F F 7 140-159 101 101 101 189 189 189 F F F 8 160-179 102 102 102 191 191 191 F F F 9 180-199 98 98 98 183 183 183 F F F 10 200-219 102 102 102 191 191 191 F F F 11 220-239 102 102 102 191 191 191 F F F 12 240-259 101 101 101 189 189 189 F F F 13 260-279 101 101 101 189 189 189 T T T 14 280-299 101 101 101 189 189 189 T T T 15 300-319 93 93 93 175 175 175 F F F 16 320-339 98 98 98 185 185 185 F F F 17 340-359 102 102 102 191 191 191 F F F 18 360-379 102 102 102 190 190 190 F F F 19 380-399 99 99 99 186 186 186 F F F 20 400-419 99 99 99 186 186 186 F F F 21 420-439 102 102 102 191 191 191 F F F 22 440-459 101 101 101 189 189 189 F F F 23 460-479 101 101 101 189 189 189 F F F 24 480-499 101 101 101 189 189 189 F F F 25 500-519 96 96 96 178 178 178 T T T 26 520-539 98 98 98 185 185 185 T T T 27 540-559 98 98 98 185 185 185 F F F 28 560-579 102 102 102 191 191 191 F F F 29 580-599 97 97 97 182 182 182 F F F Max gen: 102, Max flush: 191 A Difference in extent metadata is expected here On loop 51 Verifying data now No change after verify Loop: 51 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 97 97 97 182 182 182 F F F 1 020-039 98 98 98 185 185 185 F F F 2 040-059 100 100 100 187 187 187 F F F 3 060-079 100 100 100 187 187 187 F F F 4 080-099 100 100 100 187 187 187 F F F 5 100-119 100 100 100 187 187 187 F F F 6 120-139 102 102 102 191 191 191 F F F 7 140-159 101 101 101 189 189 189 F F F 8 160-179 102 102 102 191 191 191 F F F 9 180-199 98 98 98 183 183 183 F F F 10 200-219 102 102 102 191 191 191 F F F 11 220-239 102 102 102 191 191 191 F F F 12 240-259 101 101 101 189 189 189 F F F 13 260-279 103 103 103 192 192 192 F F F 14 280-299 103 103 103 192 192 192 F F F 15 300-319 93 93 93 175 175 175 F F F 16 320-339 98 98 98 185 185 185 F F F 17 340-359 102 102 102 191 191 191 F F F 18 360-379 102 102 102 190 190 190 F F F 19 380-399 99 99 99 186 186 186 F F F 20 400-419 99 99 99 186 186 186 F F F 21 420-439 102 102 102 191 191 191 F F F 22 440-459 101 101 101 189 189 189 F F F 23 460-479 101 101 101 189 189 189 F F F 24 480-499 101 101 101 189 189 189 F F F 25 500-519 103 103 103 192 192 192 F F F 26 520-539 103 103 103 192 192 192 F F F 27 540-559 98 98 98 185 185 185 F F F 28 560-579 102 102 102 191 191 191 F F F 29 580-599 97 97 97 182 182 182 F F F Max gen: 103, Max flush: 192 Begin loop 52 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:00.196249958Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1598,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:00.240628637Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1598,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:00.241275843Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1598} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"26e7d501b33f47379e53601720353c2983e7654b\",\n git_commit_timestamp: \"2024-12-12T20:16:40.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:00.24131183Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1598} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:00.241330829Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1598} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:00.241345968Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1598} {"msg":"Crucible 0fe0a310-a461-407e-8ec1-254a236f0dc3 has session id: aaecc00f-2f2f-4609-84ee-e592bea13374","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:00.24174972Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1598,"session_id":"aaecc00f-2f2f-4609-84ee-e592bea13374"} {"msg":"Upstairs opts: Upstairs UUID: 0fe0a310-a461-407e-8ec1-254a236f0dc3, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:00.241795697Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1598,"session_id":"aaecc00f-2f2f-4609-84ee-e592bea13374"} {"msg":"Crucible stats registered with UUID: 0fe0a310-a461-407e-8ec1-254a236f0dc3","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:00.241821675Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1598,"session_id":"aaecc00f-2f2f-4609-84ee-e592bea13374"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:00.272967617Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1598,"task":"crutest"} {"msg":"The guest has requested activation with gen:104","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:00.273002535Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1598} {"msg":"0fe0a310-a461-407e-8ec1-254a236f0dc3 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:00.27307259Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1598,"session_id":"aaecc00f-2f2f-4609-84ee-e592bea13374"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:00.273095508Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1598,"client":"0","":"downstairs","session_id":"aaecc00f-2f2f-4609-84ee-e592bea13374"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:00.273124377Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1598,"client":"0","":"downstairs","session_id":"aaecc00f-2f2f-4609-84ee-e592bea13374"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:00.273142595Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1598,"client":"1","":"downstairs","session_id":"aaecc00f-2f2f-4609-84ee-e592bea13374"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:00.273160584Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1598,"client":"1","":"downstairs","session_id":"aaecc00f-2f2f-4609-84ee-e592bea13374"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:00.273177703Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1598,"client":"2","":"downstairs","session_id":"aaecc00f-2f2f-4609-84ee-e592bea13374"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:00.273229809Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1598,"client":"2","":"downstairs","session_id":"aaecc00f-2f2f-4609-84ee-e592bea13374"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:00.273273096Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1598,"":"io task","client":"2","":"downstairs","session_id":"aaecc00f-2f2f-4609-84ee-e592bea13374"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:00.273303834Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1598,"":"io task","client":"0","":"downstairs","session_id":"aaecc00f-2f2f-4609-84ee-e592bea13374"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:00.273330202Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1598,"":"io task","client":"1","":"downstairs","session_id":"aaecc00f-2f2f-4609-84ee-e592bea13374"} {"msg":"ds_connection connected from Ok(127.0.0.1:59163)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:00.27335729Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1598,"":"io task","client":"2","":"downstairs","session_id":"aaecc00f-2f2f-4609-84ee-e592bea13374"} {"msg":"ds_connection connected from Ok(127.0.0.1:32911)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:00.273385448Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1598,"":"io task","client":"1","":"downstairs","session_id":"aaecc00f-2f2f-4609-84ee-e592bea13374"} {"msg":"ds_connection connected from Ok(127.0.0.1:43475)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:00.273490781Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1598,"":"io task","client":"0","":"downstairs","session_id":"aaecc00f-2f2f-4609-84ee-e592bea13374"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:00.274168945Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1598,"client":"2","":"downstairs","session_id":"aaecc00f-2f2f-4609-84ee-e592bea13374"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:00.274198792Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1598,"client":"0","":"downstairs","session_id":"aaecc00f-2f2f-4609-84ee-e592bea13374"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:00.274406718Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1598,"client":"1","":"downstairs","session_id":"aaecc00f-2f2f-4609-84ee-e592bea13374"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:00.274911123Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1598,"client":"2","":"downstairs","session_id":"aaecc00f-2f2f-4609-84ee-e592bea13374"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:00.274940051Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1598,"client":"0","":"downstairs","session_id":"aaecc00f-2f2f-4609-84ee-e592bea13374"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:00.27495528Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1598,"client":"1","":"downstairs","session_id":"aaecc00f-2f2f-4609-84ee-e592bea13374"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:00.275507002Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1598,"session_id":"aaecc00f-2f2f-4609-84ee-e592bea13374"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:00.27553401Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1598,"session_id":"aaecc00f-2f2f-4609-84ee-e592bea13374"} {"msg":"[0]R flush_numbers[0..12]: [182, 185, 187, 187, 187, 187, 191, 189, 191, 183, 191, 191]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:00.275666831Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1598,"":"downstairs","":"downstairs","session_id":"aaecc00f-2f2f-4609-84ee-e592bea13374"} {"msg":"[0]R generation[0..12]: [97, 98, 100, 100, 100, 100, 102, 101, 102, 98, 102, 102]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:00.27568684Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1598,"":"downstairs","":"downstairs","session_id":"aaecc00f-2f2f-4609-84ee-e592bea13374"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:00.275701479Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1598,"":"downstairs","":"downstairs","session_id":"aaecc00f-2f2f-4609-84ee-e592bea13374"} {"msg":"[1]R flush_numbers[0..12]: [182, 185, 187, 187, 187, 187, 191, 189, 191, 183, 191, 191]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:00.275715698Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1598,"":"downstairs","":"downstairs","session_id":"aaecc00f-2f2f-4609-84ee-e592bea13374"} {"msg":"[1]R generation[0..12]: [97, 98, 100, 100, 100, 100, 102, 101, 102, 98, 102, 102]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:00.275729687Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1598,"":"downstairs","":"downstairs","session_id":"aaecc00f-2f2f-4609-84ee-e592bea13374"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:00.275744106Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1598,"":"downstairs","":"downstairs","session_id":"aaecc00f-2f2f-4609-84ee-e592bea13374"} {"msg":"[2]R flush_numbers[0..12]: [182, 185, 187, 187, 187, 187, 191, 189, 191, 183, 191, 191]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:00.275758045Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1598,"":"downstairs","":"downstairs","session_id":"aaecc00f-2f2f-4609-84ee-e592bea13374"} {"msg":"[2]R generation[0..12]: [97, 98, 100, 100, 100, 100, 102, 101, 102, 98, 102, 102]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:00.275771504Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1598,"":"downstairs","":"downstairs","session_id":"aaecc00f-2f2f-4609-84ee-e592bea13374"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:00.275785093Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1598,"":"downstairs","":"downstairs","session_id":"aaecc00f-2f2f-4609-84ee-e592bea13374"} {"msg":"Max found gen is 104","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:00.275798072Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1598,"":"downstairs","":"downstairs","session_id":"aaecc00f-2f2f-4609-84ee-e592bea13374"} {"msg":"Generation requested: 104 >= found:104","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:00.275811301Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1598,"":"downstairs","":"downstairs","session_id":"aaecc00f-2f2f-4609-84ee-e592bea13374"} {"msg":"Next flush: 193","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:00.27582435Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1598,"":"downstairs","":"downstairs","session_id":"aaecc00f-2f2f-4609-84ee-e592bea13374"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-12-12T20:37:00.275837279Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1598,"":"downstairs","":"downstairs","session_id":"aaecc00f-2f2f-4609-84ee-e592bea13374"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:00.275901005Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1598,"session_id":"aaecc00f-2f2f-4609-84ee-e592bea13374"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:00.275917714Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1598,"client":"0","":"downstairs","session_id":"aaecc00f-2f2f-4609-84ee-e592bea13374"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:00.275931283Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1598,"client":"1","":"downstairs","session_id":"aaecc00f-2f2f-4609-84ee-e592bea13374"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:00.275944562Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1598,"client":"2","":"downstairs","session_id":"aaecc00f-2f2f-4609-84ee-e592bea13374"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:00.275957931Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1598,"session_id":"aaecc00f-2f2f-4609-84ee-e592bea13374"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:00.27597028Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1598,"session_id":"aaecc00f-2f2f-4609-84ee-e592bea13374"} {"msg":"0fe0a310-a461-407e-8ec1-254a236f0dc3 is now active with session: aaecc00f-2f2f-4609-84ee-e592bea13374","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:00.27598306Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1598,"session_id":"aaecc00f-2f2f-4609-84ee-e592bea13374"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:00.276009927Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1598,"session_id":"aaecc00f-2f2f-4609-84ee-e592bea13374"} {"msg":"The guest has finished waiting for activation with:104","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:00.276023497Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1598} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 271 len 32768 data: 8 10 8 9 8 7 8 10 02/30 Write block 162 len 40960 data: 9 8 9 7 7 4 4 4 5 4 03/30 Flush 04/30 Write block 25 len 20480 data: 6 6 7 7 7 05/30 Read block 590 len 12288 06/30 Read block 57 len 24576 07/30 Read block 149 len 12288 08/30 Write block 364 len 36864 data: 10 12 13 12 8 8 6 5 6 09/30 Write block 244 len 16384 data: 6 5 6 4 10/30 Write block 364 len 28672 data: 11 13 14 13 9 9 7 11/30 Read block 94 len 4096 12/30 Read block 44 len 16384 13/30 Read block 509 len 36864 14/30 Read block 137 len 36864 15/30 Write block 51 len 28672 data: 8 5 7 8 10 14 15 16/30 Write block 118 len 32768 data: 6 5 8 8 8 9 8 9 17/30 Flush 18/30 Write block 234 len 36864 data: 10 10 12 10 9 10 9 10 10 19/30 Write block 273 len 16384 data: 9 10 9 8 20/30 Flush 21/30 Flush 22/30 Write block 461 len 40960 data: 14 14 12 10 9 8 6 5 6 6 23/30 Write block 490 len 20480 data: 8 6 6 6 5 24/30 Write block 135 len 8192 data: 7 9 25/30 Read block 586 len 28672 26/30 Write block 115 len 32768 data: 8 8 6 7 6 9 9 9 27/30 Write block 178 len 28672 data: 7 10 8 8 6 7 10 28/30 Write block 177 len 8192 data: 8 8 29/30 Write block 97 len 16384 data: 4 3 6 8 30/30 Write block 304 len 24576 data: 2 4 4 5 6 5 ---------------------------------------------------------------- Crucible gen:104 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 10 Done Done Done false 1022 Acked Write 5 Done Done Done false 1023 Acked Write 2 Done Done Done false 1024 Acked Read 7 Done Done Done false 1025 Acked Write 8 Done Done Done false 1026 Acked Write 7 Done Done Done false 1027 Acked Write 2 Done Done Done false 1028 Acked Write 4 Sent Sent Sent false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 7 7 7 21 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1020 1020 1020 Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:37:00.295215753Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1598} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 97 97 97 182 182 182 F F F 1 020-039 104 104 104 194 194 194 F F F 2 040-059 104 104 104 194 194 194 F F F 3 060-079 100 100 100 187 187 187 F F F 4 080-099 100 100 100 187 187 187 T T T 5 100-119 104 104 104 194 194 194 T T T 6 120-139 104 104 104 194 194 194 T T T 7 140-159 101 101 101 189 189 189 F F F 8 160-179 104 104 104 193 193 193 T T T 9 180-199 98 98 98 183 183 183 T T T 10 200-219 102 102 102 191 191 191 F F F 11 220-239 104 104 104 195 195 195 F F F 12 240-259 104 104 104 195 195 195 F F F 13 260-279 104 104 104 195 195 195 F F F 14 280-299 103 103 103 192 192 192 F F F 15 300-319 93 93 93 175 175 175 T T T 16 320-339 98 98 98 185 185 185 F F F 17 340-359 102 102 102 191 191 191 F F F 18 360-379 104 104 104 194 194 194 F F F 19 380-399 99 99 99 186 186 186 F F F 20 400-419 99 99 99 186 186 186 F F F 21 420-439 102 102 102 191 191 191 F F F 22 440-459 101 101 101 189 189 189 F F F 23 460-479 101 101 101 189 189 189 T T T 24 480-499 101 101 101 189 189 189 T T T 25 500-519 103 103 103 192 192 192 F F F 26 520-539 103 103 103 192 192 192 F F F 27 540-559 98 98 98 185 185 185 F F F 28 560-579 102 102 102 191 191 191 F F F 29 580-599 97 97 97 182 182 182 F F F Max gen: 104, Max flush: 195 A Difference in extent metadata is expected here On loop 52 Verifying data now No change after verify Loop: 52 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 97 97 97 182 182 182 F F F 1 020-039 104 104 104 194 194 194 F F F 2 040-059 104 104 104 194 194 194 F F F 3 060-079 100 100 100 187 187 187 F F F 4 080-099 105 105 105 196 196 196 F F F 5 100-119 105 105 105 196 196 196 F F F 6 120-139 105 105 105 196 196 196 F F F 7 140-159 101 101 101 189 189 189 F F F 8 160-179 105 105 105 196 196 196 F F F 9 180-199 105 105 105 196 196 196 F F F 10 200-219 102 102 102 191 191 191 F F F 11 220-239 104 104 104 195 195 195 F F F 12 240-259 104 104 104 195 195 195 F F F 13 260-279 104 104 104 195 195 195 F F F 14 280-299 103 103 103 192 192 192 F F F 15 300-319 105 105 105 196 196 196 F F F 16 320-339 98 98 98 185 185 185 F F F 17 340-359 102 102 102 191 191 191 F F F 18 360-379 104 104 104 194 194 194 F F F 19 380-399 99 99 99 186 186 186 F F F 20 400-419 99 99 99 186 186 186 F F F 21 420-439 102 102 102 191 191 191 F F F 22 440-459 101 101 101 189 189 189 F F F 23 460-479 105 105 105 196 196 196 F F F 24 480-499 105 105 105 196 196 196 F F F 25 500-519 103 103 103 192 192 192 F F F 26 520-539 103 103 103 192 192 192 F F F 27 540-559 98 98 98 185 185 185 F F F 28 560-579 102 102 102 191 191 191 F F F 29 580-599 97 97 97 182 182 182 F F F Max gen: 105, Max flush: 196 Begin loop 53 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:04.580983034Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1606,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:37:04.624032805Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1606,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:04.624429428Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1606,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:04.656522605Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1606,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:04.657126973Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1606} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"26e7d501b33f47379e53601720353c2983e7654b\",\n git_commit_timestamp: \"2024-12-12T20:16:40.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:04.657162311Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1606} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:04.65717641Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1606} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:04.657190629Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1606} {"msg":"Crucible 0816cf8f-a714-4a16-a7bf-63c18fb83d4d has session id: 2fb84dc2-4fc7-41e6-b075-c31da5624aa1","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:04.657659407Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1606,"session_id":"2fb84dc2-4fc7-41e6-b075-c31da5624aa1"} {"msg":"Upstairs opts: Upstairs UUID: 0816cf8f-a714-4a16-a7bf-63c18fb83d4d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:04.657705183Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1606,"session_id":"2fb84dc2-4fc7-41e6-b075-c31da5624aa1"} {"msg":"Crucible stats registered with UUID: 0816cf8f-a714-4a16-a7bf-63c18fb83d4d","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:04.657730172Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1606,"session_id":"2fb84dc2-4fc7-41e6-b075-c31da5624aa1"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:04.689117927Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1606,"task":"crutest"} {"msg":"The guest has requested activation with gen:106","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:04.689150105Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1606} {"msg":"0816cf8f-a714-4a16-a7bf-63c18fb83d4d active request set","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:04.689207021Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1606,"session_id":"2fb84dc2-4fc7-41e6-b075-c31da5624aa1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:04.68922587Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1606,"client":"0","":"downstairs","session_id":"2fb84dc2-4fc7-41e6-b075-c31da5624aa1"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:04.689250938Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1606,"client":"0","":"downstairs","session_id":"2fb84dc2-4fc7-41e6-b075-c31da5624aa1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:04.689265397Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1606,"client":"1","":"downstairs","session_id":"2fb84dc2-4fc7-41e6-b075-c31da5624aa1"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:04.689281126Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1606,"client":"1","":"downstairs","session_id":"2fb84dc2-4fc7-41e6-b075-c31da5624aa1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:04.689306175Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1606,"client":"2","":"downstairs","session_id":"2fb84dc2-4fc7-41e6-b075-c31da5624aa1"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:04.689404048Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1606,"client":"2","":"downstairs","session_id":"2fb84dc2-4fc7-41e6-b075-c31da5624aa1"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:04.689438795Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1606,"":"io task","client":"2","":"downstairs","session_id":"2fb84dc2-4fc7-41e6-b075-c31da5624aa1"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:04.689469643Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1606,"":"io task","client":"0","":"downstairs","session_id":"2fb84dc2-4fc7-41e6-b075-c31da5624aa1"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:04.689497941Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1606,"":"io task","client":"1","":"downstairs","session_id":"2fb84dc2-4fc7-41e6-b075-c31da5624aa1"} {"msg":"ds_connection connected from Ok(127.0.0.1:51287)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:04.689528069Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1606,"":"io task","client":"2","":"downstairs","session_id":"2fb84dc2-4fc7-41e6-b075-c31da5624aa1"} {"msg":"ds_connection connected from Ok(127.0.0.1:37085)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:04.689637821Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1606,"":"io task","client":"0","":"downstairs","session_id":"2fb84dc2-4fc7-41e6-b075-c31da5624aa1"} {"msg":"ds_connection connected from Ok(127.0.0.1:40658)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:04.689680129Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1606,"":"io task","client":"1","":"downstairs","session_id":"2fb84dc2-4fc7-41e6-b075-c31da5624aa1"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:04.690165915Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1606,"client":"1","":"downstairs","session_id":"2fb84dc2-4fc7-41e6-b075-c31da5624aa1"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:04.690209462Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1606,"client":"2","":"downstairs","session_id":"2fb84dc2-4fc7-41e6-b075-c31da5624aa1"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:04.69024392Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1606,"client":"0","":"downstairs","session_id":"2fb84dc2-4fc7-41e6-b075-c31da5624aa1"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:04.690882356Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1606,"client":"1","":"downstairs","session_id":"2fb84dc2-4fc7-41e6-b075-c31da5624aa1"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:04.690911614Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1606,"client":"0","":"downstairs","session_id":"2fb84dc2-4fc7-41e6-b075-c31da5624aa1"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:04.690927783Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1606,"client":"2","":"downstairs","session_id":"2fb84dc2-4fc7-41e6-b075-c31da5624aa1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:04.691426598Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1606,"session_id":"2fb84dc2-4fc7-41e6-b075-c31da5624aa1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:04.691519792Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1606,"session_id":"2fb84dc2-4fc7-41e6-b075-c31da5624aa1"} {"msg":"[0]R flush_numbers[0..12]: [182, 194, 194, 187, 196, 196, 196, 189, 196, 196, 191, 195]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:04.691612535Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1606,"":"downstairs","":"downstairs","session_id":"2fb84dc2-4fc7-41e6-b075-c31da5624aa1"} {"msg":"[0]R generation[0..12]: [97, 104, 104, 100, 105, 105, 105, 101, 105, 105, 102, 104]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:04.691634034Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1606,"":"downstairs","":"downstairs","session_id":"2fb84dc2-4fc7-41e6-b075-c31da5624aa1"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:04.691650903Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1606,"":"downstairs","":"downstairs","session_id":"2fb84dc2-4fc7-41e6-b075-c31da5624aa1"} {"msg":"[1]R flush_numbers[0..12]: [182, 194, 194, 187, 196, 196, 196, 189, 196, 196, 191, 195]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:04.691665602Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1606,"":"downstairs","":"downstairs","session_id":"2fb84dc2-4fc7-41e6-b075-c31da5624aa1"} {"msg":"[1]R generation[0..12]: [97, 104, 104, 100, 105, 105, 105, 101, 105, 105, 102, 104]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:04.691679671Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1606,"":"downstairs","":"downstairs","session_id":"2fb84dc2-4fc7-41e6-b075-c31da5624aa1"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:04.69169379Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1606,"":"downstairs","":"downstairs","session_id":"2fb84dc2-4fc7-41e6-b075-c31da5624aa1"} {"msg":"[2]R flush_numbers[0..12]: [182, 194, 194, 187, 196, 196, 196, 189, 196, 196, 191, 195]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:04.691707899Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1606,"":"downstairs","":"downstairs","session_id":"2fb84dc2-4fc7-41e6-b075-c31da5624aa1"} {"msg":"[2]R generation[0..12]: [97, 104, 104, 100, 105, 105, 105, 101, 105, 105, 102, 104]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:04.691724868Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1606,"":"downstairs","":"downstairs","session_id":"2fb84dc2-4fc7-41e6-b075-c31da5624aa1"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:04.691739097Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1606,"":"downstairs","":"downstairs","session_id":"2fb84dc2-4fc7-41e6-b075-c31da5624aa1"} {"msg":"Max found gen is 106","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:04.691752056Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1606,"":"downstairs","":"downstairs","session_id":"2fb84dc2-4fc7-41e6-b075-c31da5624aa1"} {"msg":"Generation requested: 106 >= found:106","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:04.691765045Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1606,"":"downstairs","":"downstairs","session_id":"2fb84dc2-4fc7-41e6-b075-c31da5624aa1"} {"msg":"Next flush: 197","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO 2024-12-12T20:37:04.691777934Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1606,"":"downstairs","":"downstairs","session_id":"2fb84dc2-4fc7-41e6-b075-c31da5624aa1"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:04.691795413Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1606,"":"downstairs","":"downstairs","session_id":"2fb84dc2-4fc7-41e6-b075-c31da5624aa1"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:04.691808632Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1606,"session_id":"2fb84dc2-4fc7-41e6-b075-c31da5624aa1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:04.691820711Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1606,"client":"0","":"downstairs","session_id":"2fb84dc2-4fc7-41e6-b075-c31da5624aa1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:04.69183496Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1606,"client":"1","":"downstairs","session_id":"2fb84dc2-4fc7-41e6-b075-c31da5624aa1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:04.691848329Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1606,"client":"2","":"downstairs","session_id":"2fb84dc2-4fc7-41e6-b075-c31da5624aa1"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:04.691861418Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1606,"session_id":"2fb84dc2-4fc7-41e6-b075-c31da5624aa1"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:04.691873577Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1606,"session_id":"2fb84dc2-4fc7-41e6-b075-c31da5624aa1"} {"msg":"0816cf8f-a714-4a16-a7bf-63c18fb83d4d is now active with session: 2fb84dc2-4fc7-41e6-b075-c31da5624aa1","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:04.691886337Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1606,"session_id":"2fb84dc2-4fc7-41e6-b075-c31da5624aa1"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:04.691898446Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1606,"session_id":"2fb84dc2-4fc7-41e6-b075-c31da5624aa1"} {"msg":"The guest has finished waiting for activation with:106","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:04.691910785Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1606} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 467 len 32768 data: 7 6 7 7 6 5 7 7 02/30 Write block 363 len 40960 data: 9 12 14 15 14 10 10 8 6 7 03/30 Read block 38 len 4096 04/30 Read block 72 len 20480 05/30 Read block 17 len 12288 06/30 Read block 90 len 12288 07/30 Read block 259 len 28672 08/30 Read block 2 len 40960 09/30 Write block 200 len 28672 data: 6 7 8 9 12 12 12 10/30 Write block 24 len 8192 data: 8 7 11/30 Read block 68 len 16384 12/30 Write block 5 len 16384 data: 6 6 7 9 13/30 Read block 426 len 20480 14/30 Read block 28 len 20480 15/30 Write block 195 len 12288 data: 8 8 6 16/30 Read block 58 len 32768 17/30 Write block 325 len 36864 data: 5 5 4 5 9 10 11 8 9 18/30 Write block 488 len 24576 data: 9 10 9 7 7 7 19/30 Read block 352 len 8192 20/30 Write block 427 len 16384 data: 8 8 9 6 21/30 Read block 241 len 20480 22/30 Read block 67 len 24576 23/30 Write block 214 len 16384 data: 9 9 6 8 24/30 Write block 403 len 8192 data: 9 8 25/30 Read block 451 len 24576 26/30 Read block 487 len 36864 27/30 Write block 590 len 16384 data: 12 11 10 8 28/30 Write block 106 len 28672 data: 8 11 10 9 10 11 11 29/30 Read block 315 len 28672 30/30 Read block 11 len 24576 ---------------------------------------------------------------- Crucible gen:106 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 8 Done Done Done false 1001 Acked Write 10 Done Done Done false 1002 Acked Read 1 Done Done Done false 1003 Acked Read 5 Done Done Done false 1004 Acked Read 3 Done Done Done false 1005 Acked Read 3 Done Done Done false 1006 Acked Read 7 Done Done Done false 1007 Acked Read 10 Done Done Done false 1008 Acked Write 7 Done Done Done false 1009 Acked Write 2 Done Done Done false 1010 Acked Read 4 Done Done Done false 1011 Acked Write 4 Done Done Done false 1012 Acked Read 5 Done Done Done false 1013 Acked Read 5 Done Done Done false 1014 Acked Write 3 Done Done Done false 1015 Acked Read 8 Done Done Done false 1016 Acked Write 9 Done Done Done false 1017 Acked Write 6 Done Done Done false 1018 Acked Read 2 Done Done Done false 1019 Acked Write 4 Done Done Done false 1020 Acked Read 5 Done Done Done false 1021 Acked Read 6 Done Done Done false 1022 Acked Write 4 Done Done Done false 1023 Acked Write 2 Done Done Done false 1024 Acked Read 6 Done Done Done false 1025 Acked Read 9 Done Done Done false 1026 Acked Write 4 Done Done Done false 1027 Acked Write 7 Done Done Done false 1028 Acked Read 7 Done Done Done false 1029 Acked Read 6 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 30 30 30 90 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 0 0 0 Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:37:04.712159579Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1606} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 97 97 97 182 182 182 T T T 1 020-039 104 104 104 194 194 194 T T T 2 040-059 104 104 104 194 194 194 F F F 3 060-079 100 100 100 187 187 187 F F F 4 080-099 105 105 105 196 196 196 F F F 5 100-119 105 105 105 196 196 196 T T T 6 120-139 105 105 105 196 196 196 F F F 7 140-159 101 101 101 189 189 189 F F F 8 160-179 105 105 105 196 196 196 F F F 9 180-199 105 105 105 196 196 196 T T T 10 200-219 102 102 102 191 191 191 T T T 11 220-239 104 104 104 195 195 195 F F F 12 240-259 104 104 104 195 195 195 F F F 13 260-279 104 104 104 195 195 195 F F F 14 280-299 103 103 103 192 192 192 F F F 15 300-319 105 105 105 196 196 196 F F F 16 320-339 98 98 98 185 185 185 T T T 17 340-359 102 102 102 191 191 191 F F F 18 360-379 104 104 104 194 194 194 T T T 19 380-399 99 99 99 186 186 186 F F F 20 400-419 99 99 99 186 186 186 T T T 21 420-439 102 102 102 191 191 191 T T T 22 440-459 101 101 101 189 189 189 F F F 23 460-479 105 105 105 196 196 196 T T T 24 480-499 105 105 105 196 196 196 T T T 25 500-519 103 103 103 192 192 192 F F F 26 520-539 103 103 103 192 192 192 F F F 27 540-559 98 98 98 185 185 185 F F F 28 560-579 102 102 102 191 191 191 F F F 29 580-599 97 97 97 182 182 182 T T T Max gen: 105, Max flush: 196 A Difference in extent metadata is expected here On loop 53 Verifying data now No change after verify Loop: 53 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 106 106 106 197 197 197 F F F 1 020-039 106 106 106 197 197 197 F F F 2 040-059 104 104 104 194 194 194 F F F 3 060-079 100 100 100 187 187 187 F F F 4 080-099 105 105 105 196 196 196 F F F 5 100-119 106 106 106 197 197 197 F F F 6 120-139 105 105 105 196 196 196 F F F 7 140-159 101 101 101 189 189 189 F F F 8 160-179 105 105 105 196 196 196 F F F 9 180-199 106 106 106 197 197 197 F F F 10 200-219 106 106 106 197 197 197 F F F 11 220-239 104 104 104 195 195 195 F F F 12 240-259 104 104 104 195 195 195 F F F 13 260-279 104 104 104 195 195 195 F F F 14 280-299 103 103 103 192 192 192 F F F 15 300-319 105 105 105 196 196 196 F F F 16 320-339 106 106 106 197 197 197 F F F 17 340-359 102 102 102 191 191 191 F F F 18 360-379 106 106 106 197 197 197 F F F 19 380-399 99 99 99 186 186 186 F F F 20 400-419 106 106 106 197 197 197 F F F 21 420-439 106 106 106 197 197 197 F F F 22 440-459 101 101 101 189 189 189 F F F 23 460-479 106 106 106 197 197 197 F F F 24 480-499 106 106 106 197 197 197 F F F 25 500-519 103 103 103 192 192 192 F F F 26 520-539 103 103 103 192 192 192 F F F 27 540-559 98 98 98 185 185 185 F F F 28 560-579 102 102 102 191 191 191 F F F 29 580-599 106 106 106 197 197 197 F F F Max gen: 106, Max flush: 197 Begin loop 54 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:09.050433847Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1614,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:09.095132795Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1614,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:09.095711745Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1614} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"26e7d501b33f47379e53601720353c2983e7654b\",\n git_commit_timestamp: \"2024-12-12T20:16:40.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:09.095749912Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1614} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:09.095764181Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1614} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:09.09577866Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1614} {"msg":"Crucible e31b99d1-8967-4413-befc-9ee7cd21830d has session id: 32ef6e70-3914-4197-90dc-8a6b356d92a8","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:09.096206081Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1614,"session_id":"32ef6e70-3914-4197-90dc-8a6b356d92a8"} {"msg":"Upstairs opts: Upstairs UUID: e31b99d1-8967-4413-befc-9ee7cd21830d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:09.096253558Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1614,"session_id":"32ef6e70-3914-4197-90dc-8a6b356d92a8"} {"msg":"Crucible stats registered with UUID: e31b99d1-8967-4413-befc-9ee7cd21830d","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:09.096274076Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1614,"session_id":"32ef6e70-3914-4197-90dc-8a6b356d92a8"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:09.126948901Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1614,"task":"crutest"} {"msg":"The guest has requested activation with gen:108","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:09.126980719Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1614} {"msg":"e31b99d1-8967-4413-befc-9ee7cd21830d active request set","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:09.127053784Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1614,"session_id":"32ef6e70-3914-4197-90dc-8a6b356d92a8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:09.127076432Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1614,"client":"0","":"downstairs","session_id":"32ef6e70-3914-4197-90dc-8a6b356d92a8"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:09.1271146Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1614,"client":"0","":"downstairs","session_id":"32ef6e70-3914-4197-90dc-8a6b356d92a8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:09.127149017Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1614,"client":"1","":"downstairs","session_id":"32ef6e70-3914-4197-90dc-8a6b356d92a8"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:09.127185865Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1614,"client":"1","":"downstairs","session_id":"32ef6e70-3914-4197-90dc-8a6b356d92a8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:09.127219332Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1614,"client":"2","":"downstairs","session_id":"32ef6e70-3914-4197-90dc-8a6b356d92a8"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:09.127262539Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1614,"client":"2","":"downstairs","session_id":"32ef6e70-3914-4197-90dc-8a6b356d92a8"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:09.127296307Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1614,"":"io task","client":"2","":"downstairs","session_id":"32ef6e70-3914-4197-90dc-8a6b356d92a8"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:09.127329865Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1614,"":"io task","client":"0","":"downstairs","session_id":"32ef6e70-3914-4197-90dc-8a6b356d92a8"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:09.127360843Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1614,"":"io task","client":"1","":"downstairs","session_id":"32ef6e70-3914-4197-90dc-8a6b356d92a8"} {"msg":"ds_connection connected from Ok(127.0.0.1:49685)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:09.127439147Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1614,"":"io task","client":"2","":"downstairs","session_id":"32ef6e70-3914-4197-90dc-8a6b356d92a8"} {"msg":"ds_connection connected from Ok(127.0.0.1:59742)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:09.127476815Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1614,"":"io task","client":"0","":"downstairs","session_id":"32ef6e70-3914-4197-90dc-8a6b356d92a8"} {"msg":"ds_connection connected from Ok(127.0.0.1:34690)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:09.127503243Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1614,"":"io task","client":"1","":"downstairs","session_id":"32ef6e70-3914-4197-90dc-8a6b356d92a8"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:09.127950952Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1614,"client":"2","":"downstairs","session_id":"32ef6e70-3914-4197-90dc-8a6b356d92a8"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:09.128006678Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1614,"client":"0","":"downstairs","session_id":"32ef6e70-3914-4197-90dc-8a6b356d92a8"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:09.128053065Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1614,"client":"1","":"downstairs","session_id":"32ef6e70-3914-4197-90dc-8a6b356d92a8"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:09.128604857Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1614,"client":"0","":"downstairs","session_id":"32ef6e70-3914-4197-90dc-8a6b356d92a8"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:09.128742537Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1614,"client":"1","":"downstairs","session_id":"32ef6e70-3914-4197-90dc-8a6b356d92a8"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:09.128771795Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1614,"client":"2","":"downstairs","session_id":"32ef6e70-3914-4197-90dc-8a6b356d92a8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:09.129106202Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1614,"session_id":"32ef6e70-3914-4197-90dc-8a6b356d92a8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:09.129318128Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1614,"session_id":"32ef6e70-3914-4197-90dc-8a6b356d92a8"} {"msg":"[0]R flush_numbers[0..12]: [197, 197, 194, 187, 196, 197, 196, 189, 196, 197, 197, 195]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:09.129418561Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1614,"":"downstairs","":"downstairs","session_id":"32ef6e70-3914-4197-90dc-8a6b356d92a8"} {"msg":"[0]R generation[0..12]: [106, 106, 104, 100, 105, 106, 105, 101, 105, 106, 106, 104]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:09.129446999Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1614,"":"downstairs","":"downstairs","session_id":"32ef6e70-3914-4197-90dc-8a6b356d92a8"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:09.129463208Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1614,"":"downstairs","":"downstairs","session_id":"32ef6e70-3914-4197-90dc-8a6b356d92a8"} {"msg":"[1]R flush_numbers[0..12]: [197, 197, 194, 187, 196, 197, 196, 189, 196, 197, 197, 195]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:09.129477627Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1614,"":"downstairs","":"downstairs","session_id":"32ef6e70-3914-4197-90dc-8a6b356d92a8"} {"msg":"[1]R generation[0..12]: [106, 106, 104, 100, 105, 106, 105, 101, 105, 106, 106, 104]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:09.129491796Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1614,"":"downstairs","":"downstairs","session_id":"32ef6e70-3914-4197-90dc-8a6b356d92a8"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:09.129505825Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1614,"":"downstairs","":"downstairs","session_id":"32ef6e70-3914-4197-90dc-8a6b356d92a8"} {"msg":"[2]R flush_numbers[0..12]: [197, 197, 194, 187, 196, 197, 196, 189, 196, 197, 197, 195]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:09.129519574Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1614,"":"downstairs","":"downstairs","session_id":"32ef6e70-3914-4197-90dc-8a6b356d92a8"} {"msg":"[2]R generation[0..12]: [106, 106, 104, 100, 105, 106, 105, 101, 105, 106, 106, 104]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:09.129533653Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1614,"":"downstairs","":"downstairs","session_id":"32ef6e70-3914-4197-90dc-8a6b356d92a8"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:09.129547992Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1614,"":"downstairs","":"downstairs","session_id":"32ef6e70-3914-4197-90dc-8a6b356d92a8"} {"msg":"Max found gen is 107","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:09.129561251Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1614,"":"downstairs","":"downstairs","session_id":"32ef6e70-3914-4197-90dc-8a6b356d92a8"} {"msg":"Generation requested: 108 >= found:107","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:09.12957451Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1614,"":"downstairs","":"downstairs","session_id":"32ef6e70-3914-4197-90dc-8a6b356d92a8"} {"msg":"Next flush: 198","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:09.129587609Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1614,"":"downstairs","":"downstairs","session_id":"32ef6e70-3914-4197-90dc-8a6b356d92a8"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:09.129600608Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1614,"":"downstairs","":"downstairs","session_id":"32ef6e70-3914-4197-90dc-8a6b356d92a8"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:09.129618017Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1614,"session_id":"32ef6e70-3914-4197-90dc-8a6b356d92a8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"Wait for a query_work_queue command to finish before sending IO crucible","level":30,"time":"2024-12-12T20:37:09.129636076Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1614,"client":"0","":"downstairs","session_id":"32ef6e70-3914-4197-90dc-8a6b356d92a8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:09.129649615Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1614,"client":"1","":"downstairs","session_id":"32ef6e70-3914-4197-90dc-8a6b356d92a8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:09.129663064Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1614,"client":"2","":"downstairs","session_id":"32ef6e70-3914-4197-90dc-8a6b356d92a8"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:09.129676613Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1614,"session_id":"32ef6e70-3914-4197-90dc-8a6b356d92a8"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:09.129688912Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1614,"session_id":"32ef6e70-3914-4197-90dc-8a6b356d92a8"} {"msg":"e31b99d1-8967-4413-befc-9ee7cd21830d is now active with session: 32ef6e70-3914-4197-90dc-8a6b356d92a8","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:09.129701631Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1614,"session_id":"32ef6e70-3914-4197-90dc-8a6b356d92a8"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:09.12971361Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1614,"session_id":"32ef6e70-3914-4197-90dc-8a6b356d92a8"} {"msg":"The guest has finished waiting for activation with:108","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:09.1297259Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1614} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 182 len 20480 data: 7 8 11 12 11 02/30 Read block 142 len 4096 03/30 Read block 189 len 32768 04/30 Read block 53 len 20480 05/30 Write block 9 len 20480 data: 10 9 10 8 6 06/30 Write block 310 len 28672 data: 5 9 8 11 12 12 12 07/30 Write block 576 len 32768 data: 8 7 9 13 10 10 11 11 08/30 Read block 197 len 20480 09/30 Write block 461 len 32768 data: 15 15 13 11 10 9 8 7 10/30 Read block 377 len 28672 11/30 Read block 231 len 12288 12/30 Write block 518 len 28672 data: 14 17 18 17 15 19 18 13/30 Write block 337 len 16384 data: 7 8 6 7 14/30 Write block 568 len 32768 data: 8 9 7 9 8 8 9 9 15/30 Flush 16/30 Write block 354 len 12288 data: 11 10 9 17/30 Read block 252 len 36864 18/30 Read block 282 len 12288 19/30 Read block 58 len 32768 20/30 Write block 462 len 20480 data: 16 14 12 11 10 21/30 Write block 406 len 20480 data: 11 11 11 9 8 22/30 Write block 34 len 4096 data: 10 23/30 Read block 92 len 12288 24/30 Read block 207 len 40960 25/30 Read block 209 len 16384 26/30 Read block 484 len 40960 27/30 Write block 586 len 8192 data: 14 13 28/30 Read block 357 len 12288 29/30 Read block 425 len 24576 30/30 Read block 125 len 36864 ---------------------------------------------------------------- Crucible gen:108 GIO:true work queues: Upstairs:0 downstairs:15 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1015 Acked Write 3 Done Done Done false 1016 Acked Read 9 Done Done Done false 1017 Acked Read 3 Done Done Done false 1018 Acked Read 8 Done Done Done false 1019 Acked Write 5 Done Done Done false 1020 Acked Write 5 Done Done Done false 1021 Acked Write 1 Done Done Done false 1022 Acked Read 3 Done Done Done false 1023 Acked Read 10 Done Done Done false 1024 Acked Read 4 Done Done Done false 1025 Acked Read 10 Done Done Done false 1026 Acked Write 2 Done Done Done false 1027 Acked Read 3 Done Done Done false 1028 Acked Read 6 Done Done Done false 1029 Acked Read 9 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 15 15 15 45 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1014 1014 1014 Downstairs last five completed: 1014 1013 1012 1011 1010 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:37:09.149833963Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1614} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 108 108 108 198 198 198 F F F 1 020-039 106 106 106 197 197 197 T T T 2 040-059 104 104 104 194 194 194 F F F 3 060-079 100 100 100 187 187 187 F F F 4 080-099 105 105 105 196 196 196 F F F 5 100-119 106 106 106 197 197 197 F F F 6 120-139 105 105 105 196 196 196 F F F 7 140-159 101 101 101 189 189 189 F F F 8 160-179 105 105 105 196 196 196 F F F 9 180-199 108 108 108 198 198 198 F F F 10 200-219 106 106 106 197 197 197 F F F 11 220-239 104 104 104 195 195 195 F F F 12 240-259 104 104 104 195 195 195 F F F 13 260-279 104 104 104 195 195 195 F F F 14 280-299 103 103 103 192 192 192 F F F 15 300-319 108 108 108 198 198 198 F F F 16 320-339 108 108 108 198 198 198 F F F 17 340-359 108 108 108 198 198 198 T T T 18 360-379 106 106 106 197 197 197 F F F 19 380-399 99 99 99 186 186 186 F F F 20 400-419 106 106 106 197 197 197 T T T 21 420-439 106 106 106 197 197 197 F F F 22 440-459 101 101 101 189 189 189 F F F 23 460-479 108 108 108 198 198 198 T T T 24 480-499 106 106 106 197 197 197 F F F 25 500-519 108 108 108 198 198 198 F F F 26 520-539 108 108 108 198 198 198 F F F 27 540-559 98 98 98 185 185 185 F F F 28 560-579 108 108 108 198 198 198 F F F 29 580-599 108 108 108 198 198 198 T T T Max gen: 108, Max flush: 198 A Difference in extent metadata is expected here On loop 54 Verifying data now No change after verify Loop: 54 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 108 108 108 198 198 198 F F F 1 020-039 109 109 109 199 199 199 F F F 2 040-059 104 104 104 194 194 194 F F F 3 060-079 100 100 100 187 187 187 F F F 4 080-099 105 105 105 196 196 196 F F F 5 100-119 106 106 106 197 197 197 F F F 6 120-139 105 105 105 196 196 196 F F F 7 140-159 101 101 101 189 189 189 F F F 8 160-179 105 105 105 196 196 196 F F F 9 180-199 108 108 108 198 198 198 F F F 10 200-219 106 106 106 197 197 197 F F F 11 220-239 104 104 104 195 195 195 F F F 12 240-259 104 104 104 195 195 195 F F F 13 260-279 104 104 104 195 195 195 F F F 14 280-299 103 103 103 192 192 192 F F F 15 300-319 108 108 108 198 198 198 F F F 16 320-339 108 108 108 198 198 198 F F F 17 340-359 109 109 109 199 199 199 F F F 18 360-379 106 106 106 197 197 197 F F F 19 380-399 99 99 99 186 186 186 F F F 20 400-419 109 109 109 199 199 199 F F F 21 420-439 106 106 106 197 197 197 F F F 22 440-459 101 101 101 189 189 189 F F F 23 460-479 109 109 109 199 199 199 F F F 24 480-499 106 106 106 197 197 197 F F F 25 500-519 108 108 108 198 198 198 F F F 26 520-539 108 108 108 198 198 198 F F F 27 540-559 98 98 98 185 185 185 F F F 28 560-579 108 108 108 198 198 198 F F F 29 580-599 109 109 109 199 199 199 F F F Max gen: 109, Max flush: 199 Begin loop 55 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:13.425223138Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1622,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:13.469529483Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1622,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:13.470208036Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1622} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"26e7d501b33f47379e53601720353c2983e7654b\",\n git_commit_timestamp: \"2024-12-12T20:16:40.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:13.470243764Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1622} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:13.470257733Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1622} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:13.470271792Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1622} {"msg":"Crucible 5d665898-532e-441c-a442-c9dc35b8d431 has session id: d5dd9865-b0b5-45a8-b0d1-43b99712b0f7","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:13.470701552Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1622,"session_id":"d5dd9865-b0b5-45a8-b0d1-43b99712b0f7"} {"msg":"Upstairs opts: Upstairs UUID: 5d665898-532e-441c-a442-c9dc35b8d431, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:13.470746269Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1622,"session_id":"d5dd9865-b0b5-45a8-b0d1-43b99712b0f7"} {"msg":"Crucible stats registered with UUID: 5d665898-532e-441c-a442-c9dc35b8d431","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:13.470771238Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1622,"session_id":"d5dd9865-b0b5-45a8-b0d1-43b99712b0f7"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:13.502303883Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1622,"task":"crutest"} {"msg":"The guest has requested activation with gen:110","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:13.50235281Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1622} {"msg":"5d665898-532e-441c-a442-c9dc35b8d431 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:13.502423685Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1622,"session_id":"d5dd9865-b0b5-45a8-b0d1-43b99712b0f7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:13.502458353Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1622,"client":"0","":"downstairs","session_id":"d5dd9865-b0b5-45a8-b0d1-43b99712b0f7"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:13.502505739Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1622,"client":"0","":"downstairs","session_id":"d5dd9865-b0b5-45a8-b0d1-43b99712b0f7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:13.502542387Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1622,"client":"1","":"downstairs","session_id":"d5dd9865-b0b5-45a8-b0d1-43b99712b0f7"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:13.502612792Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1622,"client":"1","":"downstairs","session_id":"d5dd9865-b0b5-45a8-b0d1-43b99712b0f7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:13.502660919Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1622,"client":"2","":"downstairs","session_id":"d5dd9865-b0b5-45a8-b0d1-43b99712b0f7"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:13.502691806Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1622,"client":"2","":"downstairs","session_id":"d5dd9865-b0b5-45a8-b0d1-43b99712b0f7"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:13.502722324Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1622,"":"io task","client":"2","":"downstairs","session_id":"d5dd9865-b0b5-45a8-b0d1-43b99712b0f7"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:13.502757842Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1622,"":"io task","client":"0","":"downstairs","session_id":"d5dd9865-b0b5-45a8-b0d1-43b99712b0f7"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:13.502794139Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1622,"":"io task","client":"1","":"downstairs","session_id":"d5dd9865-b0b5-45a8-b0d1-43b99712b0f7"} {"msg":"ds_connection connected from Ok(127.0.0.1:36617)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:13.502847636Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1622,"":"io task","client":"0","":"downstairs","session_id":"d5dd9865-b0b5-45a8-b0d1-43b99712b0f7"} {"msg":"ds_connection connected from Ok(127.0.0.1:38033)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:13.502888493Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1622,"":"io task","client":"2","":"downstairs","session_id":"d5dd9865-b0b5-45a8-b0d1-43b99712b0f7"} {"msg":"ds_connection connected from Ok(127.0.0.1:33265)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:13.50292359Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1622,"":"io task","client":"1","":"downstairs","session_id":"d5dd9865-b0b5-45a8-b0d1-43b99712b0f7"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:13.503596424Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1622,"client":"0","":"downstairs","session_id":"d5dd9865-b0b5-45a8-b0d1-43b99712b0f7"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:13.503630192Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1622,"client":"2","":"downstairs","session_id":"d5dd9865-b0b5-45a8-b0d1-43b99712b0f7"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:13.50365755Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1622,"client":"1","":"downstairs","session_id":"d5dd9865-b0b5-45a8-b0d1-43b99712b0f7"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:13.504255298Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1622,"client":"1","":"downstairs","session_id":"d5dd9865-b0b5-45a8-b0d1-43b99712b0f7"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:13.504285347Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1622,"client":"0","":"downstairs","session_id":"d5dd9865-b0b5-45a8-b0d1-43b99712b0f7"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:13.504302895Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1622,"client":"2","":"downstairs","session_id":"d5dd9865-b0b5-45a8-b0d1-43b99712b0f7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:13.504783492Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1622,"session_id":"d5dd9865-b0b5-45a8-b0d1-43b99712b0f7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:13.50481234Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1622,"session_id":"d5dd9865-b0b5-45a8-b0d1-43b99712b0f7"} {"msg":"[0]R flush_numbers[0..12]: [198, 199, 194, 187, 196, 197, 196, 189, 196, 198, 197, 195]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:13.504876996Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1622,"":"downstairs","":"downstairs","session_id":"d5dd9865-b0b5-45a8-b0d1-43b99712b0f7"} {"msg":"[0]R generation[0..12]: [108, 109, 104, 100, 105, 106, 105, 101, 105, 108, 106, 104]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:13.504896124Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1622,"":"downstairs","":"downstairs","session_id":"d5dd9865-b0b5-45a8-b0d1-43b99712b0f7"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:13.504910843Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1622,"":"downstairs","":"downstairs","session_id":"d5dd9865-b0b5-45a8-b0d1-43b99712b0f7"} {"msg":"[1]R flush_numbers[0..12]: [198, 199, 194, 187, 196, 197, 196, 189, 196, 198, 197, 195]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:13.504925412Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1622,"":"downstairs","":"downstairs","session_id":"d5dd9865-b0b5-45a8-b0d1-43b99712b0f7"} {"msg":"[1]R generation[0..12]: [108, 109, 104, 100, 105, 106, 105, 101, 105, 108, 106, 104]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:13.504939251Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1622,"":"downstairs","":"downstairs","session_id":"d5dd9865-b0b5-45a8-b0d1-43b99712b0f7"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:13.504953301Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1622,"":"downstairs","":"downstairs","session_id":"d5dd9865-b0b5-45a8-b0d1-43b99712b0f7"} {"msg":"[2]R flush_numbers[0..12]: [198, 199, 194, 187, 196, 197, 196, 189, 196, 198, 197, 195]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:13.504967029Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1622,"":"downstairs","":"downstairs","session_id":"d5dd9865-b0b5-45a8-b0d1-43b99712b0f7"} {"msg":"[2]R generation[0..12]: [108, 109, 104, 100, 105, 106, 105, 101, 105, 108, 106, 104]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:13.504984968Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1622,"":"downstairs","":"downstairs","session_id":"d5dd9865-b0b5-45a8-b0d1-43b99712b0f7"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:13.505015386Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1622,"":"downstairs","":"downstairs","session_id":"d5dd9865-b0b5-45a8-b0d1-43b99712b0f7"} {"msg":"Max found gen is 110","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:13.505029255Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1622,"":"downstairs","":"downstairs","session_id":"d5dd9865-b0b5-45a8-b0d1-43b99712b0f7"} {"msg":"Generation requested: 110 >= found:110","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:13.505042524Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1622,"":"downstairs","":"downstairs","session_id":"d5dd9865-b0b5-45a8-b0d1-43b99712b0f7"} {"msg":"Next flush: 200","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:13.505055373Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1622,"":"downstairs","":"downstairs","session_id":"d5dd9865-b0b5-45a8-b0d1-43b99712b0f7"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:13.505068342Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1622,"":"downstairs","":"downstairs","Wait for a query_work_queue command to finish before sending IO session_id":"d5dd9865-b0b5-45a8-b0d1-43b99712b0f7"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:13.505086641Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1622,"session_id":"d5dd9865-b0b5-45a8-b0d1-43b99712b0f7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:13.50509891Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1622,"client":"0","":"downstairs","session_id":"d5dd9865-b0b5-45a8-b0d1-43b99712b0f7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:13.50511223Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1622,"client":"1","":"downstairs","session_id":"d5dd9865-b0b5-45a8-b0d1-43b99712b0f7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:13.505125038Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1622,"client":"2","":"downstairs","session_id":"d5dd9865-b0b5-45a8-b0d1-43b99712b0f7"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:13.505138108Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1622,"session_id":"d5dd9865-b0b5-45a8-b0d1-43b99712b0f7"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:13.505150307Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1622,"session_id":"d5dd9865-b0b5-45a8-b0d1-43b99712b0f7"} {"msg":"5d665898-532e-441c-a442-c9dc35b8d431 is now active with session: d5dd9865-b0b5-45a8-b0d1-43b99712b0f7","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:13.505162946Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1622,"session_id":"d5dd9865-b0b5-45a8-b0d1-43b99712b0f7"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:13.505174805Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1622,"session_id":"d5dd9865-b0b5-45a8-b0d1-43b99712b0f7"} {"msg":"The guest has finished waiting for activation with:110","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:13.505186894Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1622} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 361 len 4096 data: 7 02/30 Write block 192 len 16384 data: 11 11 9 9 03/30 Flush 04/30 Flush 05/30 Write block 590 len 8192 data: 13 12 06/30 Write block 566 len 12288 data: 10 7 9 07/30 Write block 112 len 40960 data: 12 10 9 9 9 7 8 7 10 10 08/30 Write block 163 len 28672 data: 9 10 8 8 5 5 5 09/30 Write block 385 len 8192 data: 8 9 10/30 Read block 289 len 40960 11/30 Flush 12/30 Flush 13/30 Write block 590 len 32768 data: 14 13 11 9 6 5 4 4 14/30 Flush 15/30 Write block 52 len 20480 data: 6 8 9 11 15 16/30 Read block 145 len 40960 17/30 Read block 209 len 40960 18/30 Write block 455 len 12288 data: 9 9 10 19/30 Read block 260 len 20480 20/30 Write block 379 len 40960 data: 10 11 11 11 9 7 9 10 8 8 21/30 Read block 221 len 12288 22/30 Flush 23/30 Write block 88 len 36864 data: 13 11 11 9 6 9 8 7 5 24/30 Read block 106 len 24576 25/30 Write block 438 len 4096 data: 9 26/30 Write block 171 len 24576 data: 5 4 4 5 7 7 27/30 Write block 469 len 20480 data: 8 8 7 6 8 28/30 Read block 461 len 12288 29/30 Write block 49 len 28672 data: 9 9 9 7 9 10 12 30/30 Read block 253 len 4096 ---------------------------------------------------------------- Crucible gen:110 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 9 Done Done Done false 1023 Acked Read 6 Done Done Done false 1024 Acked Write 1 Done Done Done false 1025 Acked Write 6 Done Done Done false 1026 Acked Write 5 Done Done Done false 1027 Acked Read 3 Done Done Done false 1028 Acked Write 7 Done Done Done false 1029 Acked Read 1 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 8 8 8 24 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1021 1021 1021 Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:37:13.524527341Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1622} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 108 108 108 198 198 198 F F F 1 020-039 109 109 109 199 199 199 F F F 2 040-059 110 110 110 205 205 205 T T T 3 060-079 100 100 100 187 187 187 F F F 4 080-099 105 105 105 196 196 196 T T T 5 100-119 110 110 110 202 202 202 F F F 6 120-139 110 110 110 202 202 202 F F F 7 140-159 101 101 101 189 189 189 F F F 8 160-179 110 110 110 202 202 202 T T T 9 180-199 110 110 110 200 200 200 F F F 10 200-219 106 106 106 197 197 197 F F F 11 220-239 104 104 104 195 195 195 F F F 12 240-259 104 104 104 195 195 195 F F F 13 260-279 104 104 104 195 195 195 F F F 14 280-299 103 103 103 192 192 192 F F F 15 300-319 108 108 108 198 198 198 F F F 16 320-339 108 108 108 198 198 198 F F F 17 340-359 109 109 109 199 199 199 F F F 18 360-379 110 110 110 205 205 205 F F F 19 380-399 110 110 110 205 205 205 F F F 20 400-419 109 109 109 199 199 199 F F F 21 420-439 106 106 106 197 197 197 T T T 22 440-459 110 110 110 205 205 205 F F F 23 460-479 109 109 109 199 199 199 T T T 24 480-499 106 106 106 197 197 197 F F F 25 500-519 108 108 108 198 198 198 F F F 26 520-539 108 108 108 198 198 198 F F F 27 540-559 98 98 98 185 185 185 F F F 28 560-579 110 110 110 202 202 202 F F F 29 580-599 110 110 110 204 204 204 F F F Max gen: 110, Max flush: 205 A Difference in extent metadata is expected here On loop 55 Verifying data now No change after verify Loop: 55 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 108 108 108 198 198 198 F F F 1 020-039 109 109 109 199 199 199 F F F 2 040-059 111 111 111 206 206 206 F F F 3 060-079 100 100 100 187 187 187 F F F 4 080-099 111 111 111 206 206 206 F F F 5 100-119 110 110 110 202 202 202 F F F 6 120-139 110 110 110 202 202 202 F F F 7 140-159 101 101 101 189 189 189 F F F 8 160-179 111 111 111 206 206 206 F F F 9 180-199 110 110 110 200 200 200 F F F 10 200-219 106 106 106 197 197 197 F F F 11 220-239 104 104 104 195 195 195 F F F 12 240-259 104 104 104 195 195 195 F F F 13 260-279 104 104 104 195 195 195 F F F 14 280-299 103 103 103 192 192 192 F F F 15 300-319 108 108 108 198 198 198 F F F 16 320-339 108 108 108 198 198 198 F F F 17 340-359 109 109 109 199 199 199 F F F 18 360-379 110 110 110 205 205 205 F F F 19 380-399 110 110 110 205 205 205 F F F 20 400-419 109 109 109 199 199 199 F F F 21 420-439 111 111 111 206 206 206 F F F 22 440-459 110 110 110 205 205 205 F F F 23 460-479 111 111 111 206 206 206 F F F 24 480-499 106 106 106 197 197 197 F F F 25 500-519 108 108 108 198 198 198 F F F 26 520-539 108 108 108 198 198 198 F F F 27 540-559 98 98 98 185 185 185 F F F 28 560-579 110 110 110 202 202 202 F F F 29 580-599 110 110 110 204 204 204 F F F Max gen: 111, Max flush: 206 Begin loop 56 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:17.796002875Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1630,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:17.840052528Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1630,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:17.840788137Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1630} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"26e7d501b33f47379e53601720353c2983e7654b\",\n git_commit_timestamp: \"2024-12-12T20:16:40.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:17.840822714Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1630} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:17.840837473Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1630} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:17.840851692Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1630} {"msg":"Crucible 05501407-042b-4ea2-855c-926559405112 has session id: 0a7a6437-59dd-4d5e-a337-bdb88b7a2472","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:17.841285862Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1630,"session_id":"0a7a6437-59dd-4d5e-a337-bdb88b7a2472"} {"msg":"Upstairs opts: Upstairs UUID: 05501407-042b-4ea2-855c-926559405112, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:17.841337169Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1630,"session_id":"0a7a6437-59dd-4d5e-a337-bdb88b7a2472"} {"msg":"Crucible stats registered with UUID: 05501407-042b-4ea2-855c-926559405112","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:17.841363127Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1630,"session_id":"0a7a6437-59dd-4d5e-a337-bdb88b7a2472"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:17.87264412Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1630,"task":"crutest"} {"msg":"The guest has requested activation with gen:112","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:17.872688687Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1630} {"msg":"05501407-042b-4ea2-855c-926559405112 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:17.872751053Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1630,"session_id":"0a7a6437-59dd-4d5e-a337-bdb88b7a2472"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:17.872779031Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1630,"client":"0","":"downstairs","session_id":"0a7a6437-59dd-4d5e-a337-bdb88b7a2472"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:17.872802409Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1630,"client":"0","":"downstairs","session_id":"0a7a6437-59dd-4d5e-a337-bdb88b7a2472"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:17.872817068Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1630,"client":"1","":"downstairs","session_id":"0a7a6437-59dd-4d5e-a337-bdb88b7a2472"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:17.872831617Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1630,"client":"1","":"downstairs","session_id":"0a7a6437-59dd-4d5e-a337-bdb88b7a2472"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:17.872910712Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1630,"client":"2","":"downstairs","session_id":"0a7a6437-59dd-4d5e-a337-bdb88b7a2472"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:17.872949259Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1630,"client":"2","":"downstairs","session_id":"0a7a6437-59dd-4d5e-a337-bdb88b7a2472"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:17.872978247Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1630,"":"io task","client":"2","":"downstairs","session_id":"0a7a6437-59dd-4d5e-a337-bdb88b7a2472"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:17.873007025Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1630,"":"io task","client":"0","":"downstairs","session_id":"0a7a6437-59dd-4d5e-a337-bdb88b7a2472"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:17.873033793Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1630,"":"io task","client":"1","":"downstairs","session_id":"0a7a6437-59dd-4d5e-a337-bdb88b7a2472"} {"msg":"ds_connection connected from Ok(127.0.0.1:50235)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:17.873067961Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1630,"":"io task","client":"0","":"downstairs","session_id":"0a7a6437-59dd-4d5e-a337-bdb88b7a2472"} {"msg":"ds_connection connected from Ok(127.0.0.1:60331)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:17.873101178Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1630,"":"io task","client":"2","":"downstairs","session_id":"0a7a6437-59dd-4d5e-a337-bdb88b7a2472"} {"msg":"ds_connection connected from Ok(127.0.0.1:43606)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:17.873177573Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1630,"":"io task","client":"1","":"downstairs","session_id":"0a7a6437-59dd-4d5e-a337-bdb88b7a2472"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:17.873756763Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1630,"client":"0","":"downstairs","session_id":"0a7a6437-59dd-4d5e-a337-bdb88b7a2472"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:17.873785221Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1630,"client":"1","":"downstairs","session_id":"0a7a6437-59dd-4d5e-a337-bdb88b7a2472"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:17.87380886Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1630,"client":"2","":"downstairs","session_id":"0a7a6437-59dd-4d5e-a337-bdb88b7a2472"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:17.874451405Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1630,"client":"1","":"downstairs","session_id":"0a7a6437-59dd-4d5e-a337-bdb88b7a2472"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:17.874485723Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1630,"client":"2","":"downstairs","session_id":"0a7a6437-59dd-4d5e-a337-bdb88b7a2472"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:17.874515861Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1630,"client":"0","":"downstairs","session_id":"0a7a6437-59dd-4d5e-a337-bdb88b7a2472"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:17.875065503Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1630,"session_id":"0a7a6437-59dd-4d5e-a337-bdb88b7a2472"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:17.8751014Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1630,"session_id":"0a7a6437-59dd-4d5e-a337-bdb88b7a2472"} {"msg":"[0]R flush_numbers[0..12]: [198, 199, 206, 187, 206, 202, 202, 189, 206, 200, 197, 195]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:17.875124389Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1630,"":"downstairs","":"downstairs","session_id":"0a7a6437-59dd-4d5e-a337-bdb88b7a2472"} {"msg":"[0]R generation[0..12]: [108, 109, 111, 100, 111, 110, 110, 101, 111, 110, 106, 104]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:17.875140198Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1630,"":"downstairs","":"downstairs","session_id":"0a7a6437-59dd-4d5e-a337-bdb88b7a2472"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:17.875154937Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1630,"":"downstairs","":"downstairs","session_id":"0a7a6437-59dd-4d5e-a337-bdb88b7a2472"} {"msg":"[1]R flush_numbers[0..12]: [198, 199, 206, 187, 206, 202, 202, 189, 206, 200, 197, 195]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:17.875172086Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1630,"":"downstairs","":"downstairs","session_id":"0a7a6437-59dd-4d5e-a337-bdb88b7a2472"} {"msg":"[1]R generation[0..12]: [108, 109, 111, 100, 111, 110, 110, 101, 111, 110, 106, 104]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:17.875186205Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1630,"":"downstairs","":"downstairs","session_id":"0a7a6437-59dd-4d5e-a337-bdb88b7a2472"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:17.875200284Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1630,"":"downstairs","":"downstairs","session_id":"0a7a6437-59dd-4d5e-a337-bdb88b7a2472"} {"msg":"[2]R flush_numbers[0..12]: [198, 199, 206, 187, 206, 202, 202, 189, 206, 200, 197, 195]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:17.875214073Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1630,"":"downstairs","":"downstairs","session_id":"0a7a6437-59dd-4d5e-a337-bdb88b7a2472"} {"msg":"[2]R generation[0..12]: [108, 109, 111, 100, 111, 110, 110, 101, 111, 110, 106, 104]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:17.875227692Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1630,"":"downstairs","":"downstairs","session_id":"0a7a6437-59dd-4d5e-a337-bdb88b7a2472"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:17.875245751Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1630,"":"downstairs","":"downstairs","session_id":"0a7a6437-59dd-4d5e-a337-bdb88b7a2472"} {"msg":"Max found gen is 112","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:17.875270299Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1630,"":"downstairs","":"downstairs","session_id":"0a7a6437-59dd-4d5e-a337-bdb88b7a2472"} {"msg":"Generation requested: 112 >= found:112","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:17.875296677Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1630,"":"downstairs","":"downstairs","session_id":"0a7a6437-59dd-4d5e-a337-bdb88b7a2472"} {"msg":"Next flush: 207","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:17.875322885Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1630,"":"downstairs","":"downstairs","session_id":"0a7a6437-59dd-4d5e-a337-bdb88b7a2472"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:17.875348503Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1630,"":"downstairs","":"downstairs","session_id":"0a7a6437-59dd-4d5e-a337-bdb88b7a2472"} {"msg":"Wait for a query_work_queue command to finish before sending IO No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:17.875379272Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1630,"session_id":"0a7a6437-59dd-4d5e-a337-bdb88b7a2472"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:17.87540339Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1630,"client":"0","":"downstairs","session_id":"0a7a6437-59dd-4d5e-a337-bdb88b7a2472"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:17.875430468Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1630,"client":"1","":"downstairs","session_id":"0a7a6437-59dd-4d5e-a337-bdb88b7a2472"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:17.875457156Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1630,"client":"2","":"downstairs","session_id":"0a7a6437-59dd-4d5e-a337-bdb88b7a2472"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:17.875485734Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1630,"session_id":"0a7a6437-59dd-4d5e-a337-bdb88b7a2472"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:17.875515952Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1630,"session_id":"0a7a6437-59dd-4d5e-a337-bdb88b7a2472"} {"msg":"05501407-042b-4ea2-855c-926559405112 is now active with session: 0a7a6437-59dd-4d5e-a337-bdb88b7a2472","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:17.87554436Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1630,"session_id":"0a7a6437-59dd-4d5e-a337-bdb88b7a2472"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:17.875570068Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1630,"session_id":"0a7a6437-59dd-4d5e-a337-bdb88b7a2472"} {"msg":"The guest has finished waiting for activation with:112","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:17.875595677Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 1630} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 35 len 8192 data: 12 11 02/30 Read block 139 len 36864 03/30 Flush 04/30 Write block 219 len 32768 data: 10 10 7 5 7 6 8 11 05/30 Read block 270 len 12288 06/30 Read block 556 len 4096 07/30 Read block 323 len 8192 08/30 Read block 301 len 8192 09/30 Read block 256 len 8192 10/30 Read block 228 len 28672 11/30 Write block 30 len 12288 data: 7 7 8 12/30 Write block 427 len 28672 data: 9 9 10 7 9 8 8 13/30 Read block 192 len 24576 14/30 Read block 73 len 16384 15/30 Read block 548 len 12288 16/30 Write block 306 len 4096 data: 5 17/30 Write block 571 len 12288 data: 10 9 9 18/30 Flush 19/30 Write block 211 len 4096 data: 11 20/30 Flush 21/30 Write block 281 len 16384 data: 8 7 7 7 22/30 Flush 23/30 Write block 193 len 4096 data: 12 24/30 Write block 458 len 32768 data: 12 13 12 16 17 15 13 12 25/30 Write block 55 len 36864 data: 13 16 16 15 15 14 9 9 10 26/30 Read block 425 len 12288 27/30 Read block 65 len 8192 28/30 Read block 569 len 24576 29/30 Write block 205 len 28672 data: 13 13 8 8 7 10 12 30/30 Read block 503 len 32768 ---------------------------------------------------------------- Crucible gen:112 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 1 Done Done Done false 1023 Acked Write 8 Done Done Done false 1024 Acked Write 9 Done Done Done false 1025 Acked Read 3 Done Done Done false 1026 Acked Read 2 Done Done Done false 1027 Acked Read 6 Done Done Done false 1028 Acked Write 7 Done Done Done false 1029 Acked Read 8 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 1 2 Done 7 8 7 22 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1021 1021 1021 Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:37:17.893847228Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1630} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 108 108 108 198 198 198 F F F 1 020-039 112 112 112 208 208 208 F F F 2 040-059 111 111 111 206 206 206 T T T 3 060-079 100 100 100 187 187 187 T T T 4 080-099 111 111 111 206 206 206 F F F 5 100-119 110 110 110 202 202 202 F F F 6 120-139 110 110 110 202 202 202 F F F 7 140-159 101 101 101 189 189 189 F F F 8 160-179 111 111 111 206 206 206 F F F 9 180-199 110 110 110 200 200 200 T T T 10 200-219 112 112 112 209 209 209 T T T 11 220-239 112 112 112 208 208 208 F F F 12 240-259 104 104 104 195 195 195 F F F 13 260-279 104 104 104 195 195 195 F F F 14 280-299 112 112 112 210 210 210 F F F 15 300-319 112 112 112 208 208 208 F F F 16 320-339 108 108 108 198 198 198 F F F 17 340-359 109 109 109 199 199 199 F F F 18 360-379 110 110 110 205 205 205 F F F 19 380-399 110 110 110 205 205 205 F F F 20 400-419 109 109 109 199 199 199 F F F 21 420-439 112 112 112 208 208 208 F F F 22 440-459 110 110 110 205 205 205 T T T 23 460-479 111 111 111 206 206 206 T T T 24 480-499 106 106 106 197 197 197 F F F 25 500-519 108 108 108 198 198 198 F F F 26 520-539 108 108 108 198 198 198 F F F 27 540-559 98 98 98 185 185 185 F F F 28 560-579 112 112 112 208 208 208 F F F 29 580-599 110 110 110 204 204 204 F F F Max gen: 112, Max flush: 210 A Difference in extent metadata is expected here On loop 56 Verifying data now No change after verify Loop: 56 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 108 108 108 198 198 198 F F F 1 020-039 112 112 112 208 208 208 F F F 2 040-059 113 113 113 211 211 211 F F F 3 060-079 113 113 113 211 211 211 F F F 4 080-099 111 111 111 206 206 206 F F F 5 100-119 110 110 110 202 202 202 F F F 6 120-139 110 110 110 202 202 202 F F F 7 140-159 101 101 101 189 189 189 F F F 8 160-179 111 111 111 206 206 206 F F F 9 180-199 113 113 113 211 211 211 F F F 10 200-219 113 113 113 211 211 211 F F F 11 220-239 112 112 112 208 208 208 F F F 12 240-259 104 104 104 195 195 195 F F F 13 260-279 104 104 104 195 195 195 F F F 14 280-299 112 112 112 210 210 210 F F F 15 300-319 112 112 112 208 208 208 F F F 16 320-339 108 108 108 198 198 198 F F F 17 340-359 109 109 109 199 199 199 F F F 18 360-379 110 110 110 205 205 205 F F F 19 380-399 110 110 110 205 205 205 F F F 20 400-419 109 109 109 199 199 199 F F F 21 420-439 112 112 112 208 208 208 F F F 22 440-459 113 113 113 211 211 211 F F F 23 460-479 113 113 113 211 211 211 F F F 24 480-499 106 106 106 197 197 197 F F F 25 500-519 108 108 108 198 198 198 F F F 26 520-539 108 108 108 198 198 198 F F F 27 540-559 98 98 98 185 185 185 F F F 28 560-579 112 112 112 208 208 208 F F F 29 580-599 110 110 110 204 204 204 F F F Max gen: 113, Max flush: 211 Begin loop 57 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:22.172545525Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1638,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:37:22.215511412Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1638,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:22.215914205Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1638,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:22.247359086Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1638,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:22.247963375Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1638} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"26e7d501b33f47379e53601720353c2983e7654b\",\n git_commit_timestamp: \"2024-12-12T20:16:40.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:22.247999312Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1638} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:22.248013351Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1638} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:22.2480275Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1638} {"msg":"Crucible 41b88dc0-ca79-4e02-a7c0-29a58ce276e8 has session id: f213a39f-168a-40a9-a25a-0ca89980f6b0","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:22.248436812Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1638,"session_id":"f213a39f-168a-40a9-a25a-0ca89980f6b0"} {"msg":"Upstairs opts: Upstairs UUID: 41b88dc0-ca79-4e02-a7c0-29a58ce276e8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:22.248481439Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1638,"session_id":"f213a39f-168a-40a9-a25a-0ca89980f6b0"} {"msg":"Crucible stats registered with UUID: 41b88dc0-ca79-4e02-a7c0-29a58ce276e8","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:22.248503727Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1638,"session_id":"f213a39f-168a-40a9-a25a-0ca89980f6b0"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:22.279096808Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1638,"task":"crutest"} {"msg":"The guest has requested activation with gen:114","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:22.279126956Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1638} {"msg":"41b88dc0-ca79-4e02-a7c0-29a58ce276e8 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:22.279175332Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1638,"session_id":"f213a39f-168a-40a9-a25a-0ca89980f6b0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:22.279195851Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1638,"client":"0","":"downstairs","session_id":"f213a39f-168a-40a9-a25a-0ca89980f6b0"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:22.279223479Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1638,"client":"0","":"downstairs","session_id":"f213a39f-168a-40a9-a25a-0ca89980f6b0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:22.279240988Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1638,"client":"1","":"downstairs","session_id":"f213a39f-168a-40a9-a25a-0ca89980f6b0"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:22.279258606Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1638,"client":"1","":"downstairs","session_id":"f213a39f-168a-40a9-a25a-0ca89980f6b0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:22.279282175Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1638,"client":"2","":"downstairs","session_id":"f213a39f-168a-40a9-a25a-0ca89980f6b0"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:22.279314743Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1638,"client":"2","":"downstairs","session_id":"f213a39f-168a-40a9-a25a-0ca89980f6b0"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:22.27934648Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1638,"":"io task","client":"2","":"downstairs","session_id":"f213a39f-168a-40a9-a25a-0ca89980f6b0"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:22.279378708Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1638,"":"io task","client":"0","":"downstairs","session_id":"f213a39f-168a-40a9-a25a-0ca89980f6b0"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:22.279412096Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1638,"":"io task","client":"1","":"downstairs","session_id":"f213a39f-168a-40a9-a25a-0ca89980f6b0"} {"msg":"ds_connection connected from Ok(127.0.0.1:38577)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:22.279587584Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1638,"":"io task","client":"2","":"downstairs","session_id":"f213a39f-168a-40a9-a25a-0ca89980f6b0"} {"msg":"ds_connection connected from Ok(127.0.0.1:51597)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:22.279621421Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1638,"":"io task","client":"1","":"downstairs","session_id":"f213a39f-168a-40a9-a25a-0ca89980f6b0"} {"msg":"ds_connection connected from Ok(127.0.0.1:33817)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:22.27964612Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1638,"":"io task","client":"0","":"downstairs","session_id":"f213a39f-168a-40a9-a25a-0ca89980f6b0"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:22.280119487Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1638,"client":"2","":"downstairs","session_id":"f213a39f-168a-40a9-a25a-0ca89980f6b0"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:22.280149345Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1638,"client":"1","":"downstairs","session_id":"f213a39f-168a-40a9-a25a-0ca89980f6b0"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:22.280323763Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1638,"client":"0","":"downstairs","session_id":"f213a39f-168a-40a9-a25a-0ca89980f6b0"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:22.280791801Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1638,"client":"1","":"downstairs","session_id":"f213a39f-168a-40a9-a25a-0ca89980f6b0"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:22.280823559Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1638,"client":"2","":"downstairs","session_id":"f213a39f-168a-40a9-a25a-0ca89980f6b0"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:22.280839537Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1638,"client":"0","":"downstairs","session_id":"f213a39f-168a-40a9-a25a-0ca89980f6b0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:22.281373971Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1638,"session_id":"f213a39f-168a-40a9-a25a-0ca89980f6b0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:22.281399559Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1638,"session_id":"f213a39f-168a-40a9-a25a-0ca89980f6b0"} {"msg":"[0]R flush_numbers[0..12]: [198, 208, 211, 211, 206, 202, 202, 189, 206, 211, 211, 208]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:22.281423797Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1638,"":"downstairs","":"downstairs","session_id":"f213a39f-168a-40a9-a25a-0ca89980f6b0"} {"msg":"[0]R generation[0..12]: [108, 112, 113, 113, 111, 110, 110, 101, 111, 113, 113, 112]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:22.281438956Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1638,"":"downstairs","":"downstairs","session_id":"f213a39f-168a-40a9-a25a-0ca89980f6b0"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:22.281453315Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1638,"":"downstairs","":"downstairs","session_id":"f213a39f-168a-40a9-a25a-0ca89980f6b0"} {"msg":"[1]R flush_numbers[0..12]: [198, 208, 211, 211, 206, 202, 202, 189, 206, 211, 211, 208]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:22.281467154Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1638,"":"downstairs","":"downstairs","session_id":"f213a39f-168a-40a9-a25a-0ca89980f6b0"} {"msg":"[1]R generation[0..12]: [108, 112, 113, 113, 111, 110, 110, 101, 111, 113, 113, 112]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:22.281480963Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1638,"":"downstairs","":"downstairs","session_id":"f213a39f-168a-40a9-a25a-0ca89980f6b0"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:22.281495022Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1638,"":"downstairs","":"downstairs","session_id":"f213a39f-168a-40a9-a25a-0ca89980f6b0"} {"msg":"[2]R flush_numbers[0..12]: [198, 208, 211, 211, 206, 202, 202, 189, 206, 211, 211, 208]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:22.281508602Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1638,"":"downstairs","":"downstairs","session_id":"f213a39f-168a-40a9-a25a-0ca89980f6b0"} {"msg":"[2]R generation[0..12]: [108, 112, 113, 113, 111, 110, 110, 101, 111, 113, 113, 112]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:22.281522241Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1638,"":"downstairs","":"downstairs","session_id":"f213a39f-168a-40a9-a25a-0ca89980f6b0"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:22.281535909Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1638,"":"downstairs","":"downstairs","session_id":"f213a39f-168a-40a9-a25a-0ca89980f6b0"} {"msg":"Max found gen is 114","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:22.281548769Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1638,"":"downstairs","":"downstairs","session_id":"f213a39f-168a-40a9-a25a-0ca89980f6b0"} {"msg":"Generation requested: 114 >= found:114","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:22.281561758Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1638,"":"downstairs","":"downstairs","session_id":"f213a39f-168a-40a9-a25a-0ca89980f6b0"} {"msg":"Next flush: 212","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:22.281574577Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1638,"":"downstairs","":"downstairs","session_id":"f213a39f-168a-40a9-a25a-0ca89980f6b0"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:22.281591076Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1638,"":"downstairs","":"downstairs","session_id":"f213a39f-168a-40a9-a25a-0ca89980f6b0"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:22.281604545Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1638,"session_id":"f213a39f-168a-40a9-a25a-0ca89980f6b0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:22.281616664Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1638,"client":"0","":"downstairs","session_id":"f213a39f-168a-40a9-a25a-0ca89980f6b0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:22.281629813Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1638,"client":"1","":"downstairs","session_id":"f213a39f-168a-40a9-a25a-0ca89980f6b0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:22.281642822Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1638,"client":"2","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO f213a39f-168a-40a9-a25a-0ca89980f6b0"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:22.281665711Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1638,"session_id":"f213a39f-168a-40a9-a25a-0ca89980f6b0"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:22.28167795Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1638,"session_id":"f213a39f-168a-40a9-a25a-0ca89980f6b0"} {"msg":"41b88dc0-ca79-4e02-a7c0-29a58ce276e8 is now active with session: f213a39f-168a-40a9-a25a-0ca89980f6b0","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:22.281690589Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1638,"session_id":"f213a39f-168a-40a9-a25a-0ca89980f6b0"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:22.281702478Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1638,"session_id":"f213a39f-168a-40a9-a25a-0ca89980f6b0"} {"msg":"The guest has finished waiting for activation with:114","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:22.281714597Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1638} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 86 len 40960 data: 7 12 14 12 12 10 7 10 9 8 02/30 Write block 295 len 8192 data: 10 9 03/30 Read block 141 len 24576 04/30 Flush 05/30 Write block 456 len 20480 data: 10 11 13 14 13 06/30 Write block 112 len 8192 data: 13 11 07/30 Flush 08/30 Flush 09/30 Write block 66 len 4096 data: 8 10/30 Flush 11/30 Write block 465 len 12288 data: 13 11 9 12/30 Write block 500 len 28672 data: 5 7 9 9 9 8 8 13/30 Read block 320 len 40960 14/30 Read block 154 len 12288 15/30 Write block 53 len 24576 data: 10 11 14 17 17 16 16/30 Read block 433 len 40960 17/30 Read block 143 len 8192 18/30 Write block 449 len 16384 data: 13 12 13 13 19/30 Read block 301 len 12288 20/30 Read block 447 len 24576 21/30 Read block 20 len 40960 22/30 Read block 162 len 8192 23/30 Read block 368 len 24576 24/30 Read block 285 len 4096 25/30 Write block 44 len 28672 data: 6 8 7 7 7 10 10 26/30 Write block 315 len 32768 data: 13 13 12 11 8 8 8 8 27/30 Write block 266 len 40960 data: 14 14 13 7 8 9 11 10 11 10 28/30 Write block 316 len 36864 data: 14 13 12 9 9 9 9 7 6 29/30 Read block 142 len 8192 30/30 Write block 455 len 8192 data: 10 11 ---------------------------------------------------------------- Crucible gen:114 GIO:true work queues: Upstairs:0 downstairs:20 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1010 Acked Write 3 Done Done Done false 1011 Acked Write 7 Done Done Done false 1012 Acked Read 10 Done Done Done false 1013 Acked Read 3 Done Done Done false 1014 Acked Write 6 Done Done Done false 1015 Acked Read 10 Done Done Done false 1016 Acked Read 2 Done Done Done false 1017 Acked Write 4 Done Done Done false 1018 Acked Read 3 Done Done Done false 1019 Acked Read 6 Done Done Done false 1020 Acked Read 10 Done Done Done false 1021 Acked Read 2 Done Done Done false 1022 Acked Read 6 Done Done Done false 1023 Acked Read 1 Done Done Done false 1024 Acked Write 7 Done Done Done false 1025 Acked Write 8 Done Done Done false 1026 Acked Write 10 Done Done Done false 1027 Acked Write 9 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 19 19 19 57 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1009 1009 1009 Downstairs last five completed: 1009 1008 1007 1006 1005 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:37:22.300620944Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1638} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 108 108 108 198 198 198 F F F 1 020-039 112 112 112 208 208 208 F F F 2 040-059 113 113 113 211 211 211 T T T 3 060-079 114 114 114 215 215 215 F F F 4 080-099 114 114 114 212 212 212 F F F 5 100-119 114 114 114 213 213 213 F F F 6 120-139 110 110 110 202 202 202 F F F 7 140-159 101 101 101 189 189 189 F F F 8 160-179 111 111 111 206 206 206 F F F 9 180-199 113 113 113 211 211 211 F F F 10 200-219 113 113 113 211 211 211 F F F 11 220-239 112 112 112 208 208 208 F F F 12 240-259 104 104 104 195 195 195 F F F 13 260-279 104 104 104 195 195 195 T T T 14 280-299 114 114 114 212 212 212 F F F 15 300-319 112 112 112 208 208 208 T T T 16 320-339 108 108 108 198 198 198 T T T 17 340-359 109 109 109 199 199 199 F F F 18 360-379 110 110 110 205 205 205 F F F 19 380-399 110 110 110 205 205 205 F F F 20 400-419 109 109 109 199 199 199 F F F 21 420-439 112 112 112 208 208 208 F F F 22 440-459 114 114 114 213 213 213 T T T 23 460-479 114 114 114 213 213 213 T T T 24 480-499 106 106 106 197 197 197 F F F 25 500-519 108 108 108 198 198 198 T T T 26 520-539 108 108 108 198 198 198 F F F 27 540-559 98 98 98 185 185 185 F F F 28 560-579 112 112 112 208 208 208 F F F 29 580-599 110 110 110 204 204 204 F F F Max gen: 114, Max flush: 215 A Difference in extent metadata is expected here On loop 57 Verifying data now No change after verify Loop: 57 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 108 108 108 198 198 198 F F F 1 020-039 112 112 112 208 208 208 F F F 2 040-059 115 115 115 216 216 216 F F F 3 060-079 114 114 114 215 215 215 F F F 4 080-099 114 114 114 212 212 212 F F F 5 100-119 114 114 114 213 213 213 F F F 6 120-139 110 110 110 202 202 202 F F F 7 140-159 101 101 101 189 189 189 F F F 8 160-179 111 111 111 206 206 206 F F F 9 180-199 113 113 113 211 211 211 F F F 10 200-219 113 113 113 211 211 211 F F F 11 220-239 112 112 112 208 208 208 F F F 12 240-259 104 104 104 195 195 195 F F F 13 260-279 115 115 115 216 216 216 F F F 14 280-299 114 114 114 212 212 212 F F F 15 300-319 115 115 115 216 216 216 F F F 16 320-339 115 115 115 216 216 216 F F F 17 340-359 109 109 109 199 199 199 F F F 18 360-379 110 110 110 205 205 205 F F F 19 380-399 110 110 110 205 205 205 F F F 20 400-419 109 109 109 199 199 199 F F F 21 420-439 112 112 112 208 208 208 F F F 22 440-459 115 115 115 216 216 216 F F F 23 460-479 115 115 115 216 216 216 F F F 24 480-499 106 106 106 197 197 197 F F F 25 500-519 115 115 115 216 216 216 F F F 26 520-539 108 108 108 198 198 198 F F F 27 540-559 98 98 98 185 185 185 F F F 28 560-579 112 112 112 208 208 208 F F F 29 580-599 110 110 110 204 204 204 F F F Max gen: 115, Max flush: 216 Begin loop 58 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:26.609396048Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1646,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:26.653718151Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1646,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:26.654481429Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1646} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"26e7d501b33f47379e53601720353c2983e7654b\",\n git_commit_timestamp: \"2024-12-12T20:16:40.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:26.654518806Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1646} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:26.654533155Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1646} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:26.654547544Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1646} {"msg":"Crucible 04c78ba1-6231-4864-a77a-a8331da48d57 has session id: 371a072b-f91f-4192-b5e5-74020ed853a8","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:26.654994313Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1646,"session_id":"371a072b-f91f-4192-b5e5-74020ed853a8"} {"msg":"Upstairs opts: Upstairs UUID: 04c78ba1-6231-4864-a77a-a8331da48d57, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:26.65503952Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1646,"session_id":"371a072b-f91f-4192-b5e5-74020ed853a8"} {"msg":"Crucible stats registered with UUID: 04c78ba1-6231-4864-a77a-a8331da48d57","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:26.655060669Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1646,"session_id":"371a072b-f91f-4192-b5e5-74020ed853a8"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:26.68651081Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1646,"task":"crutest"} {"msg":"The guest has requested activation with gen:116","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:26.686541088Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1646} {"msg":"04c78ba1-6231-4864-a77a-a8331da48d57 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:26.686620832Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1646,"session_id":"371a072b-f91f-4192-b5e5-74020ed853a8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:26.686644021Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1646,"client":"0","":"downstairs","session_id":"371a072b-f91f-4192-b5e5-74020ed853a8"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:26.686679908Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1646,"client":"0","":"downstairs","session_id":"371a072b-f91f-4192-b5e5-74020ed853a8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:26.686714296Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1646,"client":"1","":"downstairs","session_id":"371a072b-f91f-4192-b5e5-74020ed853a8"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:26.686751234Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1646,"client":"1","":"downstairs","session_id":"371a072b-f91f-4192-b5e5-74020ed853a8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:26.686784881Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1646,"client":"2","":"downstairs","session_id":"371a072b-f91f-4192-b5e5-74020ed853a8"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:26.686819949Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1646,"client":"2","":"downstairs","session_id":"371a072b-f91f-4192-b5e5-74020ed853a8"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:26.686852567Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1646,"":"io task","client":"2","":"downstairs","session_id":"371a072b-f91f-4192-b5e5-74020ed853a8"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:26.686884294Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1646,"":"io task","client":"0","":"downstairs","session_id":"371a072b-f91f-4192-b5e5-74020ed853a8"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:26.686916622Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1646,"":"io task","client":"1","":"downstairs","session_id":"371a072b-f91f-4192-b5e5-74020ed853a8"} {"msg":"ds_connection connected from Ok(127.0.0.1:64842)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:26.68694851Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1646,"":"io task","client":"2","":"downstairs","session_id":"371a072b-f91f-4192-b5e5-74020ed853a8"} {"msg":"ds_connection connected from Ok(127.0.0.1:33821)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:26.686980498Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1646,"":"io task","client":"0","":"downstairs","session_id":"371a072b-f91f-4192-b5e5-74020ed853a8"} {"msg":"ds_connection connected from Ok(127.0.0.1:65165)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:26.687015225Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1646,"":"io task","client":"1","":"downstairs","session_id":"371a072b-f91f-4192-b5e5-74020ed853a8"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:26.687552178Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1646,"client":"2","":"downstairs","session_id":"371a072b-f91f-4192-b5e5-74020ed853a8"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:26.687734286Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1646,"client":"0","":"downstairs","session_id":"371a072b-f91f-4192-b5e5-74020ed853a8"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:26.687881406Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1646,"client":"1","":"downstairs","session_id":"371a072b-f91f-4192-b5e5-74020ed853a8"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:26.688358513Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1646,"client":"0","":"downstairs","session_id":"371a072b-f91f-4192-b5e5-74020ed853a8"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:26.688388001Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1646,"client":"2","":"downstairs","session_id":"371a072b-f91f-4192-b5e5-74020ed853a8"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:26.68854437Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1646,"client":"1","":"downstairs","session_id":"371a072b-f91f-4192-b5e5-74020ed853a8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:26.688990439Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1646,"session_id":"371a072b-f91f-4192-b5e5-74020ed853a8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:26.689024917Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1646,"session_id":"371a072b-f91f-4192-b5e5-74020ed853a8"} {"msg":"[0]R flush_numbers[0..12]: [198, 208, 216, 215, 212, 213, 202, 189, 206, 211, 211, 208]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:26.689069684Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1646,"":"downstairs","":"downstairs","session_id":"371a072b-f91f-4192-b5e5-74020ed853a8"} {"msg":"[0]R generation[0..12]: [108, 112, 115, 114, 114, 114, 110, 101, 111, 113, 113, 112]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:26.689088932Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1646,"":"downstairs","":"downstairs","session_id":"371a072b-f91f-4192-b5e5-74020ed853a8"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:26.689103941Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1646,"":"downstairs","":"downstairs","session_id":"371a072b-f91f-4192-b5e5-74020ed853a8"} {"msg":"[1]R flush_numbers[0..12]: [198, 208, 216, 215, 212, 213, 202, 189, 206, 211, 211, 208]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:26.68911784Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1646,"":"downstairs","":"downstairs","session_id":"371a072b-f91f-4192-b5e5-74020ed853a8"} {"msg":"[1]R generation[0..12]: [108, 112, 115, 114, 114, 114, 110, 101, 111, 113, 113, 112]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:26.68913167Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1646,"":"downstairs","":"downstairs","session_id":"371a072b-f91f-4192-b5e5-74020ed853a8"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:26.689148818Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1646,"":"downstairs","":"downstairs","session_id":"371a072b-f91f-4192-b5e5-74020ed853a8"} {"msg":"[2]R flush_numbers[0..12]: [198, 208, 216, 215, 212, 213, 202, 189, 206, 211, 211, 208]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:26.689162877Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1646,"":"downstairs","":"downstairs","session_id":"371a072b-f91f-4192-b5e5-74020ed853a8"} {"msg":"[2]R generation[0..12]: [108, 112, 115, 114, 114, 114, 110, 101, 111, 113, 113, 112]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:26.689176466Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1646,"":"downstairs","":"downstairs","session_id":"371a072b-f91f-4192-b5e5-74020ed853a8"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:26.689190405Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1646,"":"downstairs","":"downstairs","session_id":"371a072b-f91f-4192-b5e5-74020ed853a8"} {"msg":"Max found gen is 116","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:26.689203145Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1646,"":"downstairs","":"downstairs","session_id":"371a072b-f91f-4192-b5e5-74020ed853a8"} {"msg":"Generation requested: 116 >= found:116","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:26.689216274Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1646,"":"downstairs","":"downstairs","session_id":"371a072b-f91f-4192-b5e5-74020ed853a8"} {"msg":"Next flush: 217","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:26.689229193Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1646,"":"downstairs","":"downstairs","session_id":"371a072b-f91f-4192-b5e5-74020ed853a8"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:26.689241942Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1646,"":"downstairs","":"downstairs","session_id":"371a072b-f91f-4192-b5e5-74020ed853a8"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-12-12T20:37:26.689260561Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1646,"session_id":"371a072b-f91f-4192-b5e5-74020ed853a8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:26.689291968Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1646,"client":"0","":"downstairs","session_id":"371a072b-f91f-4192-b5e5-74020ed853a8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:26.689318806Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1646,"client":"1","":"downstairs","session_id":"371a072b-f91f-4192-b5e5-74020ed853a8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:26.689338005Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1646,"client":"2","":"downstairs","session_id":"371a072b-f91f-4192-b5e5-74020ed853a8"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:26.689351764Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1646,"session_id":"371a072b-f91f-4192-b5e5-74020ed853a8"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:26.689373143Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1646,"session_id":"371a072b-f91f-4192-b5e5-74020ed853a8"} {"msg":"04c78ba1-6231-4864-a77a-a8331da48d57 is now active with session: 371a072b-f91f-4192-b5e5-74020ed853a8","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:26.689397951Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1646,"session_id":"371a072b-f91f-4192-b5e5-74020ed853a8"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:26.689425989Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1646,"session_id":"371a072b-f91f-4192-b5e5-74020ed853a8"} {"msg":"The guest has finished waiting for activation with:116","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:26.689440818Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1646} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 208 len 28672 data: 9 8 11 13 11 12 10 02/30 Read block 58 len 8192 03/30 Flush 04/30 Write block 213 len 24576 data: 13 11 10 7 9 8 05/30 Flush 06/30 Write block 137 len 8192 data: 10 12 07/30 Read block 515 len 24576 08/30 Read block 204 len 16384 09/30 Write block 394 len 32768 data: 9 11 11 12 13 14 12 13 10/30 Write block 208 len 16384 data: 10 9 12 14 11/30 Read block 435 len 20480 12/30 Read block 99 len 40960 13/30 Read block 223 len 12288 14/30 Read block 570 len 4096 15/30 Read block 244 len 12288 16/30 Write block 444 len 32768 data: 8 7 10 12 11 14 13 14 17/30 Read block 286 len 4096 18/30 Read block 360 len 16384 19/30 Write block 588 len 12288 data: 11 10 15 20/30 Read block 193 len 28672 21/30 Write block 447 len 24576 data: 13 12 15 14 15 14 22/30 Read block 534 len 8192 23/30 Write block 23 len 36864 data: 7 9 8 7 8 8 8 8 8 24/30 Write block 98 len 16384 data: 4 7 9 8 25/30 Write block 79 len 40960 data: 8 9 8 5 6 7 8 8 13 15 26/30 Write block 314 len 40960 data: 13 14 15 14 13 10 10 10 10 8 27/30 Write block 394 len 24576 data: 10 12 12 13 14 15 28/30 Write block 182 len 32768 data: 8 9 12 13 12 11 12 12 29/30 Write block 273 len 40960 data: 11 12 11 9 9 11 9 8 9 8 30/30 Read block 492 len 8192 ---------------------------------------------------------------- Crucible gen:116 GIO:true work queues: Upstairs:0 downstairs:25 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1005 Acked Write 2 Done Done Done false 1006 Acked Read 6 Done Done Done false 1007 Acked Read 4 Done Done Done false 1008 Acked Write 8 Done Done Done false 1009 Acked Write 4 Done Done Done false 1010 Acked Read 5 Done Done Done false 1011 Acked Read 10 Done Done Done false 1012 Acked Read 3 Done Done Done false 1013 Acked Read 1 Done Done Done false 1014 Acked Read 3 Done Done Done false 1015 Acked Write 8 Done Done Done false 1016 Acked Read 1 Done Done Done false 1017 Acked Read 4 Done Done Done false 1018 Acked Write 3 Done Done Done false 1019 Acked Read 7 Done Done Done false 1020 Acked Write 6 Done Done Done false 1021 Acked Read 2 Done Done Done false 1022 Acked Write 9 Done Done Done false 1023 Acked Write 4 Done Done Done false 1024 Acked Write 10 Done Done Done false 1025 Acked Write 10 Done Done Done false 1026 Acked Write 6 Done Done Done false 1027 Acked Write 8 Done Done Done false 1028 Acked Write 10 Done Sent Sent false 1029 Acked Read 2 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 2 2 4 Done 25 23 23 71 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1004 1004 1004 Downstairs last five completed: 1004 1003 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:37:26.709013399Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1646} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 108 108 108 198 198 198 F F F 1 020-039 112 112 112 208 208 208 T T T 2 040-059 115 115 115 216 216 216 F F F 3 060-079 114 114 114 215 215 215 T T T 4 080-099 114 114 114 212 212 212 T T T 5 100-119 114 114 114 213 213 213 T T T 6 120-139 110 110 110 202 202 202 T T T 7 140-159 101 101 101 189 189 189 F F F 8 160-179 111 111 111 206 206 206 F F F 9 180-199 113 113 113 211 211 211 T T T 10 200-219 116 116 116 218 218 218 T T T 11 220-239 112 112 112 208 208 208 F F F 12 240-259 104 104 104 195 195 195 F F F 13 260-279 115 115 115 216 216 216 T T T 14 280-299 114 114 114 212 212 212 T T T 15 300-319 115 115 115 216 216 216 T T T 16 320-339 115 115 115 216 216 216 T T T 17 340-359 109 109 109 199 199 199 F F F 18 360-379 110 110 110 205 205 205 F F F 19 380-399 110 110 110 205 205 205 T T T 20 400-419 109 109 109 199 199 199 T T T 21 420-439 112 112 112 208 208 208 F F F 22 440-459 115 115 115 216 216 216 T T T 23 460-479 115 115 115 216 216 216 F F F 24 480-499 106 106 106 197 197 197 F F F 25 500-519 115 115 115 216 216 216 F F F 26 520-539 108 108 108 198 198 198 F F F 27 540-559 98 98 98 185 185 185 F F F 28 560-579 112 112 112 208 208 208 F F F 29 580-599 110 110 110 204 204 204 T T T Max gen: 116, Max flush: 218 A Difference in extent metadata is expected here On loop 58 Verifying data now No change after verify Loop: 58 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 108 108 108 198 198 198 F F F 1 020-039 117 117 117 219 219 219 F F F 2 040-059 115 115 115 216 216 216 F F F 3 060-079 117 117 117 219 219 219 F F F 4 080-099 117 117 117 219 219 219 F F F 5 100-119 117 117 117 219 219 219 F F F 6 120-139 117 117 117 219 219 219 F F F 7 140-159 101 101 101 189 189 189 F F F 8 160-179 111 111 111 206 206 206 F F F 9 180-199 117 117 117 219 219 219 F F F 10 200-219 117 117 117 219 219 219 F F F 11 220-239 112 112 112 208 208 208 F F F 12 240-259 104 104 104 195 195 195 F F F 13 260-279 117 117 117 219 219 219 F F F 14 280-299 117 117 117 219 219 219 F F F 15 300-319 117 117 117 219 219 219 F F F 16 320-339 117 117 117 219 219 219 F F F 17 340-359 109 109 109 199 199 199 F F F 18 360-379 110 110 110 205 205 205 F F F 19 380-399 117 117 117 219 219 219 F F F 20 400-419 117 117 117 219 219 219 F F F 21 420-439 112 112 112 208 208 208 F F F 22 440-459 117 117 117 219 219 219 F F F 23 460-479 115 115 115 216 216 216 F F F 24 480-499 106 106 106 197 197 197 F F F 25 500-519 115 115 115 216 216 216 F F F 26 520-539 108 108 108 198 198 198 F F F 27 540-559 98 98 98 185 185 185 F F F 28 560-579 112 112 112 208 208 208 F F F 29 580-599 117 117 117 219 219 219 F F F Max gen: 117, Max flush: 219 Begin loop 59 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:31.032935978Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1654,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:31.077347825Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1654,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:31.078025538Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1654} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"26e7d501b33f47379e53601720353c2983e7654b\",\n git_commit_timestamp: \"2024-12-12T20:16:40.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:31.078062736Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1654} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:31.078076875Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1654} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:31.078092504Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1654} {"msg":"Crucible 2c6983e0-3943-4359-a75b-b025e61b76fd has session id: 07aa956a-3604-4434-9e25-eb62547bc1d3","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:31.078514825Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1654,"session_id":"07aa956a-3604-4434-9e25-eb62547bc1d3"} {"msg":"Upstairs opts: Upstairs UUID: 2c6983e0-3943-4359-a75b-b025e61b76fd, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:31.078559992Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1654,"session_id":"07aa956a-3604-4434-9e25-eb62547bc1d3"} {"msg":"Crucible stats registered with UUID: 2c6983e0-3943-4359-a75b-b025e61b76fd","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:31.07858518Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1654,"session_id":"07aa956a-3604-4434-9e25-eb62547bc1d3"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:31.109348049Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1654,"task":"crutest"} {"msg":"The guest has requested activation with gen:118","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:31.109373957Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1654} {"msg":"2c6983e0-3943-4359-a75b-b025e61b76fd active request set","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:31.109451391Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1654,"session_id":"07aa956a-3604-4434-9e25-eb62547bc1d3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:31.10946954Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1654,"client":"0","":"downstairs","session_id":"07aa956a-3604-4434-9e25-eb62547bc1d3"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:31.109492939Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1654,"client":"0","":"downstairs","session_id":"07aa956a-3604-4434-9e25-eb62547bc1d3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:31.109510238Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1654,"client":"1","":"downstairs","session_id":"07aa956a-3604-4434-9e25-eb62547bc1d3"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:31.109540845Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1654,"client":"1","":"downstairs","session_id":"07aa956a-3604-4434-9e25-eb62547bc1d3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:31.109568723Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1654,"client":"2","":"downstairs","session_id":"07aa956a-3604-4434-9e25-eb62547bc1d3"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:31.109598732Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1654,"client":"2","":"downstairs","session_id":"07aa956a-3604-4434-9e25-eb62547bc1d3"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:31.10962495Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1654,"":"io task","client":"2","":"downstairs","session_id":"07aa956a-3604-4434-9e25-eb62547bc1d3"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:31.109649888Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1654,"":"io task","client":"0","":"downstairs","session_id":"07aa956a-3604-4434-9e25-eb62547bc1d3"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:31.109676086Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1654,"":"io task","client":"1","":"downstairs","session_id":"07aa956a-3604-4434-9e25-eb62547bc1d3"} {"msg":"ds_connection connected from Ok(127.0.0.1:55017)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:31.109701584Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1654,"":"io task","client":"2","":"downstairs","session_id":"07aa956a-3604-4434-9e25-eb62547bc1d3"} {"msg":"ds_connection connected from Ok(127.0.0.1:53540)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:31.109724973Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1654,"":"io task","client":"0","":"downstairs","session_id":"07aa956a-3604-4434-9e25-eb62547bc1d3"} {"msg":"ds_connection connected from Ok(127.0.0.1:60961)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:31.109750681Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1654,"":"io task","client":"1","":"downstairs","session_id":"07aa956a-3604-4434-9e25-eb62547bc1d3"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:31.110431414Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1654,"client":"2","":"downstairs","session_id":"07aa956a-3604-4434-9e25-eb62547bc1d3"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:31.110460722Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1654,"client":"0","":"downstairs","session_id":"07aa956a-3604-4434-9e25-eb62547bc1d3"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:31.110610192Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1654,"client":"1","":"downstairs","session_id":"07aa956a-3604-4434-9e25-eb62547bc1d3"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:31.111112047Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1654,"client":"2","":"downstairs","session_id":"07aa956a-3604-4434-9e25-eb62547bc1d3"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:31.111141845Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1654,"client":"0","":"downstairs","session_id":"07aa956a-3604-4434-9e25-eb62547bc1d3"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:31.111158224Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1654,"client":"1","":"downstairs","session_id":"07aa956a-3604-4434-9e25-eb62547bc1d3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:31.111679998Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1654,"session_id":"07aa956a-3604-4434-9e25-eb62547bc1d3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:31.111713206Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1654,"session_id":"07aa956a-3604-4434-9e25-eb62547bc1d3"} {"msg":"[0]R flush_numbers[0..12]: [198, 219, 216, 219, 219, 219, 219, 189, 206, 219, 219, 208]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:31.111781751Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1654,"":"downstairs","":"downstairs","session_id":"07aa956a-3604-4434-9e25-eb62547bc1d3"} {"msg":"[0]R generation[0..12]: [108, 117, 115, 117, 117, 117, 117, 101, 111, 117, 117, 112]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:31.11180234Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1654,"":"downstairs","":"downstairs","session_id":"07aa956a-3604-4434-9e25-eb62547bc1d3"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:31.111817179Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1654,"":"downstairs","":"downstairs","session_id":"07aa956a-3604-4434-9e25-eb62547bc1d3"} {"msg":"[1]R flush_numbers[0..12]: [198, 219, 216, 219, 219, 219, 219, 189, 206, 219, 219, 208]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:31.111831098Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1654,"":"downstairs","":"downstairs","session_id":"07aa956a-3604-4434-9e25-eb62547bc1d3"} {"msg":"[1]R generation[0..12]: [108, 117, 115, 117, 117, 117, 117, 101, 111, 117, 117, 112]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:31.111844997Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1654,"":"downstairs","":"downstairs","session_id":"07aa956a-3604-4434-9e25-eb62547bc1d3"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:31.111859036Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1654,"":"downstairs","":"downstairs","session_id":"07aa956a-3604-4434-9e25-eb62547bc1d3"} {"msg":"[2]R flush_numbers[0..12]: [198, 219, 216, 219, 219, 219, 219, 189, 206, 219, 219, 208]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:31.111872685Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1654,"":"downstairs","":"downstairs","session_id":"07aa956a-3604-4434-9e25-eb62547bc1d3"} {"msg":"[2]R generation[0..12]: [108, 117, 115, 117, 117, 117, 117, 101, 111, 117, 117, 112]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:31.111886174Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1654,"":"downstairs","":"downstairs","session_id":"07aa956a-3604-4434-9e25-eb62547bc1d3"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:31.111899963Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1654,"":"downstairs","":"downstairs","session_id":"07aa956a-3604-4434-9e25-eb62547bc1d3"} {"msg":"Max found gen is 118","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:31.111912782Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1654,"":"downstairs","":"downstairs","session_id":"07aa956a-3604-4434-9e25-eb62547bc1d3"} {"msg":"Generation requested: 118 >= found:118","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:31.111925841Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1654,"":"downstairs","":"downstairs","session_id":"07aa956a-3604-4434-9e25-eb62547bc1d3"} {"msg":"Next flush: 220","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:31.11193859Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1654,"":"downstairs","":"downstairs","session_id":"07aa956a-3604-4434-9e25-eb62547bc1d3"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:31.111955239Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1654,"":"downstairs","":"downstairs","session_id":"07aa956a-3604-4434-9e25-eb62547bc1d3"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:31.111968888Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1654,"session_id":"07aa956a-3604-4434-9e25-eb62547bc1d3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-12-12T20:37:31.111980657Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1654,"client":"0","":"downstairs","session_id":"07aa956a-3604-4434-9e25-eb62547bc1d3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:31.111998536Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1654,"client":"1","":"downstairs","session_id":"07aa956a-3604-4434-9e25-eb62547bc1d3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:31.112011495Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1654,"client":"2","":"downstairs","session_id":"07aa956a-3604-4434-9e25-eb62547bc1d3"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:31.112024654Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1654,"session_id":"07aa956a-3604-4434-9e25-eb62547bc1d3"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:31.112036823Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1654,"session_id":"07aa956a-3604-4434-9e25-eb62547bc1d3"} {"msg":"2c6983e0-3943-4359-a75b-b025e61b76fd is now active with session: 07aa956a-3604-4434-9e25-eb62547bc1d3","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:31.112049323Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1654,"session_id":"07aa956a-3604-4434-9e25-eb62547bc1d3"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:31.112061112Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1654,"session_id":"07aa956a-3604-4434-9e25-eb62547bc1d3"} {"msg":"The guest has finished waiting for activation with:118","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:31.112073071Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1654} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 257 len 8192 data: 7 8 02/30 Read block 465 len 20480 03/30 Write block 569 len 20480 data: 10 8 11 10 10 04/30 Flush 05/30 Write block 291 len 32768 data: 9 9 10 10 11 10 9 6 06/30 Flush 07/30 Write block 20 len 32768 data: 7 7 8 8 10 9 8 9 08/30 Read block 402 len 28672 09/30 Read block 32 len 16384 10/30 Read block 511 len 20480 11/30 Write block 460 len 40960 data: 14 17 18 16 14 14 12 10 8 9 12/30 Read block 160 len 4096 13/30 Write block 299 len 36864 data: 6 7 6 5 5 3 5 6 6 14/30 Read block 276 len 28672 15/30 Read block 282 len 24576 16/30 Read block 480 len 24576 17/30 Flush 18/30 Write block 66 len 12288 data: 9 7 8 19/30 Write block 318 len 28672 data: 14 11 11 11 11 9 7 20/30 Write block 61 len 16384 data: 10 10 11 9 21/30 Read block 66 len 32768 22/30 Write block 200 len 36864 data: 7 8 9 10 13 14 14 9 11 23/30 Read block 256 len 20480 24/30 Write block 150 len 24576 data: 9 7 8 9 8 8 25/30 Read block 22 len 8192 26/30 Read block 136 len 32768 27/30 Write block 375 len 20480 data: 6 8 7 9 11 28/30 Read block 525 len 20480 29/30 Read block 71 len 8192 30/30 Write block 23 len 24576 data: 9 11 10 9 10 9 ---------------------------------------------------------------- Crucible gen:118 GIO:true work queues: Upstairs:0 downstairs:13 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1017 Acked Write 3 Done Done Done false 1018 Acked Write 7 Done Done Done false 1019 Acked Write 4 Done Done Done false 1020 Acked Read 8 Done Done Done false 1021 Acked Write 9 Done Done Done false 1022 Acked Read 5 Done Done Done false 1023 Acked Write 6 Done Done Done false 1024 Acked Read 2 Done Done Done false 1025 Acked Read 8 Done Done Done false 1026 Acked Write 5 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 12 12 12 36 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1016 1016 1016 Downstairs last five completed: 1016 1015 1014 1013 1012 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:37:31.13254021Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1654} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 108 108 108 198 198 198 F F F 1 020-039 118 118 118 222 222 222 T T T 2 040-059 115 115 115 216 216 216 F F F 3 060-079 117 117 117 219 219 219 T T T 4 080-099 117 117 117 219 219 219 F F F 5 100-119 117 117 117 219 219 219 F F F 6 120-139 117 117 117 219 219 219 F F F 7 140-159 101 101 101 189 189 189 T T T 8 160-179 111 111 111 206 206 206 F F F 9 180-199 117 117 117 219 219 219 F F F 10 200-219 117 117 117 219 219 219 T T T 11 220-239 112 112 112 208 208 208 F F F 12 240-259 118 118 118 220 220 220 F F F 13 260-279 117 117 117 219 219 219 F F F 14 280-299 118 118 118 222 222 222 F F F 15 300-319 118 118 118 222 222 222 T T T 16 320-339 117 117 117 219 219 219 T T T 17 340-359 109 109 109 199 199 199 F F F 18 360-379 110 110 110 205 205 205 T T T 19 380-399 117 117 117 219 219 219 F F F 20 400-419 117 117 117 219 219 219 F F F 21 420-439 112 112 112 208 208 208 F F F 22 440-459 117 117 117 219 219 219 F F F 23 460-479 118 118 118 222 222 222 F F F 24 480-499 106 106 106 197 197 197 F F F 25 500-519 115 115 115 216 216 216 F F F 26 520-539 108 108 108 198 198 198 F F F 27 540-559 98 98 98 185 185 185 F F F 28 560-579 118 118 118 220 220 220 F F F 29 580-599 117 117 117 219 219 219 F F F Max gen: 118, Max flush: 222 A Difference in extent metadata is expected here On loop 59 Verifying data now No change after verify Loop: 59 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 108 108 108 198 198 198 F F F 1 020-039 119 119 119 223 223 223 F F F 2 040-059 115 115 115 216 216 216 F F F 3 060-079 119 119 119 223 223 223 F F F 4 080-099 117 117 117 219 219 219 F F F 5 100-119 117 117 117 219 219 219 F F F 6 120-139 117 117 117 219 219 219 F F F 7 140-159 119 119 119 223 223 223 F F F 8 160-179 111 111 111 206 206 206 F F F 9 180-199 117 117 117 219 219 219 F F F 10 200-219 119 119 119 223 223 223 F F F 11 220-239 112 112 112 208 208 208 F F F 12 240-259 118 118 118 220 220 220 F F F 13 260-279 117 117 117 219 219 219 F F F 14 280-299 118 118 118 222 222 222 F F F 15 300-319 119 119 119 223 223 223 F F F 16 320-339 119 119 119 223 223 223 F F F 17 340-359 109 109 109 199 199 199 F F F 18 360-379 119 119 119 223 223 223 F F F 19 380-399 117 117 117 219 219 219 F F F 20 400-419 117 117 117 219 219 219 F F F 21 420-439 112 112 112 208 208 208 F F F 22 440-459 117 117 117 219 219 219 F F F 23 460-479 118 118 118 222 222 222 F F F 24 480-499 106 106 106 197 197 197 F F F 25 500-519 115 115 115 216 216 216 F F F 26 520-539 108 108 108 198 198 198 F F F 27 540-559 98 98 98 185 185 185 F F F 28 560-579 118 118 118 220 220 220 F F F 29 580-599 117 117 117 219 219 219 F F F Max gen: 119, Max flush: 223 Begin loop 60 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:35.412663329Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1662,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:35.457724912Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1662,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:35.458360198Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1662} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"26e7d501b33f47379e53601720353c2983e7654b\",\n git_commit_timestamp: \"2024-12-12T20:16:40.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:35.458396056Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1662} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:35.458411175Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1662} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:35.458425834Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1662} {"msg":"Crucible 09ce8621-13f6-4c39-89f2-1df87e3c53bd has session id: 720f81a5-4d05-4a32-b869-edf62ce7fbc1","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:35.458831186Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1662,"session_id":"720f81a5-4d05-4a32-b869-edf62ce7fbc1"} {"msg":"Upstairs opts: Upstairs UUID: 09ce8621-13f6-4c39-89f2-1df87e3c53bd, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:35.458877113Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1662,"session_id":"720f81a5-4d05-4a32-b869-edf62ce7fbc1"} {"msg":"Crucible stats registered with UUID: 09ce8621-13f6-4c39-89f2-1df87e3c53bd","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:35.458902651Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1662,"session_id":"720f81a5-4d05-4a32-b869-edf62ce7fbc1"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:35.489995137Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1662,"task":"crutest"} {"msg":"The guest has requested activation with gen:120","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:35.490029964Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1662} {"msg":"09ce8621-13f6-4c39-89f2-1df87e3c53bd active request set","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:35.490064832Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1662,"session_id":"720f81a5-4d05-4a32-b869-edf62ce7fbc1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:35.490082011Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1662,"client":"0","":"downstairs","session_id":"720f81a5-4d05-4a32-b869-edf62ce7fbc1"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:35.490111059Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1662,"client":"0","":"downstairs","session_id":"720f81a5-4d05-4a32-b869-edf62ce7fbc1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:35.490128688Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1662,"client":"1","":"downstairs","session_id":"720f81a5-4d05-4a32-b869-edf62ce7fbc1"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:35.490146776Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1662,"client":"1","":"downstairs","session_id":"720f81a5-4d05-4a32-b869-edf62ce7fbc1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:35.490163475Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1662,"client":"2","":"downstairs","session_id":"720f81a5-4d05-4a32-b869-edf62ce7fbc1"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:35.490180774Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1662,"client":"2","":"downstairs","session_id":"720f81a5-4d05-4a32-b869-edf62ce7fbc1"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:35.490198583Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1662,"":"io task","client":"2","":"downstairs","session_id":"720f81a5-4d05-4a32-b869-edf62ce7fbc1"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:35.490230351Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1662,"":"io task","client":"0","":"downstairs","session_id":"720f81a5-4d05-4a32-b869-edf62ce7fbc1"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:35.490264138Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1662,"":"io task","client":"1","":"downstairs","session_id":"720f81a5-4d05-4a32-b869-edf62ce7fbc1"} {"msg":"ds_connection connected from Ok(127.0.0.1:46799)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:35.490299366Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1662,"":"io task","client":"2","":"downstairs","session_id":"720f81a5-4d05-4a32-b869-edf62ce7fbc1"} {"msg":"ds_connection connected from Ok(127.0.0.1:54014)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:35.490333703Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1662,"":"io task","client":"0","":"downstairs","session_id":"720f81a5-4d05-4a32-b869-edf62ce7fbc1"} {"msg":"ds_connection connected from Ok(127.0.0.1:43172)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:35.490367401Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1662,"":"io task","client":"1","":"downstairs","session_id":"720f81a5-4d05-4a32-b869-edf62ce7fbc1"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:35.491078952Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1662,"client":"0","":"downstairs","session_id":"720f81a5-4d05-4a32-b869-edf62ce7fbc1"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:35.491236161Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1662,"client":"1","":"downstairs","session_id":"720f81a5-4d05-4a32-b869-edf62ce7fbc1"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:35.491266289Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1662,"client":"2","":"downstairs","session_id":"720f81a5-4d05-4a32-b869-edf62ce7fbc1"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:35.491699739Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1662,"client":"0","":"downstairs","session_id":"720f81a5-4d05-4a32-b869-edf62ce7fbc1"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:35.491740017Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1662,"client":"1","":"downstairs","session_id":"720f81a5-4d05-4a32-b869-edf62ce7fbc1"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:35.492058975Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1662,"client":"2","":"downstairs","session_id":"720f81a5-4d05-4a32-b869-edf62ce7fbc1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:35.49227786Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1662,"session_id":"720f81a5-4d05-4a32-b869-edf62ce7fbc1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:35.492378893Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1662,"session_id":"720f81a5-4d05-4a32-b869-edf62ce7fbc1"} {"msg":"[0]R flush_numbers[0..12]: [198, 223, 216, 223, 219, 219, 219, 223, 206, 219, 223, 208]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:35.492588918Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1662,"":"downstairs","":"downstairs","session_id":"720f81a5-4d05-4a32-b869-edf62ce7fbc1"} {"msg":"[0]R generation[0..12]: [108, 119, 115, 119, 117, 117, 117, 119, 111, 117, 119, 112]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:35.492611377Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1662,"":"downstairs","":"downstairs","session_id":"720f81a5-4d05-4a32-b869-edf62ce7fbc1"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:35.492626405Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1662,"":"downstairs","":"downstairs","session_id":"720f81a5-4d05-4a32-b869-edf62ce7fbc1"} {"msg":"[1]R flush_numbers[0..12]: [198, 223, 216, 223, 219, 219, 219, 223, 206, 219, 223, 208]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:35.492640155Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1662,"":"downstairs","":"downstairs","session_id":"720f81a5-4d05-4a32-b869-edf62ce7fbc1"} {"msg":"[1]R generation[0..12]: [108, 119, 115, 119, 117, 117, 117, 119, 111, 117, 119, 112]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:35.492653834Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1662,"":"downstairs","":"downstairs","session_id":"720f81a5-4d05-4a32-b869-edf62ce7fbc1"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:35.492667693Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1662,"":"downstairs","":"downstairs","session_id":"720f81a5-4d05-4a32-b869-edf62ce7fbc1"} {"msg":"[2]R flush_numbers[0..12]: [198, 223, 216, 223, 219, 219, 219, 223, 206, 219, 223, 208]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:35.492684612Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1662,"":"downstairs","":"downstairs","session_id":"720f81a5-4d05-4a32-b869-edf62ce7fbc1"} {"msg":"[2]R generation[0..12]: [108, 119, 115, 119, 117, 117, 117, 119, 111, 117, 119, 112]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:35.49269874Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1662,"":"downstairs","":"downstairs","session_id":"720f81a5-4d05-4a32-b869-edf62ce7fbc1"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:35.492712859Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1662,"":"downstairs","":"downstairs","session_id":"720f81a5-4d05-4a32-b869-edf62ce7fbc1"} {"msg":"Max found gen is 120","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:35.492725869Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1662,"":"downstairs","":"downstairs","session_id":"720f81a5-4d05-4a32-b869-edf62ce7fbc1"} {"msg":"Generation requested: 120 >= found:120","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:35.492739228Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1662,"":"downstairs","":"downstairs","session_id":"720f81a5-4d05-4a32-b869-edf62ce7fbc1"} {"msg":"Next flush: 224","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:35.492752267Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1662,"":"downstairs","":"downstairs","session_id":"720f81a5-4d05-4a32-b869-edf62ce7fbc1"} {"msg":"All extents match","v":0,"name":"crucible"Wait for a query_work_queue command to finish before sending IO ,"level":30,"time":"2024-12-12T20:37:35.492769646Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1662,"":"downstairs","":"downstairs","session_id":"720f81a5-4d05-4a32-b869-edf62ce7fbc1"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:35.492782785Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1662,"session_id":"720f81a5-4d05-4a32-b869-edf62ce7fbc1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:35.492794714Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1662,"client":"0","":"downstairs","session_id":"720f81a5-4d05-4a32-b869-edf62ce7fbc1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:35.492808203Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1662,"client":"1","":"downstairs","session_id":"720f81a5-4d05-4a32-b869-edf62ce7fbc1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:35.492821432Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1662,"client":"2","":"downstairs","session_id":"720f81a5-4d05-4a32-b869-edf62ce7fbc1"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:35.492834971Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1662,"session_id":"720f81a5-4d05-4a32-b869-edf62ce7fbc1"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:35.49284723Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1662,"session_id":"720f81a5-4d05-4a32-b869-edf62ce7fbc1"} {"msg":"09ce8621-13f6-4c39-89f2-1df87e3c53bd is now active with session: 720f81a5-4d05-4a32-b869-edf62ce7fbc1","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:35.492860189Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1662,"session_id":"720f81a5-4d05-4a32-b869-edf62ce7fbc1"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:35.492872128Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1662,"session_id":"720f81a5-4d05-4a32-b869-edf62ce7fbc1"} {"msg":"The guest has finished waiting for activation with:120","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:35.492884338Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1662} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 304 len 12288 data: 4 6 7 02/30 Read block 360 len 8192 03/30 Read block 544 len 4096 04/30 Write block 331 len 28672 data: 12 9 10 10 9 7 8 05/30 Read block 234 len 4096 06/30 Write block 262 len 8192 data: 5 7 07/30 Read block 340 len 12288 08/30 Write block 188 len 36864 data: 13 13 11 11 12 13 10 10 9 09/30 Read block 57 len 32768 10/30 Read block 319 len 36864 11/30 Read block 416 len 16384 12/30 Read block 494 len 4096 13/30 Write block 382 len 36864 data: 12 10 8 10 11 9 9 8 10 14/30 Read block 510 len 40960 15/30 Write block 274 len 24576 data: 13 12 10 10 12 10 16/30 Write block 44 len 40960 data: 7 9 8 8 8 11 11 10 8 11 17/30 Read block 309 len 24576 18/30 Flush 19/30 Write block 442 len 8192 data: 9 8 20/30 Flush 21/30 Write block 126 len 8192 data: 8 7 22/30 Write block 20 len 20480 data: 8 8 9 10 12 23/30 Read block 546 len 12288 24/30 Read block 554 len 16384 25/30 Read block 346 len 12288 26/30 Write block 549 len 36864 data: 8 7 7 7 8 9 9 8 8 27/30 Write block 401 len 36864 data: 14 11 10 9 10 12 12 12 10 28/30 Write block 541 len 32768 data: 7 9 8 9 7 7 7 7 29/30 Write block 488 len 28672 data: 10 11 10 8 8 8 6 30/30 Write block 239 len 8192 data: 11 10 ---------------------------------------------------------------- Crucible gen:120 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1020 Acked Write 2 Done Done Done false 1021 Acked Write 5 Done Done Done false 1022 Acked Read 3 Done Done Done false 1023 Acked Read 4 Done Done Done false 1024 Acked Read 3 Done Done Done false 1025 Acked Write 9 Done Done Done false 1026 Acked Write 9 Done Done Done false 1027 Acked Write 8 Sent Sent Sent false 1028 Acked Write 7 Sent Sent Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 3 3 9 Done 7 7 7 21 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1019 1019 1019 Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:37:35.511775585Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1662} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 108 108 108 198 198 198 F F F 1 020-039 119 119 119 223 223 223 T T T 2 040-059 120 120 120 224 224 224 F F F 3 060-079 119 119 119 223 223 223 F F F 4 080-099 117 117 117 219 219 219 F F F 5 100-119 117 117 117 219 219 219 F F F 6 120-139 117 117 117 219 219 219 T T T 7 140-159 119 119 119 223 223 223 F F F 8 160-179 111 111 111 206 206 206 F F F 9 180-199 120 120 120 224 224 224 F F F 10 200-219 119 119 119 223 223 223 F F F 11 220-239 112 112 112 208 208 208 T T T 12 240-259 118 118 118 220 220 220 T T T 13 260-279 120 120 120 224 224 224 F F F 14 280-299 118 118 118 222 222 222 F F F 15 300-319 120 120 120 224 224 224 F F F 16 320-339 120 120 120 224 224 224 F F F 17 340-359 109 109 109 199 199 199 F F F 18 360-379 119 119 119 223 223 223 F F F 19 380-399 120 120 120 224 224 224 F F F 20 400-419 117 117 117 219 219 219 T T T 21 420-439 112 112 112 208 208 208 F F F 22 440-459 120 120 120 225 225 225 F F F 23 460-479 118 118 118 222 222 222 F F F 24 480-499 106 106 106 197 197 197 T T T 25 500-519 115 115 115 216 216 216 F F F 26 520-539 108 108 108 198 198 198 F F F 27 540-559 98 98 98 185 185 185 T T T 28 560-579 118 118 118 220 220 220 F F F 29 580-599 117 117 117 219 219 219 F F F Max gen: 120, Max flush: 225 A Difference in extent metadata is expected here On loop 60 Verifying data now No change after verify Loop: 60 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 108 108 108 198 198 198 F F F 1 020-039 121 121 121 226 226 226 F F F 2 040-059 120 120 120 224 224 224 F F F 3 060-079 119 119 119 223 223 223 F F F 4 080-099 117 117 117 219 219 219 F F F 5 100-119 117 117 117 219 219 219 F F F 6 120-139 121 121 121 226 226 226 F F F 7 140-159 119 119 119 223 223 223 F F F 8 160-179 111 111 111 206 206 206 F F F 9 180-199 120 120 120 224 224 224 F F F 10 200-219 119 119 119 223 223 223 F F F 11 220-239 121 121 121 226 226 226 F F F 12 240-259 121 121 121 226 226 226 F F F 13 260-279 120 120 120 224 224 224 F F F 14 280-299 118 118 118 222 222 222 F F F 15 300-319 120 120 120 224 224 224 F F F 16 320-339 120 120 120 224 224 224 F F F 17 340-359 109 109 109 199 199 199 F F F 18 360-379 119 119 119 223 223 223 F F F 19 380-399 120 120 120 224 224 224 F F F 20 400-419 121 121 121 226 226 226 F F F 21 420-439 112 112 112 208 208 208 F F F 22 440-459 120 120 120 225 225 225 F F F 23 460-479 118 118 118 222 222 222 F F F 24 480-499 121 121 121 226 226 226 F F F 25 500-519 115 115 115 216 216 216 F F F 26 520-539 108 108 108 198 198 198 F F F 27 540-559 121 121 121 226 226 226 F F F 28 560-579 118 118 118 220 220 220 F F F 29 580-599 117 117 117 219 219 219 F F F Max gen: 121, Max flush: 226 Begin loop 61 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:39.797311891Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1670,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:37:39.841548131Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1670,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:39.841938134Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1670,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:39.874396316Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1670,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:39.874968756Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1670} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"26e7d501b33f47379e53601720353c2983e7654b\",\n git_commit_timestamp: \"2024-12-12T20:16:40.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:39.875001014Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1670} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:39.875018623Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1670} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:39.875032782Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1670} {"msg":"Crucible eb4bcf06-3b17-4fb1-b2fd-8a7ed0b6ea97 has session id: e047a7dd-8640-4810-a3bf-f0d10a3309fb","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:39.875532608Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1670,"session_id":"e047a7dd-8640-4810-a3bf-f0d10a3309fb"} {"msg":"Upstairs opts: Upstairs UUID: eb4bcf06-3b17-4fb1-b2fd-8a7ed0b6ea97, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:39.875577954Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1670,"session_id":"e047a7dd-8640-4810-a3bf-f0d10a3309fb"} {"msg":"Crucible stats registered with UUID: eb4bcf06-3b17-4fb1-b2fd-8a7ed0b6ea97","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:39.875602863Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1670,"session_id":"e047a7dd-8640-4810-a3bf-f0d10a3309fb"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:39.907229032Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1670,"task":"crutest"} {"msg":"The guest has requested activation with gen:122","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:39.90726321Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1670} {"msg":"eb4bcf06-3b17-4fb1-b2fd-8a7ed0b6ea97 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:39.907334835Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1670,"session_id":"e047a7dd-8640-4810-a3bf-f0d10a3309fb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:39.907357793Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1670,"client":"0","":"downstairs","session_id":"e047a7dd-8640-4810-a3bf-f0d10a3309fb"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:39.907391321Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1670,"client":"0","":"downstairs","session_id":"e047a7dd-8640-4810-a3bf-f0d10a3309fb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:39.907424038Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1670,"client":"1","":"downstairs","session_id":"e047a7dd-8640-4810-a3bf-f0d10a3309fb"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:39.907457636Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1670,"client":"1","":"downstairs","session_id":"e047a7dd-8640-4810-a3bf-f0d10a3309fb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:39.907485314Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1670,"client":"2","":"downstairs","session_id":"e047a7dd-8640-4810-a3bf-f0d10a3309fb"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:39.907525751Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1670,"client":"2","":"downstairs","session_id":"e047a7dd-8640-4810-a3bf-f0d10a3309fb"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:39.907563659Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1670,"":"io task","client":"2","":"downstairs","session_id":"e047a7dd-8640-4810-a3bf-f0d10a3309fb"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:39.907596397Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1670,"":"io task","client":"0","":"downstairs","session_id":"e047a7dd-8640-4810-a3bf-f0d10a3309fb"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:39.907615735Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1670,"":"io task","client":"1","":"downstairs","session_id":"e047a7dd-8640-4810-a3bf-f0d10a3309fb"} {"msg":"ds_connection connected from Ok(127.0.0.1:36601)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:39.907629684Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1670,"":"io task","client":"2","":"downstairs","session_id":"e047a7dd-8640-4810-a3bf-f0d10a3309fb"} {"msg":"ds_connection connected from Ok(127.0.0.1:41857)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:39.907648643Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1670,"":"io task","client":"0","":"downstairs","session_id":"e047a7dd-8640-4810-a3bf-f0d10a3309fb"} {"msg":"ds_connection connected from Ok(127.0.0.1:57557)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:39.907675901Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1670,"":"io task","client":"1","":"downstairs","session_id":"e047a7dd-8640-4810-a3bf-f0d10a3309fb"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:39.908441938Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1670,"client":"1","":"downstairs","session_id":"e047a7dd-8640-4810-a3bf-f0d10a3309fb"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:39.908484225Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1670,"client":"2","":"downstairs","session_id":"e047a7dd-8640-4810-a3bf-f0d10a3309fb"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:39.908587818Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1670,"client":"0","":"downstairs","session_id":"e047a7dd-8640-4810-a3bf-f0d10a3309fb"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:39.909018328Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1670,"client":"1","":"downstairs","session_id":"e047a7dd-8640-4810-a3bf-f0d10a3309fb"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:39.90913674Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1670,"client":"0","":"downstairs","session_id":"e047a7dd-8640-4810-a3bf-f0d10a3309fb"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:39.909248563Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1670,"client":"2","":"downstairs","session_id":"e047a7dd-8640-4810-a3bf-f0d10a3309fb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:39.90958093Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1670,"session_id":"e047a7dd-8640-4810-a3bf-f0d10a3309fb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:39.909664844Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1670,"session_id":"e047a7dd-8640-4810-a3bf-f0d10a3309fb"} {"msg":"[0]R flush_numbers[0..12]: [198, 226, 224, 223, 219, 219, 226, 223, 206, 224, 223, 226]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:39.909787626Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1670,"":"downstairs","":"downstairs","session_id":"e047a7dd-8640-4810-a3bf-f0d10a3309fb"} {"msg":"[0]R generation[0..12]: [108, 121, 120, 119, 117, 117, 121, 119, 111, 120, 119, 121]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:39.909807274Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1670,"":"downstairs","":"downstairs","session_id":"e047a7dd-8640-4810-a3bf-f0d10a3309fb"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:39.909821513Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1670,"":"downstairs","":"downstairs","session_id":"e047a7dd-8640-4810-a3bf-f0d10a3309fb"} {"msg":"[1]R flush_numbers[0..12]: [198, 226, 224, 223, 219, 219, 226, 223, 206, 224, 223, 226]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:39.909835322Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1670,"":"downstairs","":"downstairs","session_id":"e047a7dd-8640-4810-a3bf-f0d10a3309fb"} {"msg":"[1]R generation[0..12]: [108, 121, 120, 119, 117, 117, 121, 119, 111, 120, 119, 121]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:39.909849121Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1670,"":"downstairs","":"downstairs","session_id":"e047a7dd-8640-4810-a3bf-f0d10a3309fb"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:39.90986304Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1670,"":"downstairs","":"downstairs","session_id":"e047a7dd-8640-4810-a3bf-f0d10a3309fb"} {"msg":"[2]R flush_numbers[0..12]: [198, 226, 224, 223, 219, 219, 226, 223, 206, 224, 223, 226]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:39.909876779Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1670,"":"downstairs","":"downstairs","session_id":"e047a7dd-8640-4810-a3bf-f0d10a3309fb"} {"msg":"[2]R generation[0..12]: [108, 121, 120, 119, 117, 117, 121, 119, 111, 120, 119, 121]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:39.909890468Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1670,"":"downstairs","":"downstairs","session_id":"e047a7dd-8640-4810-a3bf-f0d10a3309fb"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:39.909904298Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1670,"":"downstairs","":"downstairs","session_id":"e047a7dd-8640-4810-a3bf-f0d10a3309fb"} {"msg":"Max found gen is 122","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:39.909917377Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1670,"":"downstairs","":"downstairs","session_id":"e047a7dd-8640-4810-a3bf-f0d10a3309fb"} {"msg":"Generation requested: 122 >= found:122","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:39.909930496Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1670,"":"downstairs","":"downstairs","session_id":"e047a7dd-8640-4810-a3bf-f0d10a3309fb"} {"msg":"Next flush: 227","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:39.909943335Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1670,"":"downstairs","":"downstairs","session_id":"e047a7dd-8640-4810-a3bf-f0d10a3309fb"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:39.909956124Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1670,"":"downstairs","":"downstairs","session_id":"e047a7dd-8640-4810-a3bf-f0d10a3309fb"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:39.909969473Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1670,"session_id":"e047a7dd-8640-4810-a3bf-f0d10a3309fb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:39.909983952Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1670,"client":"0","":"Wait for a query_work_queue command to finish before sending IO downstairs","session_id":"e047a7dd-8640-4810-a3bf-f0d10a3309fb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:39.91000795Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1670,"client":"1","":"downstairs","session_id":"e047a7dd-8640-4810-a3bf-f0d10a3309fb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:39.910033099Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1670,"client":"2","":"downstairs","session_id":"e047a7dd-8640-4810-a3bf-f0d10a3309fb"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:39.910054367Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1670,"session_id":"e047a7dd-8640-4810-a3bf-f0d10a3309fb"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:39.910077246Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1670,"session_id":"e047a7dd-8640-4810-a3bf-f0d10a3309fb"} {"msg":"eb4bcf06-3b17-4fb1-b2fd-8a7ed0b6ea97 is now active with session: e047a7dd-8640-4810-a3bf-f0d10a3309fb","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:39.910101644Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1670,"session_id":"e047a7dd-8640-4810-a3bf-f0d10a3309fb"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:39.910119143Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1670,"session_id":"e047a7dd-8640-4810-a3bf-f0d10a3309fb"} {"msg":"The guest has finished waiting for activation with:122","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:39.910131642Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1670} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 308 len 8192 data: 7 6 02/30 Read block 255 len 40960 03/30 Write block 41 len 32768 data: 9 8 8 8 10 9 9 9 04/30 Read block 320 len 4096 05/30 Read block 254 len 40960 06/30 Write block 353 len 40960 data: 9 12 11 10 10 7 7 7 8 9 07/30 Write block 574 len 4096 data: 10 08/30 Write block 95 len 24576 data: 9 6 5 5 8 10 09/30 Read block 150 len 36864 10/30 Read block 149 len 40960 11/30 Write block 331 len 24576 data: 13 10 11 11 10 8 12/30 Read block 31 len 20480 13/30 Read block 364 len 36864 14/30 Read block 413 len 16384 15/30 Write block 57 len 16384 data: 18 17 16 15 16/30 Read block 216 len 24576 17/30 Read block 168 len 36864 18/30 Read block 49 len 36864 19/30 Read block 432 len 32768 20/30 Read block 520 len 12288 21/30 Read block 517 len 32768 22/30 Read block 533 len 36864 23/30 Read block 566 len 24576 24/30 Read block 42 len 20480 25/30 Write block 297 len 8192 data: 10 7 26/30 Write block 175 len 24576 data: 8 8 9 9 11 9 27/30 Read block 196 len 24576 28/30 Write block 572 len 28672 data: 11 11 11 10 9 8 10 29/30 Read block 110 len 24576 30/30 Read block 173 len 32768 ---------------------------------------------------------------- Crucible gen:122 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 2 Done Done Done false 1001 Acked Read 10 Done Done Done false 1002 Acked Write 8 Done Done Done false 1003 Acked Read 1 Done Done Done false 1004 Acked Read 10 Done Done Done false 1005 Acked Write 10 Done Done Done false 1006 Acked Write 1 Done Done Done false 1007 Acked Write 6 Done Done Done false 1008 Acked Read 9 Done Done Done false 1009 Acked Read 10 Done Done Done false 1010 Acked Write 6 Done Done Done false 1011 Acked Read 5 Done Done Done false 1012 Acked Read 9 Done Done Done false 1013 Acked Read 4 Done Done Done false 1014 Acked Write 4 Done Done Done false 1015 Acked Read 6 Done Done Done false 1016 Acked Read 9 Done Done Done false 1017 Acked Read 9 Done Done Done false 1018 Acked Read 8 Done Done Done false 1019 Acked Read 3 Done Done Done false 1020 Acked Read 8 Done Done Done false 1021 Acked Read 9 Done Done Done false 1022 Acked Read 6 Done Done Done false 1023 Acked Read 5 Done Done Done false 1024 Acked Write 2 Done Done Done false 1025 Acked Write 6 Done Done Done false 1026 Acked Read 6 Done Done Done false 1027 Acked Write 7 Done Done Done false 1028 Acked Read 6 Done Done Done false 1029 Acked Read 8 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 30 29 29 88 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 0 0 0 Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:37:39.930889571Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1670} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 108 108 108 198 198 198 F F F 1 020-039 121 121 121 226 226 226 F F F 2 040-059 120 120 120 224 224 224 T T T 3 060-079 119 119 119 223 223 223 T T T 4 080-099 117 117 117 219 219 219 T T T 5 100-119 117 117 117 219 219 219 T T T 6 120-139 121 121 121 226 226 226 F F F 7 140-159 119 119 119 223 223 223 F F F 8 160-179 111 111 111 206 206 206 T T T 9 180-199 120 120 120 224 224 224 T T T 10 200-219 119 119 119 223 223 223 F F F 11 220-239 121 121 121 226 226 226 F F F 12 240-259 121 121 121 226 226 226 F F F 13 260-279 120 120 120 224 224 224 F F F 14 280-299 118 118 118 222 222 222 T T T 15 300-319 120 120 120 224 224 224 T T T 16 320-339 120 120 120 224 224 224 T T T 17 340-359 109 109 109 199 199 199 T T T 18 360-379 119 119 119 223 223 223 T T T 19 380-399 120 120 120 224 224 224 F F F 20 400-419 121 121 121 226 226 226 F F F 21 420-439 112 112 112 208 208 208 F F F 22 440-459 120 120 120 225 225 225 F F F 23 460-479 118 118 118 222 222 222 F F F 24 480-499 121 121 121 226 226 226 F F F 25 500-519 115 115 115 216 216 216 F F F 26 520-539 108 108 108 198 198 198 F F F 27 540-559 121 121 121 226 226 226 F F F 28 560-579 118 118 118 220 220 220 T T T 29 580-599 117 117 117 219 219 219 F F F Max gen: 121, Max flush: 226 A Difference in extent metadata is expected here On loop 61 Verifying data now No change after verify Loop: 61 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 108 108 108 198 198 198 F F F 1 020-039 121 121 121 226 226 226 F F F 2 040-059 122 122 122 227 227 227 F F F 3 060-079 122 122 122 227 227 227 F F F 4 080-099 122 122 122 227 227 227 F F F 5 100-119 122 122 122 227 227 227 F F F 6 120-139 121 121 121 226 226 226 F F F 7 140-159 119 119 119 223 223 223 F F F 8 160-179 122 122 122 227 227 227 F F F 9 180-199 122 122 122 227 227 227 F F F 10 200-219 119 119 119 223 223 223 F F F 11 220-239 121 121 121 226 226 226 F F F 12 240-259 121 121 121 226 226 226 F F F 13 260-279 120 120 120 224 224 224 F F F 14 280-299 122 122 122 227 227 227 F F F 15 300-319 122 122 122 227 227 227 F F F 16 320-339 122 122 122 227 227 227 F F F 17 340-359 122 122 122 227 227 227 F F F 18 360-379 122 122 122 227 227 227 F F F 19 380-399 120 120 120 224 224 224 F F F 20 400-419 121 121 121 226 226 226 F F F 21 420-439 112 112 112 208 208 208 F F F 22 440-459 120 120 120 225 225 225 F F F 23 460-479 118 118 118 222 222 222 F F F 24 480-499 121 121 121 226 226 226 F F F 25 500-519 115 115 115 216 216 216 F F F 26 520-539 108 108 108 198 198 198 F F F 27 540-559 121 121 121 226 226 226 F F F 28 560-579 122 122 122 227 227 227 F F F 29 580-599 117 117 117 219 219 219 F F F Max gen: 122, Max flush: 227 Begin loop 62 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:44.272374519Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1678,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:44.316771478Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1678,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:44.317456651Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1678} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"26e7d501b33f47379e53601720353c2983e7654b\",\n git_commit_timestamp: \"2024-12-12T20:16:40.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:44.317493338Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1678} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:44.317507447Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1678} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:44.317521636Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1678} {"msg":"Crucible 56d44137-6430-44c9-930a-cfcca40ec125 has session id: 6c2734fe-95f0-4178-bc77-8b1fcc92dec0","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:44.317960526Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1678,"session_id":"6c2734fe-95f0-4178-bc77-8b1fcc92dec0"} {"msg":"Upstairs opts: Upstairs UUID: 56d44137-6430-44c9-930a-cfcca40ec125, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:44.318023392Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1678,"session_id":"6c2734fe-95f0-4178-bc77-8b1fcc92dec0"} {"msg":"Crucible stats registered with UUID: 56d44137-6430-44c9-930a-cfcca40ec125","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:44.31804584Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1678,"session_id":"6c2734fe-95f0-4178-bc77-8b1fcc92dec0"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:44.348893083Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1678,"task":"crutest"} {"msg":"The guest has requested activation with gen:124","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:44.348920731Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1678} {"msg":"56d44137-6430-44c9-930a-cfcca40ec125 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:44.348975167Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1678,"session_id":"6c2734fe-95f0-4178-bc77-8b1fcc92dec0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:44.349003405Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1678,"client":"0","":"downstairs","session_id":"6c2734fe-95f0-4178-bc77-8b1fcc92dec0"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:44.349029403Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1678,"client":"0","":"downstairs","session_id":"6c2734fe-95f0-4178-bc77-8b1fcc92dec0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:44.349044102Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1678,"client":"1","":"downstairs","session_id":"6c2734fe-95f0-4178-bc77-8b1fcc92dec0"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:44.349058832Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1678,"client":"1","":"downstairs","session_id":"6c2734fe-95f0-4178-bc77-8b1fcc92dec0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:44.34907214Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1678,"client":"2","":"downstairs","session_id":"6c2734fe-95f0-4178-bc77-8b1fcc92dec0"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:44.349086079Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1678,"client":"2","":"downstairs","session_id":"6c2734fe-95f0-4178-bc77-8b1fcc92dec0"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:44.349099019Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1678,"":"io task","client":"2","":"downstairs","session_id":"6c2734fe-95f0-4178-bc77-8b1fcc92dec0"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:44.349112548Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1678,"":"io task","client":"0","":"downstairs","session_id":"6c2734fe-95f0-4178-bc77-8b1fcc92dec0"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:44.349135276Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1678,"":"io task","client":"1","":"downstairs","session_id":"6c2734fe-95f0-4178-bc77-8b1fcc92dec0"} {"msg":"ds_connection connected from Ok(127.0.0.1:52127)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:44.349244809Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1678,"":"io task","client":"2","":"downstairs","session_id":"6c2734fe-95f0-4178-bc77-8b1fcc92dec0"} {"msg":"ds_connection connected from Ok(127.0.0.1:46331)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:44.349490522Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1678,"":"io task","client":"1","":"downstairs","session_id":"6c2734fe-95f0-4178-bc77-8b1fcc92dec0"} {"msg":"ds_connection connected from Ok(127.0.0.1:62071)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:44.34951958Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1678,"":"io task","client":"0","":"downstairs","session_id":"6c2734fe-95f0-4178-bc77-8b1fcc92dec0"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:44.350017495Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1678,"client":"1","":"downstairs","session_id":"6c2734fe-95f0-4178-bc77-8b1fcc92dec0"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:44.350048423Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1678,"client":"0","":"downstairs","session_id":"6c2734fe-95f0-4178-bc77-8b1fcc92dec0"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:44.350127378Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1678,"client":"2","":"downstairs","session_id":"6c2734fe-95f0-4178-bc77-8b1fcc92dec0"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:44.350657281Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1678,"client":"1","":"downstairs","session_id":"6c2734fe-95f0-4178-bc77-8b1fcc92dec0"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:44.350686519Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1678,"client":"0","":"downstairs","session_id":"6c2734fe-95f0-4178-bc77-8b1fcc92dec0"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:44.350702538Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1678,"client":"2","":"downstairs","session_id":"6c2734fe-95f0-4178-bc77-8b1fcc92dec0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:44.351212603Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1678,"session_id":"6c2734fe-95f0-4178-bc77-8b1fcc92dec0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:44.351334664Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1678,"session_id":"6c2734fe-95f0-4178-bc77-8b1fcc92dec0"} {"msg":"[0]R flush_numbers[0..12]: [198, 226, 227, 227, 227, 227, 226, 223, 227, 227, 223, 226]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:44.351412519Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1678,"":"downstairs","":"downstairs","session_id":"6c2734fe-95f0-4178-bc77-8b1fcc92dec0"} {"msg":"[0]R generation[0..12]: [108, 121, 122, 122, 122, 122, 121, 119, 122, 122, 119, 121]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:44.351432278Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1678,"":"downstairs","":"downstairs","session_id":"6c2734fe-95f0-4178-bc77-8b1fcc92dec0"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:44.351446947Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1678,"":"downstairs","":"downstairs","session_id":"6c2734fe-95f0-4178-bc77-8b1fcc92dec0"} {"msg":"[1]R flush_numbers[0..12]: [198, 226, 227, 227, 227, 227, 226, 223, 227, 227, 223, 226]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:44.351461036Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1678,"":"downstairs","":"downstairs","session_id":"6c2734fe-95f0-4178-bc77-8b1fcc92dec0"} {"msg":"[1]R generation[0..12]: [108, 121, 122, 122, 122, 122, 121, 119, 122, 122, 119, 121]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:44.351474885Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1678,"":"downstairs","":"downstairs","session_id":"6c2734fe-95f0-4178-bc77-8b1fcc92dec0"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:44.351488734Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1678,"":"downstairs","":"downstairs","session_id":"6c2734fe-95f0-4178-bc77-8b1fcc92dec0"} {"msg":"[2]R flush_numbers[0..12]: [198, 226, 227, 227, 227, 227, 226, 223, 227, 227, 223, 226]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:44.351506553Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1678,"":"downstairs","":"downstairs","session_id":"6c2734fe-95f0-4178-bc77-8b1fcc92dec0"} {"msg":"[2]R generation[0..12]: [108, 121, 122, 122, 122, 122, 121, 119, 122, 122, 119, 121]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:44.351520652Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1678,"":"downstairs","":"downstairs","session_id":"6c2734fe-95f0-4178-bc77-8b1fcc92dec0"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:44.351534581Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1678,"":"downstairs","":"downstairs","session_id":"6c2734fe-95f0-4178-bc77-8b1fcc92dec0"} {"msg":"Max found gen is 123","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:44.35154771Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1678,"":"downstairs","":"downstairs","session_id":"6c2734fe-95f0-4178-bc77-8b1fcc92dec0"} {"msg":"Generation requested: 124 >= found:123","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:44.351561209Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1678,"":"downstairs","":"downstairs","session_id":"6c2734fe-95f0-4178-bc77-8b1fcc92dec0"} {"msg":"Next flush: 228","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:44.351574098Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1678,"":"downstairs","":"downstairs","session_id":"6c2734fe-95f0-4178-bc77-8b1fcc92dec0"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:44.351586947Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1678,"":"downstairs","":"downstairs","session_id":"6c2734fe-95f0-4178-bc77-8b1fcc92dec0"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:44.351600376Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1678,"session_id":"6c2734fe-95f0-4178-bc77-8b1fcc92dec0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-12-12T20:37:44.351612495Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1678,"client":"0","":"downstairs","session_id":"6c2734fe-95f0-4178-bc77-8b1fcc92dec0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:44.351633884Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1678,"client":"1","":"downstairs","session_id":"6c2734fe-95f0-4178-bc77-8b1fcc92dec0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:44.351647103Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1678,"client":"2","":"downstairs","session_id":"6c2734fe-95f0-4178-bc77-8b1fcc92dec0"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:44.351660442Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1678,"session_id":"6c2734fe-95f0-4178-bc77-8b1fcc92dec0"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:44.351672591Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1678,"session_id":"6c2734fe-95f0-4178-bc77-8b1fcc92dec0"} {"msg":"56d44137-6430-44c9-930a-cfcca40ec125 is now active with session: 6c2734fe-95f0-4178-bc77-8b1fcc92dec0","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:44.35168523Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1678,"session_id":"6c2734fe-95f0-4178-bc77-8b1fcc92dec0"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:44.351697159Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1678,"session_id":"6c2734fe-95f0-4178-bc77-8b1fcc92dec0"} {"msg":"The guest has finished waiting for activation with:124","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:44.351711529Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1678} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 543 len 36864 data: 9 10 8 8 8 8 9 8 8 02/30 Write block 384 len 20480 data: 9 11 12 10 10 03/30 Write block 563 len 4096 data: 8 04/30 Read block 36 len 16384 05/30 Read block 65 len 24576 06/30 Write block 154 len 20480 data: 9 9 9 9 8 07/30 Read block 456 len 8192 08/30 Read block 459 len 4096 09/30 Read block 362 len 8192 10/30 Write block 583 len 28672 data: 12 13 13 15 14 12 11 11/30 Flush 12/30 Write block 339 len 20480 data: 7 8 6 7 9 13/30 Write block 432 len 32768 data: 9 9 7 10 9 8 10 8 14/30 Write block 352 len 40960 data: 10 10 13 12 11 11 8 8 8 9 15/30 Write block 337 len 28672 data: 9 9 8 9 7 8 10 16/30 Write block 427 len 36864 data: 10 10 11 8 10 10 10 8 11 17/30 Write block 90 len 4096 data: 13 18/30 Read block 273 len 28672 19/30 Read block 387 len 28672 20/30 Read block 42 len 24576 21/30 Read block 589 len 12288 22/30 Write block 448 len 20480 data: 13 16 15 16 15 23/30 Write block 126 len 20480 data: 9 8 9 9 9 24/30 Read block 70 len 8192 25/30 Write block 241 len 36864 data: 11 11 9 7 6 7 5 5 5 26/30 Read block 110 len 32768 27/30 Write block 98 len 8192 data: 6 9 28/30 Write block 588 len 32768 data: 13 12 16 14 12 10 7 6 29/30 Read block 152 len 20480 30/30 Write block 70 len 32768 data: 9 8 9 9 7 9 7 10 ---------------------------------------------------------------- Crucible gen:124 GIO:true work queues: Upstairs:0 downstairs:19 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1011 Acked Write 5 Done Done Done false 1012 Acked Write 8 Done Done Done false 1013 Acked Write 10 Done Done Done false 1014 Acked Write 7 Done Done Done false 1015 Acked Write 9 Done Done Done false 1016 Acked Write 1 Done Done Done false 1017 Acked Read 7 Done Done Done false 1018 Acked Read 7 Done Done Done false 1019 Acked Read 6 Done Done Done false 1020 Acked Read 3 Done Done Done false 1021 Acked Write 5 Done Done Done false 1022 Acked Write 5 Done Done Done false 1023 Acked Read 2 Done Done Done false 1024 Acked Write 9 Done Done Done false 1025 Acked Read 8 Done Done Done false 1026 Acked Write 2 Done Done Done false 1027 Acked Write 8 Done Done Done false 1028 Acked Read 5 Done Done Done false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 18 18 18 54 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1010 1010 1010 Downstairs last five completed: 1010 1009 1008 1007 1006 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:37:44.370667501Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1678} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 108 108 108 198 198 198 F F F 1 020-039 121 121 121 226 226 226 F F F 2 040-059 122 122 122 227 227 227 F F F 3 060-079 122 122 122 227 227 227 T T T 4 080-099 122 122 122 227 227 227 T T T 5 100-119 122 122 122 227 227 227 F F F 6 120-139 121 121 121 226 226 226 T T T 7 140-159 124 124 124 228 228 228 F F F 8 160-179 122 122 122 227 227 227 F F F 9 180-199 122 122 122 227 227 227 F F F 10 200-219 119 119 119 223 223 223 F F F 11 220-239 121 121 121 226 226 226 F F F 12 240-259 121 121 121 226 226 226 T T T 13 260-279 120 120 120 224 224 224 F F F 14 280-299 122 122 122 227 227 227 F F F 15 300-319 122 122 122 227 227 227 F F F 16 320-339 122 122 122 227 227 227 T T T 17 340-359 122 122 122 227 227 227 T T T 18 360-379 122 122 122 227 227 227 T T T 19 380-399 124 124 124 228 228 228 F F F 20 400-419 121 121 121 226 226 226 F F F 21 420-439 112 112 112 208 208 208 T T T 22 440-459 120 120 120 225 225 225 T T T 23 460-479 118 118 118 222 222 222 F F F 24 480-499 121 121 121 226 226 226 F F F 25 500-519 115 115 115 216 216 216 F F F 26 520-539 108 108 108 198 198 198 F F F 27 540-559 124 124 124 228 228 228 F F F 28 560-579 124 124 124 228 228 228 F F F 29 580-599 124 124 124 228 228 228 T T T Max gen: 124, Max flush: 228 A Difference in extent metadata is expected here On loop 62 Verifying data now No change after verify Loop: 62 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 108 108 108 198 198 198 F F F 1 020-039 121 121 121 226 226 226 F F F 2 040-059 122 122 122 227 227 227 F F F 3 060-079 125 125 125 229 229 229 F F F 4 080-099 125 125 125 229 229 229 F F F 5 100-119 122 122 122 227 227 227 F F F 6 120-139 125 125 125 229 229 229 F F F 7 140-159 124 124 124 228 228 228 F F F 8 160-179 122 122 122 227 227 227 F F F 9 180-199 122 122 122 227 227 227 F F F 10 200-219 119 119 119 223 223 223 F F F 11 220-239 121 121 121 226 226 226 F F F 12 240-259 125 125 125 229 229 229 F F F 13 260-279 120 120 120 224 224 224 F F F 14 280-299 122 122 122 227 227 227 F F F 15 300-319 122 122 122 227 227 227 F F F 16 320-339 125 125 125 229 229 229 F F F 17 340-359 125 125 125 229 229 229 F F F 18 360-379 125 125 125 229 229 229 F F F 19 380-399 124 124 124 228 228 228 F F F 20 400-419 121 121 121 226 226 226 F F F 21 420-439 125 125 125 229 229 229 F F F 22 440-459 125 125 125 229 229 229 F F F 23 460-479 118 118 118 222 222 222 F F F 24 480-499 121 121 121 226 226 226 F F F 25 500-519 115 115 115 216 216 216 F F F 26 520-539 108 108 108 198 198 198 F F F 27 540-559 124 124 124 228 228 228 F F F 28 560-579 124 124 124 228 228 228 F F F 29 580-599 125 125 125 229 229 229 F F F Max gen: 125, Max flush: 229 Begin loop 63 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:48.667470281Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1686,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:48.711756427Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1686,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:48.712480857Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1686} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"26e7d501b33f47379e53601720353c2983e7654b\",\n git_commit_timestamp: \"2024-12-12T20:16:40.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:48.712535044Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1686} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:48.712557032Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1686} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:48.71258193Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1686} {"msg":"Crucible 99d6e494-1119-4a54-a0b7-25c5c969b47e has session id: 31acea96-fa94-4428-9a6f-8410dd85a726","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:48.713030599Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1686,"session_id":"31acea96-fa94-4428-9a6f-8410dd85a726"} {"msg":"Upstairs opts: Upstairs UUID: 99d6e494-1119-4a54-a0b7-25c5c969b47e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:48.713079336Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1686,"session_id":"31acea96-fa94-4428-9a6f-8410dd85a726"} {"msg":"Crucible stats registered with UUID: 99d6e494-1119-4a54-a0b7-25c5c969b47e","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:48.713104014Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1686,"session_id":"31acea96-fa94-4428-9a6f-8410dd85a726"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:48.743596871Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1686,"task":"crutest"} {"msg":"The guest has requested activation with gen:126","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:48.743629419Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1686} {"msg":"99d6e494-1119-4a54-a0b7-25c5c969b47e active request set","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:48.743662547Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1686,"session_id":"31acea96-fa94-4428-9a6f-8410dd85a726"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:48.743679066Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1686,"client":"0","":"downstairs","session_id":"31acea96-fa94-4428-9a6f-8410dd85a726"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:48.743707614Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1686,"client":"0","":"downstairs","session_id":"31acea96-fa94-4428-9a6f-8410dd85a726"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:48.743725253Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1686,"client":"1","":"downstairs","session_id":"31acea96-fa94-4428-9a6f-8410dd85a726"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:48.743742712Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1686,"client":"1","":"downstairs","session_id":"31acea96-fa94-4428-9a6f-8410dd85a726"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:48.74375892Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1686,"client":"2","":"downstairs","session_id":"31acea96-fa94-4428-9a6f-8410dd85a726"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:48.743791228Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1686,"client":"2","":"downstairs","session_id":"31acea96-fa94-4428-9a6f-8410dd85a726"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:48.743814376Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1686,"":"io task","client":"2","":"downstairs","session_id":"31acea96-fa94-4428-9a6f-8410dd85a726"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:48.743846164Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1686,"":"io task","client":"0","":"downstairs","session_id":"31acea96-fa94-4428-9a6f-8410dd85a726"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:48.743879182Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1686,"":"io task","client":"1","":"downstairs","session_id":"31acea96-fa94-4428-9a6f-8410dd85a726"} {"msg":"ds_connection connected from Ok(127.0.0.1:44915)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:48.74391409Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1686,"":"io task","client":"1","":"downstairs","session_id":"31acea96-fa94-4428-9a6f-8410dd85a726"} {"msg":"ds_connection connected from Ok(127.0.0.1:37839)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:48.743950577Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1686,"":"io task","client":"2","":"downstairs","session_id":"31acea96-fa94-4428-9a6f-8410dd85a726"} {"msg":"ds_connection connected from Ok(127.0.0.1:64696)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:48.743984035Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1686,"":"io task","client":"0","":"downstairs","session_id":"31acea96-fa94-4428-9a6f-8410dd85a726"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:48.74463089Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1686,"client":"0","":"downstairs","session_id":"31acea96-fa94-4428-9a6f-8410dd85a726"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:48.744663988Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1686,"client":"1","":"downstairs","session_id":"31acea96-fa94-4428-9a6f-8410dd85a726"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:48.744679097Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1686,"client":"2","":"downstairs","session_id":"31acea96-fa94-4428-9a6f-8410dd85a726"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:48.745226859Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1686,"client":"0","":"downstairs","session_id":"31acea96-fa94-4428-9a6f-8410dd85a726"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:48.74535709Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1686,"client":"1","":"downstairs","session_id":"31acea96-fa94-4428-9a6f-8410dd85a726"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:48.745389498Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1686,"client":"2","":"downstairs","session_id":"31acea96-fa94-4428-9a6f-8410dd85a726"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:48.745806549Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1686,"session_id":"31acea96-fa94-4428-9a6f-8410dd85a726"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:48.745850416Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1686,"session_id":"31acea96-fa94-4428-9a6f-8410dd85a726"} {"msg":"[0]R flush_numbers[0..12]: [198, 226, 227, 229, 229, 227, 229, 228, 227, 227, 223, 226]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:48.745962958Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1686,"":"downstairs","":"downstairs","session_id":"31acea96-fa94-4428-9a6f-8410dd85a726"} {"msg":"[0]R generation[0..12]: [108, 121, 122, 125, 125, 122, 125, 124, 122, 122, 119, 121]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:48.745983977Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1686,"":"downstairs","":"downstairs","session_id":"31acea96-fa94-4428-9a6f-8410dd85a726"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:48.745999076Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1686,"":"downstairs","":"downstairs","session_id":"31acea96-fa94-4428-9a6f-8410dd85a726"} {"msg":"[1]R flush_numbers[0..12]: [198, 226, 227, 229, 229, 227, 229, 228, 227, 227, 223, 226]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:48.746013785Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1686,"":"downstairs","":"downstairs","session_id":"31acea96-fa94-4428-9a6f-8410dd85a726"} {"msg":"[1]R generation[0..12]: [108, 121, 122, 125, 125, 122, 125, 124, 122, 122, 119, 121]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:48.746027734Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1686,"":"downstairs","":"downstairs","session_id":"31acea96-fa94-4428-9a6f-8410dd85a726"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:48.746041693Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1686,"":"downstairs","":"downstairs","session_id":"31acea96-fa94-4428-9a6f-8410dd85a726"} {"msg":"[2]R flush_numbers[0..12]: [198, 226, 227, 229, 229, 227, 229, 228, 227, 227, 223, 226]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:48.746055272Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1686,"":"downstairs","":"downstairs","session_id":"31acea96-fa94-4428-9a6f-8410dd85a726"} {"msg":"[2]R generation[0..12]: [108, 121, 122, 125, 125, 122, 125, 124, 122, 122, 119, 121]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:48.746068991Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1686,"":"downstairs","":"downstairs","session_id":"31acea96-fa94-4428-9a6f-8410dd85a726"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:48.74608287Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1686,"":"downstairs","":"downstairs","session_id":"31acea96-fa94-4428-9a6f-8410dd85a726"} {"msg":"Max found gen is 126","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:48.746095959Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1686,"":"downstairs","":"downstairs","session_id":"31acea96-fa94-4428-9a6f-8410dd85a726"} {"msg":"Generation requested: 126 >= found:126","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:48.746109138Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1686,"":"downstairs","":"downstairs","session_id":"31acea96-fa94-4428-9a6f-8410dd85a726"} {"msg":"Next flush: 230","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:48.746121977Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1686,"":"downstairs","":"downstairs","session_id":"31acea96-fa94-4428-9a6f-8410dd85a726"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:48.746134787Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1686,"":"downstairs","":"downstairs","session_id":"31acea96-fa94-4428-9a6f-8410dd85a726"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-12-12T20:37:48.746150895Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1686,"session_id":"31acea96-fa94-4428-9a6f-8410dd85a726"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:48.746165294Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1686,"client":"0","":"downstairs","session_id":"31acea96-fa94-4428-9a6f-8410dd85a726"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:48.746179044Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1686,"client":"1","":"downstairs","session_id":"31acea96-fa94-4428-9a6f-8410dd85a726"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:48.746192492Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1686,"client":"2","":"downstairs","session_id":"31acea96-fa94-4428-9a6f-8410dd85a726"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:48.746205892Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1686,"session_id":"31acea96-fa94-4428-9a6f-8410dd85a726"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:48.746218311Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1686,"session_id":"31acea96-fa94-4428-9a6f-8410dd85a726"} {"msg":"99d6e494-1119-4a54-a0b7-25c5c969b47e is now active with session: 31acea96-fa94-4428-9a6f-8410dd85a726","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:48.74623096Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1686,"session_id":"31acea96-fa94-4428-9a6f-8410dd85a726"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:48.746243149Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1686,"session_id":"31acea96-fa94-4428-9a6f-8410dd85a726"} {"msg":"The guest has finished waiting for activation with:126","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:48.746255478Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1686} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 382 len 24576 data: 13 11 10 12 13 11 02/30 Read block 109 len 36864 03/30 Read block 370 len 16384 04/30 Read block 480 len 32768 05/30 Read block 371 len 20480 06/30 Read block 98 len 40960 07/30 Read block 258 len 40960 08/30 Write block 386 len 4096 data: 14 09/30 Read block 92 len 32768 10/30 Read block 100 len 12288 11/30 Read block 408 len 40960 12/30 Read block 389 len 24576 13/30 Write block 208 len 16384 data: 12 10 13 15 14/30 Read block 245 len 24576 15/30 Flush 16/30 Write block 306 len 32768 data: 8 7 8 7 6 10 9 12 17/30 Read block 237 len 32768 18/30 Read block 184 len 24576 19/30 Write block 394 len 16384 data: 11 13 13 14 20/30 Read block 314 len 20480 21/30 Write block 533 len 8192 data: 5 5 22/30 Read block 106 len 8192 23/30 Read block 57 len 16384 24/30 Write block 587 len 28672 data: 15 14 13 17 15 13 11 25/30 Write block 379 len 24576 data: 12 12 12 14 12 11 26/30 Write block 299 len 28672 data: 7 8 7 6 6 5 7 27/30 Write block 93 len 12288 data: 11 10 10 28/30 Read block 492 len 8192 29/30 Read block 274 len 36864 30/30 Write block 222 len 36864 data: 6 8 7 9 12 10 11 13 13 ---------------------------------------------------------------- Crucible gen:126 GIO:true work queues: Upstairs:0 downstairs:15 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1015 Acked Write 8 Done Done Done false 1016 Acked Read 8 Done Done Done false 1017 Acked Read 6 Done Done Done false 1018 Acked Write 4 Done Done Done false 1019 Acked Read 5 Done Done Done false 1020 Acked Write 2 Done Done Done false 1021 Acked Read 2 Done Done Done false 1022 Acked Read 4 Done Done Done false 1023 Acked Write 7 Done Done Done false 1024 Acked Write 6 Done Done Done false 1025 Acked Write 7 Done Done Done false 1026 Acked Write 3 Done Done Done false 1027 Acked Read 2 Done Done Done false 1028 Acked Read 9 Done Done Done false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 14 14 14 42 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1014 1014 1014 Downstairs last five completed: 1014 1013 1012 1011 1010 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:37:48.76638615Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1686} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 108 108 108 198 198 198 F F F 1 020-039 121 121 121 226 226 226 F F F 2 040-059 122 122 122 227 227 227 F F F 3 060-079 125 125 125 229 229 229 F F F 4 080-099 125 125 125 229 229 229 T T T 5 100-119 122 122 122 227 227 227 F F F 6 120-139 125 125 125 229 229 229 F F F 7 140-159 124 124 124 228 228 228 F F F 8 160-179 122 122 122 227 227 227 F F F 9 180-199 122 122 122 227 227 227 F F F 10 200-219 126 126 126 230 230 230 F F F 11 220-239 121 121 121 226 226 226 T T T 12 240-259 125 125 125 229 229 229 F F F 13 260-279 120 120 120 224 224 224 F F F 14 280-299 122 122 122 227 227 227 T T T 15 300-319 122 122 122 227 227 227 T T T 16 320-339 125 125 125 229 229 229 F F F 17 340-359 125 125 125 229 229 229 F F F 18 360-379 125 125 125 229 229 229 T T T 19 380-399 126 126 126 230 230 230 T T T 20 400-419 121 121 121 226 226 226 F F F 21 420-439 125 125 125 229 229 229 F F F 22 440-459 125 125 125 229 229 229 F F F 23 460-479 118 118 118 222 222 222 F F F 24 480-499 121 121 121 226 226 226 F F F 25 500-519 115 115 115 216 216 216 F F F 26 520-539 108 108 108 198 198 198 T T T 27 540-559 124 124 124 228 228 228 F F F 28 560-579 124 124 124 228 228 228 F F F 29 580-599 125 125 125 229 229 229 T T T Max gen: 126, Max flush: 230 A Difference in extent metadata is expected here On loop 63 Verifying data now No change after verify Loop: 63 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 108 108 108 198 198 198 F F F 1 020-039 121 121 121 226 226 226 F F F 2 040-059 122 122 122 227 227 227 F F F 3 060-079 125 125 125 229 229 229 F F F 4 080-099 127 127 127 231 231 231 F F F 5 100-119 122 122 122 227 227 227 F F F 6 120-139 125 125 125 229 229 229 F F F 7 140-159 124 124 124 228 228 228 F F F 8 160-179 122 122 122 227 227 227 F F F 9 180-199 122 122 122 227 227 227 F F F 10 200-219 126 126 126 230 230 230 F F F 11 220-239 127 127 127 231 231 231 F F F 12 240-259 125 125 125 229 229 229 F F F 13 260-279 120 120 120 224 224 224 F F F 14 280-299 127 127 127 231 231 231 F F F 15 300-319 127 127 127 231 231 231 F F F 16 320-339 125 125 125 229 229 229 F F F 17 340-359 125 125 125 229 229 229 F F F 18 360-379 127 127 127 231 231 231 F F F 19 380-399 127 127 127 231 231 231 F F F 20 400-419 121 121 121 226 226 226 F F F 21 420-439 125 125 125 229 229 229 F F F 22 440-459 125 125 125 229 229 229 F F F 23 460-479 118 118 118 222 222 222 F F F 24 480-499 121 121 121 226 226 226 F F F 25 500-519 115 115 115 216 216 216 F F F 26 520-539 127 127 127 231 231 231 F F F 27 540-559 124 124 124 228 228 228 F F F 28 560-579 124 124 124 228 228 228 F F F 29 580-599 127 127 127 231 231 231 F F F Max gen: 127, Max flush: 231 Begin loop 64 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:53.058490953Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1694,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:53.103275895Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1694,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:53.103954888Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1694} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"26e7d501b33f47379e53601720353c2983e7654b\",\n git_commit_timestamp: \"2024-12-12T20:16:40.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:53.103993175Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1694} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:53.104007744Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1694} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:53.104022583Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1694} {"msg":"Crucible 137493dd-f031-4eec-bec7-e0819794c7a1 has session id: d0f1718d-88bf-4334-964f-4d91f08432df","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:53.104483541Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1694,"session_id":"d0f1718d-88bf-4334-964f-4d91f08432df"} {"msg":"Upstairs opts: Upstairs UUID: 137493dd-f031-4eec-bec7-e0819794c7a1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:53.104528328Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1694,"session_id":"d0f1718d-88bf-4334-964f-4d91f08432df"} {"msg":"Crucible stats registered with UUID: 137493dd-f031-4eec-bec7-e0819794c7a1","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:53.104545277Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1694,"session_id":"d0f1718d-88bf-4334-964f-4d91f08432df"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:53.135995778Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1694,"task":"crutest"} {"msg":"The guest has requested activation with gen:128","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:53.136028386Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1694} {"msg":"137493dd-f031-4eec-bec7-e0819794c7a1 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:53.136096181Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1694,"session_id":"d0f1718d-88bf-4334-964f-4d91f08432df"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:53.136129989Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1694,"client":"0","":"downstairs","session_id":"d0f1718d-88bf-4334-964f-4d91f08432df"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:53.136163677Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1694,"client":"0","":"downstairs","session_id":"d0f1718d-88bf-4334-964f-4d91f08432df"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:53.136197334Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1694,"client":"1","":"downstairs","session_id":"d0f1718d-88bf-4334-964f-4d91f08432df"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:53.136231772Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1694,"client":"1","":"downstairs","session_id":"d0f1718d-88bf-4334-964f-4d91f08432df"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:53.13625911Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1694,"client":"2","":"downstairs","session_id":"d0f1718d-88bf-4334-964f-4d91f08432df"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:53.136290658Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1694,"client":"2","":"downstairs","session_id":"d0f1718d-88bf-4334-964f-4d91f08432df"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:53.136319106Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1694,"":"io task","client":"2","":"downstairs","session_id":"d0f1718d-88bf-4334-964f-4d91f08432df"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:53.136340975Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1694,"":"io task","client":"0","":"downstairs","session_id":"d0f1718d-88bf-4334-964f-4d91f08432df"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:53.136372432Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1694,"":"io task","client":"1","":"downstairs","session_id":"d0f1718d-88bf-4334-964f-4d91f08432df"} {"msg":"ds_connection connected from Ok(127.0.0.1:34768)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:53.13640497Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1694,"":"io task","client":"0","":"downstairs","session_id":"d0f1718d-88bf-4334-964f-4d91f08432df"} {"msg":"ds_connection connected from Ok(127.0.0.1:35039)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:53.136437298Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1694,"":"io task","client":"2","":"downstairs","session_id":"d0f1718d-88bf-4334-964f-4d91f08432df"} {"msg":"ds_connection connected from Ok(127.0.0.1:49966)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:53.136528192Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1694,"":"io task","client":"1","":"downstairs","session_id":"d0f1718d-88bf-4334-964f-4d91f08432df"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:53.137228133Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1694,"client":"1","":"downstairs","session_id":"d0f1718d-88bf-4334-964f-4d91f08432df"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:53.137258361Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1694,"client":"2","":"downstairs","session_id":"d0f1718d-88bf-4334-964f-4d91f08432df"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:53.137285689Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1694,"client":"0","":"downstairs","session_id":"d0f1718d-88bf-4334-964f-4d91f08432df"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:53.13785152Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1694,"client":"1","":"downstairs","session_id":"d0f1718d-88bf-4334-964f-4d91f08432df"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:53.137900867Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1694,"client":"2","":"downstairs","session_id":"d0f1718d-88bf-4334-964f-4d91f08432df"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:53.138135571Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1694,"client":"0","":"downstairs","session_id":"d0f1718d-88bf-4334-964f-4d91f08432df"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:53.138409882Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1694,"session_id":"d0f1718d-88bf-4334-964f-4d91f08432df"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:53.138524754Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1694,"session_id":"d0f1718d-88bf-4334-964f-4d91f08432df"} {"msg":"[0]R flush_numbers[0..12]: [198, 226, 227, 229, 231, 227, 229, 228, 227, 227, 230, 231]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:53.138755598Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1694,"":"downstairs","":"downstairs","session_id":"d0f1718d-88bf-4334-964f-4d91f08432df"} {"msg":"[0]R generation[0..12]: [108, 121, 122, 125, 127, 122, 125, 124, 122, 122, 126, 127]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:53.138775127Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1694,"":"downstairs","":"downstairs","session_id":"d0f1718d-88bf-4334-964f-4d91f08432df"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:53.138789756Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1694,"":"downstairs","":"downstairs","session_id":"d0f1718d-88bf-4334-964f-4d91f08432df"} {"msg":"[1]R flush_numbers[0..12]: [198, 226, 227, 229, 231, 227, 229, 228, 227, 227, 230, 231]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:53.138803675Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1694,"":"downstairs","":"downstairs","session_id":"d0f1718d-88bf-4334-964f-4d91f08432df"} {"msg":"[1]R generation[0..12]: [108, 121, 122, 125, 127, 122, 125, 124, 122, 122, 126, 127]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:53.138817374Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1694,"":"downstairs","":"downstairs","session_id":"d0f1718d-88bf-4334-964f-4d91f08432df"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:53.138831333Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1694,"":"downstairs","":"downstairs","session_id":"d0f1718d-88bf-4334-964f-4d91f08432df"} {"msg":"[2]R flush_numbers[0..12]: [198, 226, 227, 229, 231, 227, 229, 228, 227, 227, 230, 231]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:53.138844912Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1694,"":"downstairs","":"downstairs","session_id":"d0f1718d-88bf-4334-964f-4d91f08432df"} {"msg":"[2]R generation[0..12]: [108, 121, 122, 125, 127, 122, 125, 124, 122, 122, 126, 127]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:53.138862801Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1694,"":"downstairs","":"downstairs","session_id":"d0f1718d-88bf-4334-964f-4d91f08432df"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:53.13887752Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1694,"":"downstairs","":"downstairs","session_id":"d0f1718d-88bf-4334-964f-4d91f08432df"} {"msg":"Max found gen is 128","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:53.138890759Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1694,"":"downstairs","":"downstairs","session_id":"d0f1718d-88bf-4334-964f-4d91f08432df"} {"msg":"Generation requested: 128 >= found:128","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:53.138903998Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1694,"":"downstairs","":"downstairs","session_id":"d0f1718d-88bf-4334-964f-4d91f08432df"} {"msg":"Next flush: 232","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:53.138916947Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1694,"":"downstairs","":"downstairs","session_id":"d0f1718d-88bf-4334-964f-4d91f08432df"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:53.138929836Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1694Wait for a query_work_queue command to finish before sending IO ,"":"downstairs","":"downstairs","session_id":"d0f1718d-88bf-4334-964f-4d91f08432df"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:53.138946925Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1694,"session_id":"d0f1718d-88bf-4334-964f-4d91f08432df"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:53.138958864Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1694,"client":"0","":"downstairs","session_id":"d0f1718d-88bf-4334-964f-4d91f08432df"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:53.138972123Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1694,"client":"1","":"downstairs","session_id":"d0f1718d-88bf-4334-964f-4d91f08432df"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:53.138985102Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1694,"client":"2","":"downstairs","session_id":"d0f1718d-88bf-4334-964f-4d91f08432df"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:53.138998561Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1694,"session_id":"d0f1718d-88bf-4334-964f-4d91f08432df"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:53.13901068Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1694,"session_id":"d0f1718d-88bf-4334-964f-4d91f08432df"} {"msg":"137493dd-f031-4eec-bec7-e0819794c7a1 is now active with session: d0f1718d-88bf-4334-964f-4d91f08432df","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:53.13902327Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1694,"session_id":"d0f1718d-88bf-4334-964f-4d91f08432df"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:53.139035109Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1694,"session_id":"d0f1718d-88bf-4334-964f-4d91f08432df"} {"msg":"The guest has finished waiting for activation with:128","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:53.139047198Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1694} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 260 len 40960 data: 9 5 6 8 10 13 15 15 14 8 03/30 Write block 527 len 12288 data: 9 8 8 04/30 Read block 524 len 24576 05/30 Read block 106 len 12288 06/30 Read block 68 len 20480 07/30 Read block 76 len 32768 08/30 Read block 522 len 12288 09/30 Read block 71 len 4096 10/30 Read block 487 len 24576 11/30 Read block 13 len 28672 12/30 Write block 585 len 32768 data: 14 16 16 15 14 18 16 14 13/30 Write block 14 len 16384 data: 7 8 5 5 14/30 Read block 115 len 4096 15/30 Read block 30 len 28672 16/30 Write block 326 len 28672 data: 6 5 6 10 11 14 11 17/30 Read block 552 len 40960 18/30 Write block 430 len 16384 data: 9 11 11 11 19/30 Read block 57 len 28672 20/30 Flush 21/30 Write block 39 len 28672 data: 12 10 10 9 9 9 11 22/30 Flush 23/30 Write block 289 len 24576 data: 8 7 10 10 11 11 24/30 Flush 25/30 Write block 115 len 32768 data: 10 10 8 9 8 11 11 10 26/30 Write block 570 len 28672 data: 9 12 12 12 12 11 10 27/30 Write block 355 len 40960 data: 13 12 12 9 9 9 10 10 10 13 28/30 Write block 427 len 24576 data: 11 11 12 10 12 12 29/30 Read block 514 len 4096 30/30 Write block 350 len 28672 data: 12 8 11 11 14 14 13 ---------------------------------------------------------------- Crucible gen:128 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 8 Done Done Done false 1025 Acked Write 7 Done Done Done false 1026 Acked Write 10 Done Done Done false 1027 Acked Write 6 Done Done Done false 1028 Acked Read 1 Done Done Done false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:37:53.159271853Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1694} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 128 128 128 233 233 233 F F F 1 020-039 128 128 128 234 234 234 F F F 2 040-059 128 128 128 234 234 234 F F F 3 060-079 125 125 125 229 229 229 F F F 4 080-099 127 127 127 231 231 231 F F F 5 100-119 122 122 122 227 227 227 T T T 6 120-139 125 125 125 229 229 229 T T T 7 140-159 124 124 124 228 228 228 F F F 8 160-179 122 122 122 227 227 227 F F F 9 180-199 122 122 122 227 227 227 F F F 10 200-219 126 126 126 230 230 230 F F F 11 220-239 127 127 127 231 231 231 F F F 12 240-259 125 125 125 229 229 229 F F F 13 260-279 128 128 128 233 233 233 F F F 14 280-299 128 128 128 235 235 235 F F F 15 300-319 127 127 127 231 231 231 F F F 16 320-339 128 128 128 233 233 233 F F F 17 340-359 125 125 125 229 229 229 T T T 18 360-379 127 127 127 231 231 231 T T T 19 380-399 127 127 127 231 231 231 F F F 20 400-419 121 121 121 226 226 226 F F F 21 420-439 128 128 128 233 233 233 T T T 22 440-459 125 125 125 229 229 229 F F F 23 460-479 118 118 118 222 222 222 F F F 24 480-499 121 121 121 226 226 226 F F F 25 500-519 115 115 115 216 216 216 F F F 26 520-539 128 128 128 233 233 233 F F F 27 540-559 124 124 124 228 228 228 F F F 28 560-579 124 124 124 228 228 228 T T T 29 580-599 128 128 128 233 233 233 F F F Max gen: 128, Max flush: 235 A Difference in extent metadata is expected here On loop 64 Verifying data now No change after verify Loop: 64 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 128 128 128 233 233 233 F F F 1 020-039 128 128 128 234 234 234 F F F 2 040-059 128 128 128 234 234 234 F F F 3 060-079 125 125 125 229 229 229 F F F 4 080-099 127 127 127 231 231 231 F F F 5 100-119 129 129 129 236 236 236 F F F 6 120-139 129 129 129 236 236 236 F F F 7 140-159 124 124 124 228 228 228 F F F 8 160-179 122 122 122 227 227 227 F F F 9 180-199 122 122 122 227 227 227 F F F 10 200-219 126 126 126 230 230 230 F F F 11 220-239 127 127 127 231 231 231 F F F 12 240-259 125 125 125 229 229 229 F F F 13 260-279 128 128 128 233 233 233 F F F 14 280-299 128 128 128 235 235 235 F F F 15 300-319 127 127 127 231 231 231 F F F 16 320-339 128 128 128 233 233 233 F F F 17 340-359 129 129 129 236 236 236 F F F 18 360-379 129 129 129 236 236 236 F F F 19 380-399 127 127 127 231 231 231 F F F 20 400-419 121 121 121 226 226 226 F F F 21 420-439 129 129 129 236 236 236 F F F 22 440-459 125 125 125 229 229 229 F F F 23 460-479 118 118 118 222 222 222 F F F 24 480-499 121 121 121 226 226 226 F F F 25 500-519 115 115 115 216 216 216 F F F 26 520-539 128 128 128 233 233 233 F F F 27 540-559 124 124 124 228 228 228 F F F 28 560-579 129 129 129 236 236 236 F F F 29 580-599 128 128 128 233 233 233 F F F Max gen: 129, Max flush: 236 Begin loop 65 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:57.432372027Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1702,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:57.476630666Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1702,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:57.477385873Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1702} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"26e7d501b33f47379e53601720353c2983e7654b\",\n git_commit_timestamp: \"2024-12-12T20:16:40.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:57.477421281Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1702} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:57.47743513Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1702} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:57.477449189Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1702} {"msg":"Crucible a0341d19-bbb2-4735-8e8a-92ab8eeb5e18 has session id: d7571ec3-2ae5-4ba3-99d7-4a643ae7635b","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:57.47786386Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1702,"session_id":"d7571ec3-2ae5-4ba3-99d7-4a643ae7635b"} {"msg":"Upstairs opts: Upstairs UUID: a0341d19-bbb2-4735-8e8a-92ab8eeb5e18, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:57.477910277Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1702,"session_id":"d7571ec3-2ae5-4ba3-99d7-4a643ae7635b"} {"msg":"Crucible stats registered with UUID: a0341d19-bbb2-4735-8e8a-92ab8eeb5e18","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:57.477937305Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1702,"session_id":"d7571ec3-2ae5-4ba3-99d7-4a643ae7635b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:57.509256396Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1702,"task":"crutest"} {"msg":"The guest has requested activation with gen:130","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:57.509290363Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1702} {"msg":"a0341d19-bbb2-4735-8e8a-92ab8eeb5e18 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:57.509361629Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1702,"session_id":"d7571ec3-2ae5-4ba3-99d7-4a643ae7635b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:57.509383277Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1702,"client":"0","":"downstairs","session_id":"d7571ec3-2ae5-4ba3-99d7-4a643ae7635b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:57.509410425Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1702,"client":"0","":"downstairs","session_id":"d7571ec3-2ae5-4ba3-99d7-4a643ae7635b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:57.509441943Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1702,"client":"1","":"downstairs","session_id":"d7571ec3-2ae5-4ba3-99d7-4a643ae7635b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:57.509476601Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1702,"client":"1","":"downstairs","session_id":"d7571ec3-2ae5-4ba3-99d7-4a643ae7635b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:57.509512488Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1702,"client":"2","":"downstairs","session_id":"d7571ec3-2ae5-4ba3-99d7-4a643ae7635b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:57.509549545Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1702,"client":"2","":"downstairs","session_id":"d7571ec3-2ae5-4ba3-99d7-4a643ae7635b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:57.509583413Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1702,"":"io task","client":"2","":"downstairs","session_id":"d7571ec3-2ae5-4ba3-99d7-4a643ae7635b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:57.509616941Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1702,"":"io task","client":"0","":"downstairs","session_id":"d7571ec3-2ae5-4ba3-99d7-4a643ae7635b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:57.509649749Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1702,"":"io task","client":"1","":"downstairs","session_id":"d7571ec3-2ae5-4ba3-99d7-4a643ae7635b"} {"msg":"ds_connection connected from Ok(127.0.0.1:61381)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:57.509682086Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1702,"":"io task","client":"2","":"downstairs","session_id":"d7571ec3-2ae5-4ba3-99d7-4a643ae7635b"} {"msg":"ds_connection connected from Ok(127.0.0.1:50230)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:57.50991373Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1702,"":"io task","client":"0","":"downstairs","session_id":"d7571ec3-2ae5-4ba3-99d7-4a643ae7635b"} {"msg":"ds_connection connected from Ok(127.0.0.1:62532)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:57.509964837Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1702,"":"io task","client":"1","":"downstairs","session_id":"d7571ec3-2ae5-4ba3-99d7-4a643ae7635b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:57.510417096Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1702,"client":"2","":"downstairs","session_id":"d7571ec3-2ae5-4ba3-99d7-4a643ae7635b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:57.51050042Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1702,"client":"0","":"downstairs","session_id":"d7571ec3-2ae5-4ba3-99d7-4a643ae7635b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:57.510544727Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1702,"client":"1","":"downstairs","session_id":"d7571ec3-2ae5-4ba3-99d7-4a643ae7635b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:57.511019304Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1702,"client":"0","":"downstairs","session_id":"d7571ec3-2ae5-4ba3-99d7-4a643ae7635b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:57.511061371Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1702,"client":"2","":"downstairs","session_id":"d7571ec3-2ae5-4ba3-99d7-4a643ae7635b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:57.511166754Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1702,"client":"1","":"downstairs","session_id":"d7571ec3-2ae5-4ba3-99d7-4a643ae7635b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:57.511583085Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1702,"session_id":"d7571ec3-2ae5-4ba3-99d7-4a643ae7635b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:57.51166234Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1702,"session_id":"d7571ec3-2ae5-4ba3-99d7-4a643ae7635b"} {"msg":"[0]R flush_numbers[0..12]: [233, 234, 234, 229, 231, 236, 236, 228, 227, 227, 230, 231]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:57.511777832Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1702,"":"downstairs","":"downstairs","session_id":"d7571ec3-2ae5-4ba3-99d7-4a643ae7635b"} {"msg":"[0]R generation[0..12]: [128, 128, 128, 125, 127, 129, 129, 124, 122, 122, 126, 127]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:57.511798401Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1702,"":"downstairs","":"downstairs","session_id":"d7571ec3-2ae5-4ba3-99d7-4a643ae7635b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:57.511813219Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1702,"":"downstairs","":"downstairs","session_id":"d7571ec3-2ae5-4ba3-99d7-4a643ae7635b"} {"msg":"[1]R flush_numbers[0..12]: [233, 234, 234, 229, 231, 236, 236, 228, 227, 227, 230, 231]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:57.511827219Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1702,"":"downstairs","":"downstairs","session_id":"d7571ec3-2ae5-4ba3-99d7-4a643ae7635b"} {"msg":"[1]R generation[0..12]: [128, 128, 128, 125, 127, 129, 129, 124, 122, 122, 126, 127]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:57.511844857Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1702,"":"downstairs","":"downstairs","session_id":"d7571ec3-2ae5-4ba3-99d7-4a643ae7635b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:57.511859316Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1702,"":"downstairs","":"downstairs","session_id":"d7571ec3-2ae5-4ba3-99d7-4a643ae7635b"} {"msg":"[2]R flush_numbers[0..12]: [233, 234, 234, 229, 231, 236, 236, 228, 227, 227, 230, 231]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:57.511872985Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1702,"":"downstairs","":"downstairs","session_id":"d7571ec3-2ae5-4ba3-99d7-4a643ae7635b"} {"msg":"[2]R generation[0..12]: [128, 128, 128, 125, 127, 129, 129, 124, 122, 122, 126, 127]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:57.511886714Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1702,"":"downstairs","":"downstairs","session_id":"d7571ec3-2ae5-4ba3-99d7-4a643ae7635b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:57.511900703Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1702,"":"downstairs","":"downstairs","session_id":"d7571ec3-2ae5-4ba3-99d7-4a643ae7635b"} {"msg":"Max found gen is 130","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:57.511913822Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1702,"":"downstairs","":"downstairs","session_id":"d7571ec3-2ae5-4ba3-99d7-4a643ae7635b"} {"msg":"Generation requested: 130 >= found:130","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:57.511927001Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1702,"":"downstairs","":"downstairs","session_id":"d7571ec3-2ae5-4ba3-99d7-4a643ae7635b"} {"msg":"Next flush: 237","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:57.511940741Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1702,"":"downstairs","":"downstairs","session_id":"d7571ec3-2ae5-4ba3-99d7-4a643ae7635b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:57.51195372Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1702,"":"downstairs","":"downstairs","session_id":"d7571ec3-2ae5-4ba3-99d7-4a643ae7635b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:57.511966989Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1702,"session_id":"d7571ec3-2ae5-4ba3-99d7-4a643ae7635b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:57.511978948Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1702,"client":"0","":"downstairs","session_id":"d7571ec3-2ae5-4ba3-99d7-4a643ae7635b"} Wait for a query_work_queue command to finish before sending IO {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:57.512012146Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1702,"client":"1","":"downstairs","session_id":"d7571ec3-2ae5-4ba3-99d7-4a643ae7635b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:57.512027045Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1702,"client":"2","":"downstairs","session_id":"d7571ec3-2ae5-4ba3-99d7-4a643ae7635b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:57.512040334Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1702,"session_id":"d7571ec3-2ae5-4ba3-99d7-4a643ae7635b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:57.512052603Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1702,"session_id":"d7571ec3-2ae5-4ba3-99d7-4a643ae7635b"} {"msg":"a0341d19-bbb2-4735-8e8a-92ab8eeb5e18 is now active with session: d7571ec3-2ae5-4ba3-99d7-4a643ae7635b","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:57.512067212Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1702,"session_id":"d7571ec3-2ae5-4ba3-99d7-4a643ae7635b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:57.512079231Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1702,"session_id":"d7571ec3-2ae5-4ba3-99d7-4a643ae7635b"} {"msg":"The guest has finished waiting for activation with:130","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:37:57.51209139Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1702} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 445 len 20480 data: 8 11 14 14 17 02/30 Write block 124 len 28672 data: 9 10 10 9 10 10 10 03/30 Write block 251 len 12288 data: 6 7 10 04/30 Write block 458 len 28672 data: 14 15 15 18 19 17 15 05/30 Write block 50 len 32768 data: 12 11 9 12 12 15 18 19 06/30 Write block 308 len 32768 data: 9 8 7 11 10 13 14 15 07/30 Write block 485 len 20480 data: 12 11 10 11 12 08/30 Write block 566 len 28672 data: 11 8 10 11 10 13 13 09/30 Read block 427 len 36864 10/30 Write block 266 len 28672 data: 16 16 15 9 9 10 12 11/30 Read block 114 len 8192 12/30 Read block 395 len 8192 13/30 Read block 95 len 16384 14/30 Read block 518 len 28672 15/30 Write block 415 len 4096 data: 8 16/30 Read block 490 len 16384 17/30 Read block 174 len 24576 18/30 Read block 230 len 40960 19/30 Write block 99 len 24576 data: 10 11 9 10 10 10 20/30 Read block 296 len 24576 21/30 Write block 320 len 40960 data: 12 12 12 10 8 6 7 6 7 11 22/30 Read block 244 len 4096 23/30 Read block 377 len 16384 24/30 Read block 105 len 12288 25/30 Read block 110 len 36864 26/30 Flush 27/30 Write block 487 len 16384 data: 11 12 13 11 28/30 Write block 457 len 8192 data: 12 15 29/30 Write block 260 len 36864 data: 10 6 7 9 11 14 17 17 16 30/30 Read block 383 len 12288 ---------------------------------------------------------------- Crucible gen:130 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 4 Done Done Done false 1027 Acked Write 2 Done Done Done false 1028 Acked Write 9 Done Done Sent false 1029 Acked Read 3 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 2 2 Done 4 4 2 10 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:37:57.5312369Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1702} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 128 128 128 233 233 233 F F F 1 020-039 128 128 128 234 234 234 F F F 2 040-059 130 130 130 237 237 237 F F F 3 060-079 125 125 125 229 229 229 F F F 4 080-099 130 130 130 237 237 237 F F F 5 100-119 130 130 130 237 237 237 F F F 6 120-139 130 130 130 237 237 237 F F F 7 140-159 124 124 124 228 228 228 F F F 8 160-179 122 122 122 227 227 227 F F F 9 180-199 122 122 122 227 227 227 F F F 10 200-219 126 126 126 230 230 230 F F F 11 220-239 127 127 127 231 231 231 F F F 12 240-259 130 130 130 237 237 237 F F F 13 260-279 130 130 130 237 237 237 T T T 14 280-299 128 128 128 235 235 235 F F F 15 300-319 130 130 130 237 237 237 F F F 16 320-339 130 130 130 237 237 237 F F F 17 340-359 129 129 129 236 236 236 F F F 18 360-379 129 129 129 236 236 236 F F F 19 380-399 127 127 127 231 231 231 F F F 20 400-419 130 130 130 237 237 237 F F F 21 420-439 129 129 129 236 236 236 F F F 22 440-459 130 130 130 237 237 237 T T T 23 460-479 130 130 130 237 237 237 F F F 24 480-499 130 130 130 237 237 237 T T T 25 500-519 115 115 115 216 216 216 F F F 26 520-539 128 128 128 233 233 233 F F F 27 540-559 124 124 124 228 228 228 F F F 28 560-579 130 130 130 237 237 237 F F F 29 580-599 128 128 128 233 233 233 F F F Max gen: 130, Max flush: 237 A Difference in extent metadata is expected here On loop 65 Verifying data now No change after verify Loop: 65 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 128 128 128 233 233 233 F F F 1 020-039 128 128 128 234 234 234 F F F 2 040-059 130 130 130 237 237 237 F F F 3 060-079 125 125 125 229 229 229 F F F 4 080-099 130 130 130 237 237 237 F F F 5 100-119 130 130 130 237 237 237 F F F 6 120-139 130 130 130 237 237 237 F F F 7 140-159 124 124 124 228 228 228 F F F 8 160-179 122 122 122 227 227 227 F F F 9 180-199 122 122 122 227 227 227 F F F 10 200-219 126 126 126 230 230 230 F F F 11 220-239 127 127 127 231 231 231 F F F 12 240-259 130 130 130 237 237 237 F F F 13 260-279 131 131 131 238 238 238 F F F 14 280-299 128 128 128 235 235 235 F F F 15 300-319 130 130 130 237 237 237 F F F 16 320-339 130 130 130 237 237 237 F F F 17 340-359 129 129 129 236 236 236 F F F 18 360-379 129 129 129 236 236 236 F F F 19 380-399 127 127 127 231 231 231 F F F 20 400-419 130 130 130 237 237 237 F F F 21 420-439 129 129 129 236 236 236 F F F 22 440-459 131 131 131 238 238 238 F F F 23 460-479 130 130 130 237 237 237 F F F 24 480-499 131 131 131 238 238 238 F F F 25 500-519 115 115 115 216 216 216 F F F 26 520-539 128 128 128 233 233 233 F F F 27 540-559 124 124 124 228 228 228 F F F 28 560-579 130 130 130 237 237 237 F F F 29 580-599 128 128 128 233 233 233 F F F Max gen: 131, Max flush: 238 Begin loop 66 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:01.792340851Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1710,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:01.836663155Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1710,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:01.837301191Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1710} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"26e7d501b33f47379e53601720353c2983e7654b\",\n git_commit_timestamp: \"2024-12-12T20:16:40.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:01.837337938Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1710} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:01.837356857Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1710} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:01.837372046Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1710} {"msg":"Crucible 7b9f97e8-fd26-41e5-b163-e36d78af1862 has session id: 6f39d8f9-bd46-4d3d-b5bc-3ca3271020a4","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:01.837849153Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1710,"session_id":"6f39d8f9-bd46-4d3d-b5bc-3ca3271020a4"} {"msg":"Upstairs opts: Upstairs UUID: 7b9f97e8-fd26-41e5-b163-e36d78af1862, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:01.837897699Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1710,"session_id":"6f39d8f9-bd46-4d3d-b5bc-3ca3271020a4"} {"msg":"Crucible stats registered with UUID: 7b9f97e8-fd26-41e5-b163-e36d78af1862","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:01.837925708Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1710,"session_id":"6f39d8f9-bd46-4d3d-b5bc-3ca3271020a4"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:01.868677987Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1710,"task":"crutest"} {"msg":"The guest has requested activation with gen:132","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:01.868709605Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1710} {"msg":"7b9f97e8-fd26-41e5-b163-e36d78af1862 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:01.868788919Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1710,"session_id":"6f39d8f9-bd46-4d3d-b5bc-3ca3271020a4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:01.868823927Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1710,"client":"0","":"downstairs","session_id":"6f39d8f9-bd46-4d3d-b5bc-3ca3271020a4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:01.868853445Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1710,"client":"0","":"downstairs","session_id":"6f39d8f9-bd46-4d3d-b5bc-3ca3271020a4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:01.868879403Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1710,"client":"1","":"downstairs","session_id":"6f39d8f9-bd46-4d3d-b5bc-3ca3271020a4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:01.868912891Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1710,"client":"1","":"downstairs","session_id":"6f39d8f9-bd46-4d3d-b5bc-3ca3271020a4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:01.868946749Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1710,"client":"2","":"downstairs","session_id":"6f39d8f9-bd46-4d3d-b5bc-3ca3271020a4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:01.868983536Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1710,"client":"2","":"downstairs","session_id":"6f39d8f9-bd46-4d3d-b5bc-3ca3271020a4"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:01.869021403Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1710,"":"io task","client":"2","":"downstairs","session_id":"6f39d8f9-bd46-4d3d-b5bc-3ca3271020a4"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:01.869054511Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1710,"":"io task","client":"0","":"downstairs","session_id":"6f39d8f9-bd46-4d3d-b5bc-3ca3271020a4"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:01.869085929Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1710,"":"io task","client":"1","":"downstairs","session_id":"6f39d8f9-bd46-4d3d-b5bc-3ca3271020a4"} {"msg":"ds_connection connected from Ok(127.0.0.1:35338)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:01.869118227Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1710,"":"io task","client":"1","":"downstairs","session_id":"6f39d8f9-bd46-4d3d-b5bc-3ca3271020a4"} {"msg":"ds_connection connected from Ok(127.0.0.1:56575)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:01.869315853Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1710,"":"io task","client":"2","":"downstairs","session_id":"6f39d8f9-bd46-4d3d-b5bc-3ca3271020a4"} {"msg":"ds_connection connected from Ok(127.0.0.1:61026)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:01.86935713Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1710,"":"io task","client":"0","":"downstairs","session_id":"6f39d8f9-bd46-4d3d-b5bc-3ca3271020a4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:01.869863145Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1710,"client":"0","":"downstairs","session_id":"6f39d8f9-bd46-4d3d-b5bc-3ca3271020a4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:01.869892693Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1710,"client":"1","":"downstairs","session_id":"6f39d8f9-bd46-4d3d-b5bc-3ca3271020a4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:01.869907922Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1710,"client":"2","":"downstairs","session_id":"6f39d8f9-bd46-4d3d-b5bc-3ca3271020a4"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:01.870563147Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1710,"client":"1","":"downstairs","session_id":"6f39d8f9-bd46-4d3d-b5bc-3ca3271020a4"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:01.870592775Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1710,"client":"2","":"downstairs","session_id":"6f39d8f9-bd46-4d3d-b5bc-3ca3271020a4"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:01.870608404Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1710,"client":"0","":"downstairs","session_id":"6f39d8f9-bd46-4d3d-b5bc-3ca3271020a4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:01.87109968Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1710,"session_id":"6f39d8f9-bd46-4d3d-b5bc-3ca3271020a4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:01.871206383Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1710,"session_id":"6f39d8f9-bd46-4d3d-b5bc-3ca3271020a4"} {"msg":"[0]R flush_numbers[0..12]: [233, 234, 237, 229, 237, 237, 237, 228, 227, 227, 230, 231]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:01.87124791Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1710,"":"downstairs","":"downstairs","session_id":"6f39d8f9-bd46-4d3d-b5bc-3ca3271020a4"} {"msg":"[0]R generation[0..12]: [128, 128, 130, 125, 130, 130, 130, 124, 122, 122, 126, 127]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:01.871266438Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1710,"":"downstairs","":"downstairs","session_id":"6f39d8f9-bd46-4d3d-b5bc-3ca3271020a4"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:01.871281508Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1710,"":"downstairs","":"downstairs","session_id":"6f39d8f9-bd46-4d3d-b5bc-3ca3271020a4"} {"msg":"[1]R flush_numbers[0..12]: [233, 234, 237, 229, 237, 237, 237, 228, 227, 227, 230, 231]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:01.871296036Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1710,"":"downstairs","":"downstairs","session_id":"6f39d8f9-bd46-4d3d-b5bc-3ca3271020a4"} {"msg":"[1]R generation[0..12]: [128, 128, 130, 125, 130, 130, 130, 124, 122, 122, 126, 127]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:01.871310026Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1710,"":"downstairs","":"downstairs","session_id":"6f39d8f9-bd46-4d3d-b5bc-3ca3271020a4"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:01.871323785Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1710,"":"downstairs","":"downstairs","session_id":"6f39d8f9-bd46-4d3d-b5bc-3ca3271020a4"} {"msg":"[2]R flush_numbers[0..12]: [233, 234, 237, 229, 237, 237, 237, 228, 227, 227, 230, 231]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:01.871337584Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1710,"":"downstairs","":"downstairs","session_id":"6f39d8f9-bd46-4d3d-b5bc-3ca3271020a4"} {"msg":"[2]R generation[0..12]: [128, 128, 130, 125, 130, 130, 130, 124, 122, 122, 126, 127]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:01.871351573Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1710,"":"downstairs","":"downstairs","session_id":"6f39d8f9-bd46-4d3d-b5bc-3ca3271020a4"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:01.871365552Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1710,"":"downstairs","":"downstairs","session_id":"6f39d8f9-bd46-4d3d-b5bc-3ca3271020a4"} {"msg":"Max found gen is 132","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:01.871378621Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1710,"":"downstairs","":"downstairs","session_id":"6f39d8f9-bd46-4d3d-b5bc-3ca3271020a4"} {"msg":"Generation requested: 132 >= found:132","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:01.87139177Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1710,"":"downstairs","":"downstairs","session_id":"6f39d8f9-bd46-4d3d-b5bc-3ca3271020a4"} {"msg":"Next flush: 239","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:01.871404909Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1710,"":"downstairs","":"downstairs","session_id":"6f39d8f9-bd46-4d3d-b5bc-3ca3271020a4"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:01.871428377Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1710,"":"downstairs","":"downstairs","session_id":"6f39d8f9-bd46-4d3d-b5bc-3ca3271020a4"} Wait for a query_work_queue command to finish before sending IO {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:01.871458195Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1710,"session_id":"6f39d8f9-bd46-4d3d-b5bc-3ca3271020a4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:01.871482494Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1710,"client":"0","":"downstairs","session_id":"6f39d8f9-bd46-4d3d-b5bc-3ca3271020a4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:01.871497393Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1710,"client":"1","":"downstairs","session_id":"6f39d8f9-bd46-4d3d-b5bc-3ca3271020a4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:01.871510702Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1710,"client":"2","":"downstairs","session_id":"6f39d8f9-bd46-4d3d-b5bc-3ca3271020a4"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:01.871523851Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1710,"session_id":"6f39d8f9-bd46-4d3d-b5bc-3ca3271020a4"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:01.87153602Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1710,"session_id":"6f39d8f9-bd46-4d3d-b5bc-3ca3271020a4"} {"msg":"7b9f97e8-fd26-41e5-b163-e36d78af1862 is now active with session: 6f39d8f9-bd46-4d3d-b5bc-3ca3271020a4","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:01.871548839Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1710,"session_id":"6f39d8f9-bd46-4d3d-b5bc-3ca3271020a4"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:01.871560728Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1710,"session_id":"6f39d8f9-bd46-4d3d-b5bc-3ca3271020a4"} {"msg":"The guest has finished waiting for activation with:132","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:01.871572748Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1710} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 506 len 8192 data: 9 10 02/30 Flush 03/30 Write block 556 len 24576 data: 9 9 7 10 6 6 04/30 Read block 467 len 40960 05/30 Read block 121 len 24576 06/30 Write block 333 len 24576 data: 12 12 11 9 10 10 07/30 Read block 496 len 28672 08/30 Write block 450 len 40960 data: 16 17 16 12 12 11 12 13 16 16 09/30 Read block 564 len 32768 10/30 Flush 11/30 Write block 115 len 4096 data: 11 12/30 Read block 127 len 8192 13/30 Flush 14/30 Write block 474 len 28672 data: 8 9 9 8 9 9 7 15/30 Write block 67 len 4096 data: 8 16/30 Read block 452 len 32768 17/30 Read block 464 len 4096 18/30 Read block 33 len 8192 19/30 Read block 122 len 32768 20/30 Write block 14 len 24576 data: 8 9 6 6 7 6 21/30 Write block 31 len 32768 data: 9 9 10 11 13 12 11 12 22/30 Read block 297 len 20480 23/30 Read block 260 len 40960 24/30 Read block 271 len 4096 25/30 Read block 583 len 16384 26/30 Write block 4 len 16384 data: 6 7 7 8 27/30 Read block 153 len 8192 28/30 Write block 300 len 40960 data: 9 8 7 7 6 8 9 8 10 9 29/30 Write block 16 len 20480 data: 7 7 8 7 9 30/30 Read block 99 len 40960 ---------------------------------------------------------------- Crucible gen:132 GIO:true work queues: Upstairs:0 downstairs:17 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1013 Acked Write 7 Done Done Done false 1014 Acked Write 1 Done Done Done false 1015 Acked Read 8 Done Done Done false 1016 Acked Read 1 Done Done Done false 1017 Acked Read 2 Done Done Done false 1018 Acked Read 8 Done Done Done false 1019 Acked Write 6 Done Done Done false 1020 Acked Write 8 Done Done Done false 1021 Acked Read 5 Done Done Done false 1022 Acked Read 10 Done Done Done false 1023 Acked Read 1 Done Done Done false 1024 Acked Read 4 Done Done Done false 1025 Acked Write 4 Done Done Done false 1026 Acked Read 2 Done Done Done false 1027 Acked Write 10 Done Done Done false 1028 Acked Write 5 Done Done Sent false 1029 Acked Read 10 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 2 3 Done 16 17 15 48 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1012 1012 1012 Downstairs last five completed: 1012 1011 1010 1009 1008 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:38:01.890872866Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1710} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 128 128 128 233 233 233 T T T 1 020-039 128 128 128 234 234 234 T T T 2 040-059 130 130 130 237 237 237 F F F 3 060-079 125 125 125 229 229 229 T T T 4 080-099 130 130 130 237 237 237 F F F 5 100-119 132 132 132 241 241 241 F F F 6 120-139 130 130 130 237 237 237 F F F 7 140-159 124 124 124 228 228 228 F F F 8 160-179 122 122 122 227 227 227 F F F 9 180-199 122 122 122 227 227 227 F F F 10 200-219 126 126 126 230 230 230 F F F 11 220-239 127 127 127 231 231 231 F F F 12 240-259 130 130 130 237 237 237 F F F 13 260-279 131 131 131 238 238 238 F F F 14 280-299 128 128 128 235 235 235 F F F 15 300-319 130 130 130 237 237 237 T T T 16 320-339 132 132 132 240 240 240 F F F 17 340-359 129 129 129 236 236 236 F F F 18 360-379 129 129 129 236 236 236 F F F 19 380-399 127 127 127 231 231 231 F F F 20 400-419 130 130 130 237 237 237 F F F 21 420-439 129 129 129 236 236 236 F F F 22 440-459 132 132 132 240 240 240 F F F 23 460-479 130 130 130 237 237 237 T T T 24 480-499 131 131 131 238 238 238 T T T 25 500-519 132 132 132 239 239 239 F F F 26 520-539 128 128 128 233 233 233 F F F 27 540-559 132 132 132 240 240 240 F F F 28 560-579 132 132 132 240 240 240 F F F 29 580-599 128 128 128 233 233 233 F F F Max gen: 132, Max flush: 241 A Difference in extent metadata is expected here On loop 66 Verifying data now No change after verify Loop: 66 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 133 133 133 242 242 242 F F F 1 020-039 133 133 133 242 242 242 F F F 2 040-059 130 130 130 237 237 237 F F F 3 060-079 133 133 133 242 242 242 F F F 4 080-099 130 130 130 237 237 237 F F F 5 100-119 132 132 132 241 241 241 F F F 6 120-139 130 130 130 237 237 237 F F F 7 140-159 124 124 124 228 228 228 F F F 8 160-179 122 122 122 227 227 227 F F F 9 180-199 122 122 122 227 227 227 F F F 10 200-219 126 126 126 230 230 230 F F F 11 220-239 127 127 127 231 231 231 F F F 12 240-259 130 130 130 237 237 237 F F F 13 260-279 131 131 131 238 238 238 F F F 14 280-299 128 128 128 235 235 235 F F F 15 300-319 133 133 133 242 242 242 F F F 16 320-339 132 132 132 240 240 240 F F F 17 340-359 129 129 129 236 236 236 F F F 18 360-379 129 129 129 236 236 236 F F F 19 380-399 127 127 127 231 231 231 F F F 20 400-419 130 130 130 237 237 237 F F F 21 420-439 129 129 129 236 236 236 F F F 22 440-459 132 132 132 240 240 240 F F F 23 460-479 133 133 133 242 242 242 F F F 24 480-499 133 133 133 242 242 242 F F F 25 500-519 132 132 132 239 239 239 F F F 26 520-539 128 128 128 233 233 233 F F F 27 540-559 132 132 132 240 240 240 F F F 28 560-579 132 132 132 240 240 240 F F F 29 580-599 128 128 128 233 233 233 F F F Max gen: 133, Max flush: 242 Begin loop 67 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:06.16600347Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1718,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:38:06.209481312Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1718,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:06.209896014Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1718,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:06.241728379Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1718,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:06.242297749Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1718} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"26e7d501b33f47379e53601720353c2983e7654b\",\n git_commit_timestamp: \"2024-12-12T20:16:40.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:06.242331567Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1718} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:06.242345206Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1718} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:06.242359575Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1718} {"msg":"Crucible 808c9300-a5f0-45ad-87ca-ac1f21605ffb has session id: 4307eb1b-1fc6-4ff2-ad03-f3a875e71cab","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:06.242786386Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1718,"session_id":"4307eb1b-1fc6-4ff2-ad03-f3a875e71cab"} {"msg":"Upstairs opts: Upstairs UUID: 808c9300-a5f0-45ad-87ca-ac1f21605ffb, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:06.242835022Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1718,"session_id":"4307eb1b-1fc6-4ff2-ad03-f3a875e71cab"} {"msg":"Crucible stats registered with UUID: 808c9300-a5f0-45ad-87ca-ac1f21605ffb","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:06.242857681Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1718,"session_id":"4307eb1b-1fc6-4ff2-ad03-f3a875e71cab"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:06.27360484Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1718,"task":"crutest"} {"msg":"The guest has requested activation with gen:134","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:06.273635028Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1718} {"msg":"808c9300-a5f0-45ad-87ca-ac1f21605ffb active request set","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:06.273708893Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1718,"session_id":"4307eb1b-1fc6-4ff2-ad03-f3a875e71cab"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:06.273731002Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1718,"client":"0","":"downstairs","session_id":"4307eb1b-1fc6-4ff2-ad03-f3a875e71cab"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:06.27376031Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1718,"client":"0","":"downstairs","session_id":"4307eb1b-1fc6-4ff2-ad03-f3a875e71cab"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:06.273778098Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1718,"client":"1","":"downstairs","session_id":"4307eb1b-1fc6-4ff2-ad03-f3a875e71cab"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:06.273795877Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1718,"client":"1","":"downstairs","session_id":"4307eb1b-1fc6-4ff2-ad03-f3a875e71cab"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:06.273812356Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1718,"client":"2","":"downstairs","session_id":"4307eb1b-1fc6-4ff2-ad03-f3a875e71cab"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:06.273829875Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1718,"client":"2","":"downstairs","session_id":"4307eb1b-1fc6-4ff2-ad03-f3a875e71cab"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:06.273846254Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1718,"":"io task","client":"0","":"downstairs","session_id":"4307eb1b-1fc6-4ff2-ad03-f3a875e71cab"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:06.273863263Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1718,"":"io task","client":"2","":"downstairs","session_id":"4307eb1b-1fc6-4ff2-ad03-f3a875e71cab"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:06.273880141Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1718,"":"io task","client":"1","":"downstairs","session_id":"4307eb1b-1fc6-4ff2-ad03-f3a875e71cab"} {"msg":"ds_connection connected from Ok(127.0.0.1:37350)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:06.273915079Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1718,"":"io task","client":"1","":"downstairs","session_id":"4307eb1b-1fc6-4ff2-ad03-f3a875e71cab"} {"msg":"ds_connection connected from Ok(127.0.0.1:45260)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:06.274054459Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1718,"":"io task","client":"2","":"downstairs","session_id":"4307eb1b-1fc6-4ff2-ad03-f3a875e71cab"} {"msg":"ds_connection connected from Ok(127.0.0.1:54255)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:06.274138473Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1718,"":"io task","client":"0","":"downstairs","session_id":"4307eb1b-1fc6-4ff2-ad03-f3a875e71cab"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:06.274599092Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1718,"client":"2","":"downstairs","session_id":"4307eb1b-1fc6-4ff2-ad03-f3a875e71cab"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:06.274632039Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1718,"client":"1","":"downstairs","session_id":"4307eb1b-1fc6-4ff2-ad03-f3a875e71cab"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:06.274834825Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1718,"client":"0","":"downstairs","session_id":"4307eb1b-1fc6-4ff2-ad03-f3a875e71cab"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:06.275269366Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1718,"client":"1","":"downstairs","session_id":"4307eb1b-1fc6-4ff2-ad03-f3a875e71cab"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:06.275311073Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1718,"client":"2","":"downstairs","session_id":"4307eb1b-1fc6-4ff2-ad03-f3a875e71cab"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:06.275378838Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1718,"client":"0","":"downstairs","session_id":"4307eb1b-1fc6-4ff2-ad03-f3a875e71cab"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:06.275805379Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1718,"session_id":"4307eb1b-1fc6-4ff2-ad03-f3a875e71cab"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:06.275910521Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1718,"session_id":"4307eb1b-1fc6-4ff2-ad03-f3a875e71cab"} {"msg":"[0]R flush_numbers[0..12]: [242, 242, 237, 242, 237, 241, 237, 228, 227, 227, 230, 231]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:06.275982786Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1718,"":"downstairs","":"downstairs","session_id":"4307eb1b-1fc6-4ff2-ad03-f3a875e71cab"} {"msg":"[0]R generation[0..12]: [133, 133, 130, 133, 130, 132, 130, 124, 122, 122, 126, 127]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:06.276003985Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1718,"":"downstairs","":"downstairs","session_id":"4307eb1b-1fc6-4ff2-ad03-f3a875e71cab"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:06.276020164Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1718,"":"downstairs","":"downstairs","session_id":"4307eb1b-1fc6-4ff2-ad03-f3a875e71cab"} {"msg":"[1]R flush_numbers[0..12]: [242, 242, 237, 242, 237, 241, 237, 228, 227, 227, 230, 231]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:06.276034483Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1718,"":"downstairs","":"downstairs","session_id":"4307eb1b-1fc6-4ff2-ad03-f3a875e71cab"} {"msg":"[1]R generation[0..12]: [133, 133, 130, 133, 130, 132, 130, 124, 122, 122, 126, 127]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:06.276048372Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1718,"":"downstairs","":"downstairs","session_id":"4307eb1b-1fc6-4ff2-ad03-f3a875e71cab"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:06.276062561Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1718,"":"downstairs","":"downstairs","session_id":"4307eb1b-1fc6-4ff2-ad03-f3a875e71cab"} {"msg":"[2]R flush_numbers[0..12]: [242, 242, 237, 242, 237, 241, 237, 228, 227, 227, 230, 231]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:06.27607646Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1718,"":"downstairs","":"downstairs","session_id":"4307eb1b-1fc6-4ff2-ad03-f3a875e71cab"} {"msg":"[2]R generation[0..12]: [133, 133, 130, 133, 130, 132, 130, 124, 122, 122, 126, 127]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:06.276093559Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1718,"":"downstairs","":"downstairs","session_id":"4307eb1b-1fc6-4ff2-ad03-f3a875e71cab"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:06.276107918Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1718,"":"downstairs","":"downstairs","session_id":"4307eb1b-1fc6-4ff2-ad03-f3a875e71cab"} {"msg":"Max found gen is 134","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:06.276121167Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1718,"":"downstairs","":"downstairs","session_id":"4307eb1b-1fc6-4ff2-ad03-f3a875e71cab"} {"msg":"Generation requested: 134 >= found:134","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-12-12T20:38:06.276134596Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1718,"":"downstairs","":"downstairs","session_id":"4307eb1b-1fc6-4ff2-ad03-f3a875e71cab"} {"msg":"Next flush: 243","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:06.276152405Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1718,"":"downstairs","":"downstairs","session_id":"4307eb1b-1fc6-4ff2-ad03-f3a875e71cab"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:06.276165574Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1718,"":"downstairs","":"downstairs","session_id":"4307eb1b-1fc6-4ff2-ad03-f3a875e71cab"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:06.276178753Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1718,"session_id":"4307eb1b-1fc6-4ff2-ad03-f3a875e71cab"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:06.276190832Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1718,"client":"0","":"downstairs","session_id":"4307eb1b-1fc6-4ff2-ad03-f3a875e71cab"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:06.276204211Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1718,"client":"1","":"downstairs","session_id":"4307eb1b-1fc6-4ff2-ad03-f3a875e71cab"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:06.27621733Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1718,"client":"2","":"downstairs","session_id":"4307eb1b-1fc6-4ff2-ad03-f3a875e71cab"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:06.276230559Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1718,"session_id":"4307eb1b-1fc6-4ff2-ad03-f3a875e71cab"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:06.276242779Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1718,"session_id":"4307eb1b-1fc6-4ff2-ad03-f3a875e71cab"} {"msg":"808c9300-a5f0-45ad-87ca-ac1f21605ffb is now active with session: 4307eb1b-1fc6-4ff2-ad03-f3a875e71cab","v":0,"name":"crucible","level":30,"time":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 2024-12-12T20:38:06.276255638Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1718,"session_id":"4307eb1b-1fc6-4ff2-ad03-f3a875e71cab"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:06.276271566Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1718,"session_id":"4307eb1b-1fc6-4ff2-ad03-f3a875e71cab"} {"msg":"The guest has finished waiting for activation with:134","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:06.276283876Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1718} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 56 len 20480 data: 19 20 18 17 16 02/30 Write block 332 len 4096 data: 12 03/30 Write block 577 len 4096 data: 9 04/30 Read block 36 len 36864 05/30 Write block 510 len 20480 data: 9 9 9 8 7 06/30 Read block 307 len 8192 07/30 Write block 173 len 20480 data: 5 6 9 9 10 08/30 Read block 94 len 12288 09/30 Read block 428 len 8192 10/30 Write block 24 len 24576 data: 13 11 10 11 10 9 11/30 Read block 208 len 24576 12/30 Read block 253 len 28672 13/30 Read block 371 len 16384 14/30 Write block 77 len 28672 data: 11 10 9 10 9 6 7 15/30 Write block 143 len 40960 data: 9 9 11 12 12 13 12 10 8 9 16/30 Read block 465 len 32768 17/30 Write block 22 len 32768 data: 10 11 14 12 11 12 11 10 18/30 Write block 423 len 8192 data: 11 12 19/30 Write block 503 len 40960 data: 10 10 9 10 11 9 9 10 10 10 20/30 Write block 29 len 32768 data: 11 9 10 10 11 12 14 13 21/30 Write block 407 len 16384 data: 13 13 11 9 22/30 Flush 23/30 Write block 161 len 8192 data: 7 10 24/30 Flush 25/30 Write block 55 len 4096 data: 16 26/30 Write block 77 len 12288 data: 12 11 10 27/30 Read block 522 len 8192 28/30 Read block 489 len 24576 29/30 Write block 168 len 24576 data: 6 6 6 6 5 6 30/30 Read block 225 len 12288 ---------------------------------------------------------------- Crucible gen:134 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 1 Done Done Done false 1025 Acked Write 3 Done Done Done false 1026 Acked Read 2 Done Done Done false 1027 Acked Read 6 Done Done Done false 1028 Acked Write 6 Done Done Sent false 1029 Acked Read 3 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 2 3 Done 6 5 4 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:38:06.294469762Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1718} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 133 133 133 242 242 242 F F F 1 020-039 134 134 134 243 243 243 F F F 2 040-059 134 134 134 243 243 243 T T T 3 060-079 134 134 134 243 243 243 T T T 4 080-099 134 134 134 243 243 243 F F F 5 100-119 132 132 132 241 241 241 F F F 6 120-139 130 130 130 237 237 237 F F F 7 140-159 134 134 134 243 243 243 F F F 8 160-179 134 134 134 244 244 244 T T T 9 180-199 122 122 122 227 227 227 F F F 10 200-219 126 126 126 230 230 230 F F F 11 220-239 127 127 127 231 231 231 F F F 12 240-259 130 130 130 237 237 237 F F F 13 260-279 131 131 131 238 238 238 F F F 14 280-299 128 128 128 235 235 235 F F F 15 300-319 133 133 133 242 242 242 F F F 16 320-339 134 134 134 243 243 243 F F F 17 340-359 129 129 129 236 236 236 F F F 18 360-379 129 129 129 236 236 236 F F F 19 380-399 127 127 127 231 231 231 F F F 20 400-419 134 134 134 243 243 243 F F F 21 420-439 134 134 134 243 243 243 F F F 22 440-459 132 132 132 240 240 240 F F F 23 460-479 133 133 133 242 242 242 F F F 24 480-499 133 133 133 242 242 242 F F F 25 500-519 134 134 134 243 243 243 F F F 26 520-539 128 128 128 233 233 233 F F F 27 540-559 132 132 132 240 240 240 F F F 28 560-579 134 134 134 243 243 243 F F F 29 580-599 128 128 128 233 233 233 F F F Max gen: 134, Max flush: 244 A Difference in extent metadata is expected here On loop 67 Verifying data now No change after verify Loop: 67 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 133 133 133 242 242 242 F F F 1 020-039 134 134 134 243 243 243 F F F 2 040-059 135 135 135 245 245 245 F F F 3 060-079 135 135 135 245 245 245 F F F 4 080-099 134 134 134 243 243 243 F F F 5 100-119 132 132 132 241 241 241 F F F 6 120-139 130 130 130 237 237 237 F F F 7 140-159 134 134 134 243 243 243 F F F 8 160-179 135 135 135 245 245 245 F F F 9 180-199 122 122 122 227 227 227 F F F 10 200-219 126 126 126 230 230 230 F F F 11 220-239 127 127 127 231 231 231 F F F 12 240-259 130 130 130 237 237 237 F F F 13 260-279 131 131 131 238 238 238 F F F 14 280-299 128 128 128 235 235 235 F F F 15 300-319 133 133 133 242 242 242 F F F 16 320-339 134 134 134 243 243 243 F F F 17 340-359 129 129 129 236 236 236 F F F 18 360-379 129 129 129 236 236 236 F F F 19 380-399 127 127 127 231 231 231 F F F 20 400-419 134 134 134 243 243 243 F F F 21 420-439 134 134 134 243 243 243 F F F 22 440-459 132 132 132 240 240 240 F F F 23 460-479 133 133 133 242 242 242 F F F 24 480-499 133 133 133 242 242 242 F F F 25 500-519 134 134 134 243 243 243 F F F 26 520-539 128 128 128 233 233 233 F F F 27 540-559 132 132 132 240 240 240 F F F 28 560-579 134 134 134 243 243 243 F F F 29 580-599 128 128 128 233 233 233 F F F Max gen: 135, Max flush: 245 Begin loop 68 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:10.582537364Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1726,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:10.626894195Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1726,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:10.627580618Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1726} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"26e7d501b33f47379e53601720353c2983e7654b\",\n git_commit_timestamp: \"2024-12-12T20:16:40.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:10.627618085Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1726} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:10.627633234Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1726} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:10.627647803Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1726} {"msg":"Crucible 91c010bb-36f0-4f8d-a1c2-1db2d1ce39ff has session id: b0560132-7e0b-43af-baf8-cb2cc2567c44","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:10.628166617Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1726,"session_id":"b0560132-7e0b-43af-baf8-cb2cc2567c44"} {"msg":"Upstairs opts: Upstairs UUID: 91c010bb-36f0-4f8d-a1c2-1db2d1ce39ff, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:10.628207465Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1726,"session_id":"b0560132-7e0b-43af-baf8-cb2cc2567c44"} {"msg":"Crucible stats registered with UUID: 91c010bb-36f0-4f8d-a1c2-1db2d1ce39ff","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:10.628223913Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1726,"session_id":"b0560132-7e0b-43af-baf8-cb2cc2567c44"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:10.659221856Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1726,"task":"crutest"} {"msg":"The guest has requested activation with gen:136","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:10.659253894Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1726} {"msg":"91c010bb-36f0-4f8d-a1c2-1db2d1ce39ff active request set","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:10.659286151Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1726,"session_id":"b0560132-7e0b-43af-baf8-cb2cc2567c44"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:10.65930257Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1726,"client":"0","":"downstairs","session_id":"b0560132-7e0b-43af-baf8-cb2cc2567c44"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:10.659331398Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1726,"client":"0","":"downstairs","session_id":"b0560132-7e0b-43af-baf8-cb2cc2567c44"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:10.659349157Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1726,"client":"1","":"downstairs","session_id":"b0560132-7e0b-43af-baf8-cb2cc2567c44"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:10.659367266Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1726,"client":"1","":"downstairs","session_id":"b0560132-7e0b-43af-baf8-cb2cc2567c44"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:10.659384015Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1726,"client":"2","":"downstairs","session_id":"b0560132-7e0b-43af-baf8-cb2cc2567c44"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:10.659401963Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1726,"client":"2","":"downstairs","session_id":"b0560132-7e0b-43af-baf8-cb2cc2567c44"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:10.659423442Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1726,"":"io task","client":"2","":"downstairs","session_id":"b0560132-7e0b-43af-baf8-cb2cc2567c44"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:10.65945554Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1726,"":"io task","client":"0","":"downstairs","session_id":"b0560132-7e0b-43af-baf8-cb2cc2567c44"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:10.659490137Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1726,"":"io task","client":"1","":"downstairs","session_id":"b0560132-7e0b-43af-baf8-cb2cc2567c44"} {"msg":"ds_connection connected from Ok(127.0.0.1:62876)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:10.65958977Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1726,"":"io task","client":"2","":"downstairs","session_id":"b0560132-7e0b-43af-baf8-cb2cc2567c44"} {"msg":"ds_connection connected from Ok(127.0.0.1:43006)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:10.65974794Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1726,"":"io task","client":"1","":"downstairs","session_id":"b0560132-7e0b-43af-baf8-cb2cc2567c44"} {"msg":"ds_connection connected from Ok(127.0.0.1:55506)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:10.659791616Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1726,"":"io task","client":"0","":"downstairs","session_id":"b0560132-7e0b-43af-baf8-cb2cc2567c44"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:10.660383686Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1726,"client":"0","":"downstairs","session_id":"b0560132-7e0b-43af-baf8-cb2cc2567c44"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:10.660413234Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1726,"client":"1","":"downstairs","session_id":"b0560132-7e0b-43af-baf8-cb2cc2567c44"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:10.660441002Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1726,"client":"2","":"downstairs","session_id":"b0560132-7e0b-43af-baf8-cb2cc2567c44"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:10.661147163Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1726,"client":"0","":"downstairs","session_id":"b0560132-7e0b-43af-baf8-cb2cc2567c44"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:10.661177541Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1726,"client":"1","":"downstairs","session_id":"b0560132-7e0b-43af-baf8-cb2cc2567c44"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:10.6611939Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1726,"client":"2","":"downstairs","session_id":"b0560132-7e0b-43af-baf8-cb2cc2567c44"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:10.661679217Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1726,"session_id":"b0560132-7e0b-43af-baf8-cb2cc2567c44"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:10.66177355Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1726,"session_id":"b0560132-7e0b-43af-baf8-cb2cc2567c44"} {"msg":"[0]R flush_numbers[0..12]: [242, 243, 245, 245, 243, 241, 237, 243, 245, 227, 230, 231]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:10.661874353Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1726,"":"downstairs","":"downstairs","session_id":"b0560132-7e0b-43af-baf8-cb2cc2567c44"} {"msg":"[0]R generation[0..12]: [133, 134, 135, 135, 134, 132, 130, 134, 135, 122, 126, 127]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:10.661895302Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1726,"":"downstairs","":"downstairs","session_id":"b0560132-7e0b-43af-baf8-cb2cc2567c44"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:10.661910241Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1726,"":"downstairs","":"downstairs","session_id":"b0560132-7e0b-43af-baf8-cb2cc2567c44"} {"msg":"[1]R flush_numbers[0..12]: [242, 243, 245, 245, 243, 241, 237, 243, 245, 227, 230, 231]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:10.661927929Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1726,"":"downstairs","":"downstairs","session_id":"b0560132-7e0b-43af-baf8-cb2cc2567c44"} {"msg":"[1]R generation[0..12]: [133, 134, 135, 135, 134, 132, 130, 134, 135, 122, 126, 127]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:10.661942268Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1726,"":"downstairs","":"downstairs","session_id":"b0560132-7e0b-43af-baf8-cb2cc2567c44"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:10.661956317Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1726,"":"downstairs","":"downstairs","session_id":"b0560132-7e0b-43af-baf8-cb2cc2567c44"} {"msg":"[2]R flush_numbers[0..12]: [242, 243, 245, 245, 243, 241, 237, 243, 245, 227, 230, 231]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:10.661970096Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1726,"":"downstairs","":"downstairs","session_id":"b0560132-7e0b-43af-baf8-cb2cc2567c44"} {"msg":"[2]R generation[0..12]: [133, 134, 135, 135, 134, 132, 130, 134, 135, 122, 126, 127]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:10.661983636Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1726,"":"downstairs","":"downstairs","session_id":"b0560132-7e0b-43af-baf8-cb2cc2567c44"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:10.661997575Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1726,"":"downstairs","":"downstairs","session_id":"b0560132-7e0b-43af-baf8-cb2cc2567c44"} {"msg":"Max found gen is 136","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:10.662010674Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1726,"":"downstairs","":"downstairs","session_id":"b0560132-7e0b-43af-baf8-cb2cc2567c44"} {"msg":"Generation requested: 136 >= found:136","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:10.662023943Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1726,"":"downstairs","":"downstairs","session_id":"b0560132-7e0b-43af-baf8-cb2cc2567c44"} {"msg":"Next flush: 246","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:10.662036852Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1726,"":"downstairs","":"downstairs","session_id":"b0560132-7e0b-43af-baf8-cb2cc2567c44"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:10.662049751Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1726,"":"downstairs","":"downstairs","session_id":"b0560132-7e0b-43af-baf8-cb2cc2567c44"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:10.66206297Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1726,"session_id":"b0560132-7e0b-43af-baf8-cb2cc2567c44"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-12-12T20:38:10.662075159Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1726,"client":"0","":"downstairs","session_id":"b0560132-7e0b-43af-baf8-cb2cc2567c44"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:10.662090798Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1726,"client":"1","":"downstairs","session_id":"b0560132-7e0b-43af-baf8-cb2cc2567c44"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:10.662104047Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1726,"client":"2","":"downstairs","session_id":"b0560132-7e0b-43af-baf8-cb2cc2567c44"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:10.662118956Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1726,"session_id":"b0560132-7e0b-43af-baf8-cb2cc2567c44"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:10.662131435Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1726,"session_id":"b0560132-7e0b-43af-baf8-cb2cc2567c44"} {"msg":"91c010bb-36f0-4f8d-a1c2-1db2d1ce39ff is now active with session: b0560132-7e0b-43af-baf8-cb2cc2567c44","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:10.662144164Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1726,"session_id":"b0560132-7e0b-43af-baf8-cb2cc2567c44"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:10.662156104Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1726,"session_id":"b0560132-7e0b-43af-baf8-cb2cc2567c44"} {"msg":"The guest has finished waiting for activation with:136","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:10.662168313Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1726} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 527 len 12288 data: 10 9 9 02/30 Write block 190 len 40960 data: 12 12 13 14 11 11 10 7 5 6 03/30 Read block 394 len 12288 04/30 Read block 228 len 16384 05/30 Read block 289 len 4096 06/30 Flush 07/30 Write block 398 len 28672 data: 15 16 13 15 12 11 10 08/30 Read block 85 len 12288 09/30 Write block 555 len 12288 data: 10 10 10 10/30 Write block 340 len 28672 data: 10 8 9 11 13 12 13 11/30 Write block 249 len 24576 data: 6 5 7 8 11 11 12/30 Write block 255 len 36864 data: 11 9 8 9 9 11 7 8 10 13/30 Read block 508 len 36864 14/30 Read block 468 len 12288 15/30 Read block 90 len 24576 16/30 Write block 256 len 32768 data: 10 9 10 10 12 8 9 11 17/30 Write block 310 len 8192 data: 8 12 18/30 Flush 19/30 Write block 267 len 16384 data: 18 17 10 10 20/30 Write block 150 len 16384 data: 11 9 10 10 21/30 Write block 281 len 28672 data: 10 9 8 8 8 8 7 22/30 Flush 23/30 Write block 28 len 32768 data: 12 12 10 11 11 12 13 15 24/30 Write block 232 len 28672 data: 11 11 11 11 13 11 10 25/30 Write block 482 len 4096 data: 7 26/30 Read block 197 len 32768 27/30 Write block 161 len 28672 data: 8 11 10 11 9 9 6 28/30 Read block 588 len 20480 29/30 Read block 394 len 4096 30/30 Write block 195 len 20480 data: 12 11 8 6 7 ---------------------------------------------------------------- Crucible gen:136 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 8 Done Done Done false 1023 Acked Write 7 Done Done Done false 1024 Acked Write 1 Done Done Done false 1025 Acked Read 8 Done Done Done false 1026 Acked Write 7 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Read 1 Done Done Done false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 7 7 7 21 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1021 1021 1021 Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:38:10.680964847Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1726} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 133 133 133 242 242 242 F F F 1 020-039 134 134 134 243 243 243 T T T 2 040-059 135 135 135 245 245 245 F F F 3 060-079 135 135 135 245 245 245 F F F 4 080-099 134 134 134 243 243 243 F F F 5 100-119 132 132 132 241 241 241 F F F 6 120-139 130 130 130 237 237 237 F F F 7 140-159 136 136 136 248 248 248 F F F 8 160-179 135 135 135 245 245 245 T T T 9 180-199 136 136 136 246 246 246 T T T 10 200-219 126 126 126 230 230 230 F F F 11 220-239 127 127 127 231 231 231 T T T 12 240-259 136 136 136 247 247 247 F F F 13 260-279 136 136 136 248 248 248 F F F 14 280-299 136 136 136 248 248 248 F F F 15 300-319 136 136 136 247 247 247 F F F 16 320-339 134 134 134 243 243 243 F F F 17 340-359 136 136 136 247 247 247 F F F 18 360-379 129 129 129 236 236 236 F F F 19 380-399 136 136 136 247 247 247 F F F 20 400-419 136 136 136 247 247 247 F F F 21 420-439 134 134 134 243 243 243 F F F 22 440-459 132 132 132 240 240 240 F F F 23 460-479 133 133 133 242 242 242 F F F 24 480-499 133 133 133 242 242 242 T T T 25 500-519 134 134 134 243 243 243 F F F 26 520-539 136 136 136 246 246 246 F F F 27 540-559 136 136 136 247 247 247 F F F 28 560-579 134 134 134 243 243 243 F F F 29 580-599 128 128 128 233 233 233 F F F Max gen: 136, Max flush: 248 A Difference in extent metadata is expected here On loop 68 Verifying data now No change after verify Loop: 68 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 133 133 133 242 242 242 F F F 1 020-039 137 137 137 249 249 249 F F F 2 040-059 135 135 135 245 245 245 F F F 3 060-079 135 135 135 245 245 245 F F F 4 080-099 134 134 134 243 243 243 F F F 5 100-119 132 132 132 241 241 241 F F F 6 120-139 130 130 130 237 237 237 F F F 7 140-159 136 136 136 248 248 248 F F F 8 160-179 137 137 137 249 249 249 F F F 9 180-199 137 137 137 249 249 249 F F F 10 200-219 126 126 126 230 230 230 F F F 11 220-239 137 137 137 249 249 249 F F F 12 240-259 136 136 136 247 247 247 F F F 13 260-279 136 136 136 248 248 248 F F F 14 280-299 136 136 136 248 248 248 F F F 15 300-319 136 136 136 247 247 247 F F F 16 320-339 134 134 134 243 243 243 F F F 17 340-359 136 136 136 247 247 247 F F F 18 360-379 129 129 129 236 236 236 F F F 19 380-399 136 136 136 247 247 247 F F F 20 400-419 136 136 136 247 247 247 F F F 21 420-439 134 134 134 243 243 243 F F F 22 440-459 132 132 132 240 240 240 F F F 23 460-479 133 133 133 242 242 242 F F F 24 480-499 137 137 137 249 249 249 F F F 25 500-519 134 134 134 243 243 243 F F F 26 520-539 136 136 136 246 246 246 F F F 27 540-559 136 136 136 247 247 247 F F F 28 560-579 134 134 134 243 243 243 F F F 29 580-599 128 128 128 233 233 233 F F F Max gen: 137, Max flush: 249 Begin loop 69 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:14.953541267Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1734,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:14.997796145Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1734,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:14.998503866Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1734} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"26e7d501b33f47379e53601720353c2983e7654b\",\n git_commit_timestamp: \"2024-12-12T20:16:40.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:14.998554163Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1734} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:14.998580171Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1734} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:14.998606189Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1734} {"msg":"Crucible 780a4056-d27c-44f0-b8f2-4bcf518c6f5f has session id: 368a0293-609d-4440-88e9-ecac06f7caaf","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:14.998968854Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1734,"session_id":"368a0293-609d-4440-88e9-ecac06f7caaf"} {"msg":"Upstairs opts: Upstairs UUID: 780a4056-d27c-44f0-b8f2-4bcf518c6f5f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:14.999023231Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1734,"session_id":"368a0293-609d-4440-88e9-ecac06f7caaf"} {"msg":"Crucible stats registered with UUID: 780a4056-d27c-44f0-b8f2-4bcf518c6f5f","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:14.999041409Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1734,"session_id":"368a0293-609d-4440-88e9-ecac06f7caaf"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:15.029701625Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1734,"task":"crutest"} {"msg":"The guest has requested activation with gen:138","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:15.029727963Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1734} {"msg":"780a4056-d27c-44f0-b8f2-4bcf518c6f5f active request set","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:15.029790769Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1734,"session_id":"368a0293-609d-4440-88e9-ecac06f7caaf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:15.029808188Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1734,"client":"0","":"downstairs","session_id":"368a0293-609d-4440-88e9-ecac06f7caaf"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:15.029831446Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1734,"client":"0","":"downstairs","session_id":"368a0293-609d-4440-88e9-ecac06f7caaf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:15.029846005Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1734,"client":"1","":"downstairs","session_id":"368a0293-609d-4440-88e9-ecac06f7caaf"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:15.029860294Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1734,"client":"1","":"downstairs","session_id":"368a0293-609d-4440-88e9-ecac06f7caaf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:15.029877123Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1734,"client":"2","":"downstairs","session_id":"368a0293-609d-4440-88e9-ecac06f7caaf"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:15.029891762Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1734,"client":"2","":"downstairs","session_id":"368a0293-609d-4440-88e9-ecac06f7caaf"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:15.029905061Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1734,"":"io task","client":"2","":"downstairs","session_id":"368a0293-609d-4440-88e9-ecac06f7caaf"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:15.02992321Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1734,"":"io task","client":"0","":"downstairs","session_id":"368a0293-609d-4440-88e9-ecac06f7caaf"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:15.029947878Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1734,"":"io task","client":"1","":"downstairs","session_id":"368a0293-609d-4440-88e9-ecac06f7caaf"} {"msg":"ds_connection connected from Ok(127.0.0.1:63246)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:15.030187131Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1734,"":"io task","client":"2","":"downstairs","session_id":"368a0293-609d-4440-88e9-ecac06f7caaf"} {"msg":"ds_connection connected from Ok(127.0.0.1:54612)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:15.030229928Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1734,"":"io task","client":"1","":"downstairs","session_id":"368a0293-609d-4440-88e9-ecac06f7caaf"} {"msg":"ds_connection connected from Ok(127.0.0.1:44266)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:15.030257307Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1734,"":"io task","client":"0","":"downstairs","session_id":"368a0293-609d-4440-88e9-ecac06f7caaf"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:15.030766811Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1734,"client":"0","":"downstairs","session_id":"368a0293-609d-4440-88e9-ecac06f7caaf"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:15.030795609Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1734,"client":"2","":"downstairs","session_id":"368a0293-609d-4440-88e9-ecac06f7caaf"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:15.030810788Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1734,"client":"1","":"downstairs","session_id":"368a0293-609d-4440-88e9-ecac06f7caaf"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:15.031402078Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1734,"client":"0","":"downstairs","session_id":"368a0293-609d-4440-88e9-ecac06f7caaf"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:15.031444825Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1734,"client":"1","":"downstairs","session_id":"368a0293-609d-4440-88e9-ecac06f7caaf"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:15.03151285Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1734,"client":"2","":"downstairs","session_id":"368a0293-609d-4440-88e9-ecac06f7caaf"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:15.032047923Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1734,"session_id":"368a0293-609d-4440-88e9-ecac06f7caaf"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:15.032069622Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1734,"session_id":"368a0293-609d-4440-88e9-ecac06f7caaf"} {"msg":"[0]R flush_numbers[0..12]: [242, 249, 245, 245, 243, 241, 237, 248, 249, 249, 230, 249]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:15.032141277Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1734,"":"downstairs","":"downstairs","session_id":"368a0293-609d-4440-88e9-ecac06f7caaf"} {"msg":"[0]R generation[0..12]: [133, 137, 135, 135, 134, 132, 130, 136, 137, 137, 126, 137]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:15.032160675Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1734,"":"downstairs","":"downstairs","session_id":"368a0293-609d-4440-88e9-ecac06f7caaf"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:15.032175454Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1734,"":"downstairs","":"downstairs","session_id":"368a0293-609d-4440-88e9-ecac06f7caaf"} {"msg":"[1]R flush_numbers[0..12]: [242, 249, 245, 245, 243, 241, 237, 248, 249, 249, 230, 249]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:15.032189713Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1734,"":"downstairs","":"downstairs","session_id":"368a0293-609d-4440-88e9-ecac06f7caaf"} {"msg":"[1]R generation[0..12]: [133, 137, 135, 135, 134, 132, 130, 136, 137, 137, 126, 137]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:15.032203583Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1734,"":"downstairs","":"downstairs","session_id":"368a0293-609d-4440-88e9-ecac06f7caaf"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:15.032217691Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1734,"":"downstairs","":"downstairs","session_id":"368a0293-609d-4440-88e9-ecac06f7caaf"} {"msg":"[2]R flush_numbers[0..12]: [242, 249, 245, 245, 243, 241, 237, 248, 249, 249, 230, 249]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:15.03223149Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1734,"":"downstairs","":"downstairs","session_id":"368a0293-609d-4440-88e9-ecac06f7caaf"} {"msg":"[2]R generation[0..12]: [133, 137, 135, 135, 134, 132, 130, 136, 137, 137, 126, 137]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:15.032245329Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1734,"":"downstairs","":"downstairs","session_id":"368a0293-609d-4440-88e9-ecac06f7caaf"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:15.032259619Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1734,"":"downstairs","":"downstairs","session_id":"368a0293-609d-4440-88e9-ecac06f7caaf"} {"msg":"Max found gen is 138","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:15.032272878Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1734,"":"downstairs","":"downstairs","session_id":"368a0293-609d-4440-88e9-ecac06f7caaf"} {"msg":"Generation requested: 138 >= found:138","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:15.032286287Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1734,"":"downstairs","":"downstairs","session_id":"368a0293-609d-4440-88e9-ecac06f7caaf"} {"msg":"Next flush: 250","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:15.032299336Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1734,"":"downstairs","":"downstairs","session_id":"368a0293-609d-4440-88e9-ecac06f7caaf"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:15.032316155Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1734,"":"downstairs","":"downstairs","session_id":"368a0293-609d-4440-88e9-ecac06f7caaf"} {"msg":Wait for a query_work_queue command to finish before sending IO "No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:15.032334973Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1734,"session_id":"368a0293-609d-4440-88e9-ecac06f7caaf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:15.032347373Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1734,"client":"0","":"downstairs","session_id":"368a0293-609d-4440-88e9-ecac06f7caaf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:15.032360702Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1734,"client":"1","":"downstairs","session_id":"368a0293-609d-4440-88e9-ecac06f7caaf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:15.032373751Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1734,"client":"2","":"downstairs","session_id":"368a0293-609d-4440-88e9-ecac06f7caaf"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:15.0323871Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1734,"session_id":"368a0293-609d-4440-88e9-ecac06f7caaf"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:15.032399439Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1734,"session_id":"368a0293-609d-4440-88e9-ecac06f7caaf"} {"msg":"780a4056-d27c-44f0-b8f2-4bcf518c6f5f is now active with session: 368a0293-609d-4440-88e9-ecac06f7caaf","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:15.032412178Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1734,"session_id":"368a0293-609d-4440-88e9-ecac06f7caaf"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:15.032437316Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1734,"session_id":"368a0293-609d-4440-88e9-ecac06f7caaf"} {"msg":"The guest has finished waiting for activation with:138","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:15.032450735Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1734} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 223 len 16384 data: 9 8 10 13 02/30 Read block 188 len 40960 03/30 Write block 399 len 28672 data: 17 14 16 13 12 11 11 04/30 Flush 05/30 Write block 342 len 16384 data: 10 12 14 13 06/30 Flush 07/30 Write block 356 len 24576 data: 14 13 10 10 10 11 08/30 Flush 09/30 Flush 10/30 Write block 512 len 32768 data: 11 9 8 7 8 10 15 18 11/30 Write block 494 len 32768 data: 7 6 6 7 4 4 6 8 12/30 Read block 264 len 12288 13/30 Flush 14/30 Write block 220 len 20480 data: 11 8 7 10 9 15/30 Write block 202 len 8192 data: 10 11 16/30 Read block 95 len 36864 17/30 Write block 250 len 24576 data: 6 8 9 12 12 12 18/30 Flush 19/30 Write block 377 len 28672 data: 8 10 13 13 13 15 13 20/30 Write block 408 len 16384 data: 14 12 10 7 21/30 Flush 22/30 Write block 583 len 36864 data: 13 14 15 17 17 16 15 19 17 23/30 Read block 141 len 4096 24/30 Write block 532 len 28672 data: 6 6 6 5 9 9 7 25/30 Read block 274 len 16384 26/30 Flush 27/30 Write block 584 len 40960 data: 15 16 18 18 17 16 20 18 15 12 28/30 Write block 409 len 24576 data: 13 11 8 6 7 9 29/30 Read block 433 len 8192 30/30 Write block 574 len 8192 data: 13 12 ---------------------------------------------------------------- Crucible gen:138 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 10 Done Done Done false 1027 Acked Write 6 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:38:15.05210741Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1734} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 133 133 133 242 242 242 F F F 1 020-039 137 137 137 249 249 249 F F F 2 040-059 135 135 135 245 245 245 F F F 3 060-079 135 135 135 245 245 245 F F F 4 080-099 134 134 134 243 243 243 F F F 5 100-119 132 132 132 241 241 241 F F F 6 120-139 130 130 130 237 237 237 F F F 7 140-159 136 136 136 248 248 248 F F F 8 160-179 137 137 137 249 249 249 F F F 9 180-199 137 137 137 249 249 249 F F F 10 200-219 138 138 138 255 255 255 F F F 11 220-239 138 138 138 255 255 255 F F F 12 240-259 138 138 138 255 255 255 F F F 13 260-279 136 136 136 248 248 248 F F F 14 280-299 136 136 136 248 248 248 F F F 15 300-319 136 136 136 247 247 247 F F F 16 320-339 134 134 134 243 243 243 F F F 17 340-359 138 138 138 252 252 252 F F F 18 360-379 138 138 138 256 256 256 F F F 19 380-399 138 138 138 256 256 256 F F F 20 400-419 138 138 138 256 256 256 T T T 21 420-439 134 134 134 243 243 243 F F F 22 440-459 132 132 132 240 240 240 F F F 23 460-479 133 133 133 242 242 242 F F F 24 480-499 138 138 138 254 254 254 F F F 25 500-519 138 138 138 254 254 254 F F F 26 520-539 138 138 138 257 257 257 F F F 27 540-559 136 136 136 247 247 247 F F F 28 560-579 134 134 134 243 243 243 T T T 29 580-599 138 138 138 257 257 257 T T T Max gen: 138, Max flush: 257 A Difference in extent metadata is expected here On loop 69 Verifying data now No change after verify Loop: 69 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 133 133 133 242 242 242 F F F 1 020-039 137 137 137 249 249 249 F F F 2 040-059 135 135 135 245 245 245 F F F 3 060-079 135 135 135 245 245 245 F F F 4 080-099 134 134 134 243 243 243 F F F 5 100-119 132 132 132 241 241 241 F F F 6 120-139 130 130 130 237 237 237 F F F 7 140-159 136 136 136 248 248 248 F F F 8 160-179 137 137 137 249 249 249 F F F 9 180-199 137 137 137 249 249 249 F F F 10 200-219 138 138 138 255 255 255 F F F 11 220-239 138 138 138 255 255 255 F F F 12 240-259 138 138 138 255 255 255 F F F 13 260-279 136 136 136 248 248 248 F F F 14 280-299 136 136 136 248 248 248 F F F 15 300-319 136 136 136 247 247 247 F F F 16 320-339 134 134 134 243 243 243 F F F 17 340-359 138 138 138 252 252 252 F F F 18 360-379 138 138 138 256 256 256 F F F 19 380-399 138 138 138 256 256 256 F F F 20 400-419 139 139 139 258 258 258 F F F 21 420-439 134 134 134 243 243 243 F F F 22 440-459 132 132 132 240 240 240 F F F 23 460-479 133 133 133 242 242 242 F F F 24 480-499 138 138 138 254 254 254 F F F 25 500-519 138 138 138 254 254 254 F F F 26 520-539 138 138 138 257 257 257 F F F 27 540-559 136 136 136 247 247 247 F F F 28 560-579 139 139 139 258 258 258 F F F 29 580-599 139 139 139 258 258 258 F F F Max gen: 139, Max flush: 258 Begin loop 70 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:19.313098109Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1742,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:19.358119694Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1742,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:19.358832165Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1742} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"26e7d501b33f47379e53601720353c2983e7654b\",\n git_commit_timestamp: \"2024-12-12T20:16:40.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:19.358882082Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1742} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:19.35890406Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1742} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:19.358927329Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1742} {"msg":"Crucible 580fa785-7830-47a2-b1e1-1c76a68851e5 has session id: c45553d1-ba4d-49dd-8953-23e7f4088758","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:19.359362689Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1742,"session_id":"c45553d1-ba4d-49dd-8953-23e7f4088758"} {"msg":"Upstairs opts: Upstairs UUID: 580fa785-7830-47a2-b1e1-1c76a68851e5, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:19.359410925Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1742,"session_id":"c45553d1-ba4d-49dd-8953-23e7f4088758"} {"msg":"Crucible stats registered with UUID: 580fa785-7830-47a2-b1e1-1c76a68851e5","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:19.359436184Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1742,"session_id":"c45553d1-ba4d-49dd-8953-23e7f4088758"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:19.390874036Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1742,"task":"crutest"} {"msg":"The guest has requested activation with gen:140","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:19.390932122Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1742} {"msg":"580fa785-7830-47a2-b1e1-1c76a68851e5 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:19.390949791Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1742,"session_id":"c45553d1-ba4d-49dd-8953-23e7f4088758"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:19.3909675Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1742,"client":"0","":"downstairs","session_id":"c45553d1-ba4d-49dd-8953-23e7f4088758"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:19.391005197Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1742,"client":"0","":"downstairs","session_id":"c45553d1-ba4d-49dd-8953-23e7f4088758"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:19.391022766Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1742,"client":"1","":"downstairs","session_id":"c45553d1-ba4d-49dd-8953-23e7f4088758"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:19.391037235Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1742,"client":"1","":"downstairs","session_id":"c45553d1-ba4d-49dd-8953-23e7f4088758"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:19.391050444Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1742,"client":"2","":"downstairs","session_id":"c45553d1-ba4d-49dd-8953-23e7f4088758"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:19.391064243Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1742,"client":"2","":"downstairs","session_id":"c45553d1-ba4d-49dd-8953-23e7f4088758"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:19.391077232Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1742,"":"io task","client":"2","":"downstairs","session_id":"c45553d1-ba4d-49dd-8953-23e7f4088758"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:19.39109754Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1742,"":"io task","client":"0","":"downstairs","session_id":"c45553d1-ba4d-49dd-8953-23e7f4088758"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:19.391122139Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1742,"":"io task","client":"1","":"downstairs","session_id":"c45553d1-ba4d-49dd-8953-23e7f4088758"} {"msg":"ds_connection connected from Ok(127.0.0.1:60998)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:19.391264129Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1742,"":"io task","client":"0","":"downstairs","session_id":"c45553d1-ba4d-49dd-8953-23e7f4088758"} {"msg":"ds_connection connected from Ok(127.0.0.1:40246)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:19.391441447Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1742,"":"io task","client":"1","":"downstairs","session_id":"c45553d1-ba4d-49dd-8953-23e7f4088758"} {"msg":"ds_connection connected from Ok(127.0.0.1:35661)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:19.391474674Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1742,"":"io task","client":"2","":"downstairs","session_id":"c45553d1-ba4d-49dd-8953-23e7f4088758"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:19.391997158Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1742,"client":"2","":"downstairs","session_id":"c45553d1-ba4d-49dd-8953-23e7f4088758"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:19.392026347Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1742,"client":"0","":"downstairs","session_id":"c45553d1-ba4d-49dd-8953-23e7f4088758"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:19.392245221Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1742,"client":"1","":"downstairs","session_id":"c45553d1-ba4d-49dd-8953-23e7f4088758"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:19.392762196Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1742,"client":"0","":"downstairs","session_id":"c45553d1-ba4d-49dd-8953-23e7f4088758"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:19.392792844Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1742,"client":"2","":"downstairs","session_id":"c45553d1-ba4d-49dd-8953-23e7f4088758"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:19.39285423Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1742,"client":"1","":"downstairs","session_id":"c45553d1-ba4d-49dd-8953-23e7f4088758"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:19.393400242Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1742,"session_id":"c45553d1-ba4d-49dd-8953-23e7f4088758"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:19.39342566Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1742,"session_id":"c45553d1-ba4d-49dd-8953-23e7f4088758"} {"msg":"[0]R flush_numbers[0..12]: [242, 249, 245, 245, 243, 241, 237, 248, 249, 249, 255, 255]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:19.393467407Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1742,"":"downstairs","":"downstairs","session_id":"c45553d1-ba4d-49dd-8953-23e7f4088758"} {"msg":"[0]R generation[0..12]: [133, 137, 135, 135, 134, 132, 130, 136, 137, 137, 138, 138]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:19.393485086Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1742,"":"downstairs","":"downstairs","session_id":"c45553d1-ba4d-49dd-8953-23e7f4088758"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:19.393499555Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1742,"":"downstairs","":"downstairs","session_id":"c45553d1-ba4d-49dd-8953-23e7f4088758"} {"msg":"[1]R flush_numbers[0..12]: [242, 249, 245, 245, 243, 241, 237, 248, 249, 249, 255, 255]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:19.393513514Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1742,"":"downstairs","":"downstairs","session_id":"c45553d1-ba4d-49dd-8953-23e7f4088758"} {"msg":"[1]R generation[0..12]: [133, 137, 135, 135, 134, 132, 130, 136, 137, 137, 138, 138]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:19.393527303Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1742,"":"downstairs","":"downstairs","session_id":"c45553d1-ba4d-49dd-8953-23e7f4088758"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:19.393541062Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1742,"":"downstairs","":"downstairs","session_id":"c45553d1-ba4d-49dd-8953-23e7f4088758"} {"msg":"[2]R flush_numbers[0..12]: [242, 249, 245, 245, 243, 241, 237, 248, 249, 249, 255, 255]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:19.39356895Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1742,"":"downstairs","":"downstairs","session_id":"c45553d1-ba4d-49dd-8953-23e7f4088758"} {"msg":"[2]R generation[0..12]: [133, 137, 135, 135, 134, 132, 130, 136, 137, 137, 138, 138]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:19.393583709Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1742,"":"downstairs","":"downstairs","session_id":"c45553d1-ba4d-49dd-8953-23e7f4088758"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:19.393597628Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1742,"":"downstairs","":"downstairs","session_id":"c45553d1-ba4d-49dd-8953-23e7f4088758"} {"msg":"Max found gen is 140","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:19.393614417Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1742,"":"downstairs","":"downstairs","session_id":"c45553d1-ba4d-49dd-8953-23e7f4088758"} {"msg":"Generation requested: 140 >= found:140","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:19.393628236Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1742,"":"downstairs","":"downstairs","session_id":"c45553d1-ba4d-49dd-8953-23e7f4088758"} {"msg":"Next flush: 259","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:19.393641005Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1742,"":"downstairs","":"downstairs","session_id":"c45553d1-ba4d-49dd-8953-23e7f4088758"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:19.393653854Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1742,"":"downstairs","":"downstairs","session_id":"c45553d1-ba4d-49dd-8953-23e7f4088758"} {"msg":"No downstairs reconciliation required","v":0,"name":Wait for a query_work_queue command to finish before sending IO "crucible","level":30,"time":"2024-12-12T20:38:19.393673453Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1742,"session_id":"c45553d1-ba4d-49dd-8953-23e7f4088758"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:19.393685642Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1742,"client":"0","":"downstairs","session_id":"c45553d1-ba4d-49dd-8953-23e7f4088758"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:19.393699181Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1742,"client":"1","":"downstairs","session_id":"c45553d1-ba4d-49dd-8953-23e7f4088758"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:19.39371223Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1742,"client":"2","":"downstairs","session_id":"c45553d1-ba4d-49dd-8953-23e7f4088758"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:19.393725389Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1742,"session_id":"c45553d1-ba4d-49dd-8953-23e7f4088758"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:19.393737478Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1742,"session_id":"c45553d1-ba4d-49dd-8953-23e7f4088758"} {"msg":"580fa785-7830-47a2-b1e1-1c76a68851e5 is now active with session: c45553d1-ba4d-49dd-8953-23e7f4088758","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:19.393750138Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1742,"session_id":"c45553d1-ba4d-49dd-8953-23e7f4088758"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:19.393762257Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1742,"session_id":"c45553d1-ba4d-49dd-8953-23e7f4088758"} {"msg":"The guest has finished waiting for activation with:140","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:19.393774526Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1742} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 23 len 12288 data: 12 15 13 02/30 Read block 488 len 4096 03/30 Read block 451 len 16384 04/30 Read block 498 len 40960 05/30 Read block 359 len 28672 06/30 Read block 97 len 24576 07/30 Write block 149 len 24576 data: 13 12 10 11 11 10 08/30 Write block 141 len 16384 data: 6 9 10 10 09/30 Read block 502 len 4096 10/30 Read block 134 len 36864 11/30 Read block 257 len 20480 12/30 Write block 485 len 28672 data: 13 12 12 13 14 12 9 13/30 Write block 326 len 12288 data: 8 7 8 14/30 Write block 416 len 20480 data: 9 8 10 10 10 15/30 Write block 152 len 24576 data: 12 12 11 10 10 10 16/30 Write block 10 len 36864 data: 10 11 9 7 9 10 8 8 9 17/30 Flush 18/30 Write block 315 len 8192 data: 16 16 19/30 Read block 353 len 28672 20/30 Read block 390 len 40960 21/30 Write block 216 len 16384 data: 8 10 9 11 22/30 Read block 35 len 24576 23/30 Read block 150 len 36864 24/30 Read block 372 len 20480 25/30 Flush 26/30 Write block 50 len 32768 data: 13 12 10 13 13 17 20 21 27/30 Read block 494 len 40960 28/30 Write block 169 len 4096 data: 7 29/30 Read block 585 len 40960 30/30 Write block 517 len 36864 data: 11 16 19 19 18 16 20 19 16 ---------------------------------------------------------------- Crucible gen:140 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 8 Done Done Done false 1026 Acked Read 10 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Read 10 Done Done Done false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:38:19.413679294Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1742} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 140 140 140 259 259 259 F F F 1 020-039 140 140 140 259 259 259 F F F 2 040-059 135 135 135 245 245 245 T T T 3 060-079 135 135 135 245 245 245 F F F 4 080-099 134 134 134 243 243 243 F F F 5 100-119 132 132 132 241 241 241 F F F 6 120-139 130 130 130 237 237 237 F F F 7 140-159 140 140 140 259 259 259 F F F 8 160-179 137 137 137 249 249 249 T T T 9 180-199 137 137 137 249 249 249 F F F 10 200-219 140 140 140 260 260 260 F F F 11 220-239 138 138 138 255 255 255 F F F 12 240-259 138 138 138 255 255 255 F F F 13 260-279 136 136 136 248 248 248 F F F 14 280-299 136 136 136 248 248 248 F F F 15 300-319 140 140 140 260 260 260 F F F 16 320-339 140 140 140 259 259 259 F F F 17 340-359 138 138 138 252 252 252 F F F 18 360-379 138 138 138 256 256 256 F F F 19 380-399 138 138 138 256 256 256 F F F 20 400-419 140 140 140 259 259 259 F F F 21 420-439 140 140 140 259 259 259 F F F 22 440-459 132 132 132 240 240 240 F F F 23 460-479 133 133 133 242 242 242 F F F 24 480-499 140 140 140 259 259 259 F F F 25 500-519 138 138 138 254 254 254 T T T 26 520-539 138 138 138 257 257 257 T T T 27 540-559 136 136 136 247 247 247 F F F 28 560-579 139 139 139 258 258 258 F F F 29 580-599 139 139 139 258 258 258 F F F Max gen: 140, Max flush: 260 A Difference in extent metadata is expected here On loop 70 Verifying data now No change after verify Loop: 70 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 140 140 140 259 259 259 F F F 1 020-039 140 140 140 259 259 259 F F F 2 040-059 141 141 141 261 261 261 F F F 3 060-079 135 135 135 245 245 245 F F F 4 080-099 134 134 134 243 243 243 F F F 5 100-119 132 132 132 241 241 241 F F F 6 120-139 130 130 130 237 237 237 F F F 7 140-159 140 140 140 259 259 259 F F F 8 160-179 141 141 141 261 261 261 F F F 9 180-199 137 137 137 249 249 249 F F F 10 200-219 140 140 140 260 260 260 F F F 11 220-239 138 138 138 255 255 255 F F F 12 240-259 138 138 138 255 255 255 F F F 13 260-279 136 136 136 248 248 248 F F F 14 280-299 136 136 136 248 248 248 F F F 15 300-319 140 140 140 260 260 260 F F F 16 320-339 140 140 140 259 259 259 F F F 17 340-359 138 138 138 252 252 252 F F F 18 360-379 138 138 138 256 256 256 F F F 19 380-399 138 138 138 256 256 256 F F F 20 400-419 140 140 140 259 259 259 F F F 21 420-439 140 140 140 259 259 259 F F F 22 440-459 132 132 132 240 240 240 F F F 23 460-479 133 133 133 242 242 242 F F F 24 480-499 140 140 140 259 259 259 F F F 25 500-519 141 141 141 261 261 261 F F F 26 520-539 141 141 141 261 261 261 F F F 27 540-559 136 136 136 247 247 247 F F F 28 560-579 139 139 139 258 258 258 F F F 29 580-599 139 139 139 258 258 258 F F F Max gen: 141, Max flush: 261 Begin loop 71 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:23.681195343Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1750,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:23.725784308Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1750,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:23.726409795Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1750} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"26e7d501b33f47379e53601720353c2983e7654b\",\n git_commit_timestamp: \"2024-12-12T20:16:40.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:23.726445412Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1750} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:23.726459501Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1750} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:23.72647393Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1750} {"msg":"Crucible fbd22258-3b7f-4d9a-8b9c-1094ce491394 has session id: 93060cf3-07e5-4ff4-ab03-c9f80ceafe1d","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:23.726903071Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1750,"session_id":"93060cf3-07e5-4ff4-ab03-c9f80ceafe1d"} {"msg":"Upstairs opts: Upstairs UUID: fbd22258-3b7f-4d9a-8b9c-1094ce491394, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:23.726936378Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1750,"session_id":"93060cf3-07e5-4ff4-ab03-c9f80ceafe1d"} {"msg":"Crucible stats registered with UUID: fbd22258-3b7f-4d9a-8b9c-1094ce491394","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:23.726955277Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1750,"session_id":"93060cf3-07e5-4ff4-ab03-c9f80ceafe1d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:23.75780029Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1750,"task":"crutest"} {"msg":"The guest has requested activation with gen:142","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:23.757826868Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1750} {"msg":"fbd22258-3b7f-4d9a-8b9c-1094ce491394 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:23.757900063Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1750,"session_id":"93060cf3-07e5-4ff4-ab03-c9f80ceafe1d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:23.757919212Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1750,"client":"0","":"downstairs","session_id":"93060cf3-07e5-4ff4-ab03-c9f80ceafe1d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:23.75794258Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1750,"client":"0","":"downstairs","session_id":"93060cf3-07e5-4ff4-ab03-c9f80ceafe1d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:23.757956569Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1750,"client":"1","":"downstairs","session_id":"93060cf3-07e5-4ff4-ab03-c9f80ceafe1d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:23.757972298Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1750,"client":"1","":"downstairs","session_id":"93060cf3-07e5-4ff4-ab03-c9f80ceafe1d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:23.757985737Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1750,"client":"2","":"downstairs","session_id":"93060cf3-07e5-4ff4-ab03-c9f80ceafe1d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:23.758010626Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1750,"client":"2","":"downstairs","session_id":"93060cf3-07e5-4ff4-ab03-c9f80ceafe1d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:23.758036104Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1750,"":"io task","client":"2","":"downstairs","session_id":"93060cf3-07e5-4ff4-ab03-c9f80ceafe1d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:23.758061362Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1750,"":"io task","client":"0","":"downstairs","session_id":"93060cf3-07e5-4ff4-ab03-c9f80ceafe1d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:23.75808569Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1750,"":"io task","client":"1","":"downstairs","session_id":"93060cf3-07e5-4ff4-ab03-c9f80ceafe1d"} {"msg":"ds_connection connected from Ok(127.0.0.1:55994)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:23.758190893Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1750,"":"io task","client":"0","":"downstairs","session_id":"93060cf3-07e5-4ff4-ab03-c9f80ceafe1d"} {"msg":"ds_connection connected from Ok(127.0.0.1:39325)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:23.75838814Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1750,"":"io task","client":"1","":"downstairs","session_id":"93060cf3-07e5-4ff4-ab03-c9f80ceafe1d"} {"msg":"ds_connection connected from Ok(127.0.0.1:43262)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:23.758426727Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1750,"":"io task","client":"2","":"downstairs","session_id":"93060cf3-07e5-4ff4-ab03-c9f80ceafe1d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:23.758928632Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1750,"client":"2","":"downstairs","session_id":"93060cf3-07e5-4ff4-ab03-c9f80ceafe1d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:23.758990958Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1750,"client":"0","":"downstairs","session_id":"93060cf3-07e5-4ff4-ab03-c9f80ceafe1d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:23.759017436Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1750,"client":"1","":"downstairs","session_id":"93060cf3-07e5-4ff4-ab03-c9f80ceafe1d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:23.759593917Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1750,"client":"1","":"downstairs","session_id":"93060cf3-07e5-4ff4-ab03-c9f80ceafe1d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:23.759624714Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1750,"client":"2","":"downstairs","session_id":"93060cf3-07e5-4ff4-ab03-c9f80ceafe1d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:23.759641013Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1750,"client":"0","":"downstairs","session_id":"93060cf3-07e5-4ff4-ab03-c9f80ceafe1d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:23.760182666Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1750,"session_id":"93060cf3-07e5-4ff4-ab03-c9f80ceafe1d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:23.760209594Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1750,"session_id":"93060cf3-07e5-4ff4-ab03-c9f80ceafe1d"} {"msg":"[0]R flush_numbers[0..12]: [259, 259, 261, 245, 243, 241, 237, 259, 261, 249, 260, 255]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:23.760273939Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1750,"":"downstairs","":"downstairs","session_id":"93060cf3-07e5-4ff4-ab03-c9f80ceafe1d"} {"msg":"[0]R generation[0..12]: [140, 140, 141, 135, 134, 132, 130, 140, 141, 137, 140, 138]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:23.760293758Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1750,"":"downstairs","":"downstairs","session_id":"93060cf3-07e5-4ff4-ab03-c9f80ceafe1d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:23.760308547Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1750,"":"downstairs","":"downstairs","session_id":"93060cf3-07e5-4ff4-ab03-c9f80ceafe1d"} {"msg":"[1]R flush_numbers[0..12]: [259, 259, 261, 245, 243, 241, 237, 259, 261, 249, 260, 255]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:23.760322676Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1750,"":"downstairs","":"downstairs","session_id":"93060cf3-07e5-4ff4-ab03-c9f80ceafe1d"} {"msg":"[1]R generation[0..12]: [140, 140, 141, 135, 134, 132, 130, 140, 141, 137, 140, 138]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:23.760337255Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1750,"":"downstairs","":"downstairs","session_id":"93060cf3-07e5-4ff4-ab03-c9f80ceafe1d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:23.760354444Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1750,"":"downstairs","":"downstairs","session_id":"93060cf3-07e5-4ff4-ab03-c9f80ceafe1d"} {"msg":"[2]R flush_numbers[0..12]: [259, 259, 261, 245, 243, 241, 237, 259, 261, 249, 260, 255]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:23.760368543Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1750,"":"downstairs","":"downstairs","session_id":"93060cf3-07e5-4ff4-ab03-c9f80ceafe1d"} {"msg":"[2]R generation[0..12]: [140, 140, 141, 135, 134, 132, 130, 140, 141, 137, 140, 138]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:23.760382202Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1750,"":"downstairs","":"downstairs","session_id":"93060cf3-07e5-4ff4-ab03-c9f80ceafe1d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:23.760396921Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1750,"":"downstairs","":"downstairs","session_id":"93060cf3-07e5-4ff4-ab03-c9f80ceafe1d"} {"msg":"Max found gen is 142","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:23.76041001Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1750,"":"downstairs","":"downstairs","session_id":"93060cf3-07e5-4ff4-ab03-c9f80ceafe1d"} {"msg":"Generation requested: 142 >= found:142","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:23.760423209Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1750,"":"downstairs","":"downstairs","session_id":"93060cf3-07e5-4ff4-ab03-c9f80ceafe1d"} {"msg":"Next flush: 262","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:23.760436158Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1750,"":"downstairs","":"downstairs","session_id":"93060cf3-07e5-4ff4-ab03-c9f80ceafe1d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:23.760448987Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1750,"":"downstairs","":"downstairs","session_id":"93060cf3-07e5-4ff4-ab03-c9f80ceafe1d"} {"msg":"Wait for a query_work_queue command to finish before sending IO No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:23.760466336Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1750,"session_id":"93060cf3-07e5-4ff4-ab03-c9f80ceafe1d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:23.760478556Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1750,"client":"0","":"downstairs","session_id":"93060cf3-07e5-4ff4-ab03-c9f80ceafe1d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:23.760491684Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1750,"client":"1","":"downstairs","session_id":"93060cf3-07e5-4ff4-ab03-c9f80ceafe1d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:23.760504734Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1750,"client":"2","":"downstairs","session_id":"93060cf3-07e5-4ff4-ab03-c9f80ceafe1d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:23.760518063Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1750,"session_id":"93060cf3-07e5-4ff4-ab03-c9f80ceafe1d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:23.760530242Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1750,"session_id":"93060cf3-07e5-4ff4-ab03-c9f80ceafe1d"} {"msg":"fbd22258-3b7f-4d9a-8b9c-1094ce491394 is now active with session: 93060cf3-07e5-4ff4-ab03-c9f80ceafe1d","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:23.760543121Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1750,"session_id":"93060cf3-07e5-4ff4-ab03-c9f80ceafe1d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:23.76055736Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1750,"session_id":"93060cf3-07e5-4ff4-ab03-c9f80ceafe1d"} {"msg":"The guest has finished waiting for activation with:142","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:23.760570019Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1750} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 470 len 4096 data: 9 02/30 Read block 587 len 32768 03/30 Flush 04/30 Flush 05/30 Write block 373 len 12288 data: 7 9 7 06/30 Write block 342 len 20480 data: 11 13 15 14 14 07/30 Read block 248 len 20480 08/30 Write block 89 len 24576 data: 13 14 11 8 12 11 09/30 Write block 467 len 20480 data: 11 9 10 10 8 10/30 Flush 11/30 Write block 212 len 28672 data: 12 14 12 11 9 11 10 12/30 Write block 569 len 4096 data: 12 13/30 Write block 270 len 20480 data: 11 11 13 12 14 14/30 Write block 237 len 16384 data: 12 11 12 11 15/30 Write block 481 len 40960 data: 8 8 7 11 14 13 13 14 15 13 16/30 Write block 4 len 12288 data: 7 8 8 17/30 Read block 461 len 8192 18/30 Read block 347 len 12288 19/30 Read block 217 len 8192 20/30 Read block 306 len 36864 21/30 Read block 258 len 24576 22/30 Read block 55 len 36864 23/30 Write block 550 len 4096 data: 9 24/30 Read block 155 len 4096 25/30 Flush 26/30 Write block 384 len 32768 data: 12 13 15 12 11 9 11 10 27/30 Read block 359 len 32768 28/30 Write block 93 len 28672 data: 13 12 11 7 6 7 11 29/30 Read block 335 len 24576 30/30 Read block 153 len 28672 ---------------------------------------------------------------- Crucible gen:142 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 8 Done Done Done false 1026 Acked Read 8 Done Done Done false 1027 Acked Write 7 Done Done Done false 1028 Acked Read 6 Done Done Done false 1029 Acked Read 7 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:38:23.779064783Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1750} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 142 142 142 265 265 265 F F F 1 020-039 140 140 140 259 259 259 F F F 2 040-059 141 141 141 261 261 261 F F F 3 060-079 135 135 135 245 245 245 F F F 4 080-099 142 142 142 264 264 264 T T T 5 100-119 132 132 132 241 241 241 F F F 6 120-139 130 130 130 237 237 237 F F F 7 140-159 140 140 140 259 259 259 F F F 8 160-179 141 141 141 261 261 261 F F F 9 180-199 137 137 137 249 249 249 F F F 10 200-219 142 142 142 265 265 265 F F F 11 220-239 142 142 142 265 265 265 F F F 12 240-259 142 142 142 265 265 265 F F F 13 260-279 142 142 142 265 265 265 F F F 14 280-299 136 136 136 248 248 248 F F F 15 300-319 140 140 140 260 260 260 F F F 16 320-339 140 140 140 259 259 259 F F F 17 340-359 142 142 142 264 264 264 F F F 18 360-379 142 142 142 264 264 264 F F F 19 380-399 138 138 138 256 256 256 T T T 20 400-419 140 140 140 259 259 259 F F F 21 420-439 140 140 140 259 259 259 F F F 22 440-459 132 132 132 240 240 240 F F F 23 460-479 142 142 142 264 264 264 F F F 24 480-499 142 142 142 265 265 265 F F F 25 500-519 141 141 141 261 261 261 F F F 26 520-539 141 141 141 261 261 261 F F F 27 540-559 142 142 142 265 265 265 F F F 28 560-579 142 142 142 265 265 265 F F F 29 580-599 139 139 139 258 258 258 F F F Max gen: 142, Max flush: 265 A Difference in extent metadata is expected here On loop 71 Verifying data now No change after verify Loop: 71 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 142 142 142 265 265 265 F F F 1 020-039 140 140 140 259 259 259 F F F 2 040-059 141 141 141 261 261 261 F F F 3 060-079 135 135 135 245 245 245 F F F 4 080-099 143 143 143 266 266 266 F F F 5 100-119 132 132 132 241 241 241 F F F 6 120-139 130 130 130 237 237 237 F F F 7 140-159 140 140 140 259 259 259 F F F 8 160-179 141 141 141 261 261 261 F F F 9 180-199 137 137 137 249 249 249 F F F 10 200-219 142 142 142 265 265 265 F F F 11 220-239 142 142 142 265 265 265 F F F 12 240-259 142 142 142 265 265 265 F F F 13 260-279 142 142 142 265 265 265 F F F 14 280-299 136 136 136 248 248 248 F F F 15 300-319 140 140 140 260 260 260 F F F 16 320-339 140 140 140 259 259 259 F F F 17 340-359 142 142 142 264 264 264 F F F 18 360-379 142 142 142 264 264 264 F F F 19 380-399 143 143 143 266 266 266 F F F 20 400-419 140 140 140 259 259 259 F F F 21 420-439 140 140 140 259 259 259 F F F 22 440-459 132 132 132 240 240 240 F F F 23 460-479 142 142 142 264 264 264 F F F 24 480-499 142 142 142 265 265 265 F F F 25 500-519 141 141 141 261 261 261 F F F 26 520-539 141 141 141 261 261 261 F F F 27 540-559 142 142 142 265 265 265 F F F 28 560-579 142 142 142 265 265 265 F F F 29 580-599 139 139 139 258 258 258 F F F Max gen: 143, Max flush: 266 Begin loop 72 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:28.035569431Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1758,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:28.080320565Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1758,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:28.081029466Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1758} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"26e7d501b33f47379e53601720353c2983e7654b\",\n git_commit_timestamp: \"2024-12-12T20:16:40.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:28.081068404Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1758} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:28.081083083Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1758} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:28.081101821Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1758} {"msg":"Crucible 33335113-3d0b-4560-a1a6-1e70fa0aff0a has session id: 4337ec24-7020-4d59-a71e-dfa1347f4768","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:28.081598987Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1758,"session_id":"4337ec24-7020-4d59-a71e-dfa1347f4768"} {"msg":"Upstairs opts: Upstairs UUID: 33335113-3d0b-4560-a1a6-1e70fa0aff0a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:28.081634005Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1758,"session_id":"4337ec24-7020-4d59-a71e-dfa1347f4768"} {"msg":"Crucible stats registered with UUID: 33335113-3d0b-4560-a1a6-1e70fa0aff0a","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:28.081648774Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1758,"session_id":"4337ec24-7020-4d59-a71e-dfa1347f4768"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:28.112531804Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1758,"task":"crutest"} {"msg":"The guest has requested activation with gen:144","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:28.112558152Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1758} {"msg":"33335113-3d0b-4560-a1a6-1e70fa0aff0a active request set","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:28.112624288Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1758,"session_id":"4337ec24-7020-4d59-a71e-dfa1347f4768"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:28.112652856Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1758,"client":"0","":"downstairs","session_id":"4337ec24-7020-4d59-a71e-dfa1347f4768"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:28.112676514Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1758,"client":"0","":"downstairs","session_id":"4337ec24-7020-4d59-a71e-dfa1347f4768"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:28.112690833Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1758,"client":"1","":"downstairs","session_id":"4337ec24-7020-4d59-a71e-dfa1347f4768"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:28.112705942Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1758,"client":"1","":"downstairs","session_id":"4337ec24-7020-4d59-a71e-dfa1347f4768"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:28.112719201Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1758,"client":"2","":"downstairs","session_id":"4337ec24-7020-4d59-a71e-dfa1347f4768"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:28.11273325Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1758,"client":"2","":"downstairs","session_id":"4337ec24-7020-4d59-a71e-dfa1347f4768"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:28.112746449Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1758,"":"io task","client":"0","":"downstairs","session_id":"4337ec24-7020-4d59-a71e-dfa1347f4768"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:28.112760258Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1758,"":"io task","client":"2","":"downstairs","session_id":"4337ec24-7020-4d59-a71e-dfa1347f4768"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:28.112777197Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1758,"":"io task","client":"1","":"downstairs","session_id":"4337ec24-7020-4d59-a71e-dfa1347f4768"} {"msg":"ds_connection connected from Ok(127.0.0.1:49092)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:28.112939666Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1758,"":"io task","client":"2","":"downstairs","session_id":"4337ec24-7020-4d59-a71e-dfa1347f4768"} {"msg":"ds_connection connected from Ok(127.0.0.1:60511)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:28.112982433Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1758,"":"io task","client":"1","":"downstairs","session_id":"4337ec24-7020-4d59-a71e-dfa1347f4768"} {"msg":"ds_connection connected from Ok(127.0.0.1:60052)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:28.113012011Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1758,"":"io task","client":"0","":"downstairs","session_id":"4337ec24-7020-4d59-a71e-dfa1347f4768"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:28.113772878Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1758,"client":"0","":"downstairs","session_id":"4337ec24-7020-4d59-a71e-dfa1347f4768"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:28.113814586Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1758,"client":"1","":"downstairs","session_id":"4337ec24-7020-4d59-a71e-dfa1347f4768"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:28.113846373Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1758,"client":"2","":"downstairs","session_id":"4337ec24-7020-4d59-a71e-dfa1347f4768"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:28.114354298Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1758,"client":"0","":"downstairs","session_id":"4337ec24-7020-4d59-a71e-dfa1347f4768"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:28.114466781Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1758,"client":"1","":"downstairs","session_id":"4337ec24-7020-4d59-a71e-dfa1347f4768"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:28.114486799Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1758,"client":"2","":"downstairs","session_id":"4337ec24-7020-4d59-a71e-dfa1347f4768"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:28.114900821Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1758,"session_id":"4337ec24-7020-4d59-a71e-dfa1347f4768"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:28.115002394Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1758,"session_id":"4337ec24-7020-4d59-a71e-dfa1347f4768"} {"msg":"[0]R flush_numbers[0..12]: [265, 259, 261, 245, 266, 241, 237, 259, 261, 249, 265, 265]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:28.115110466Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1758,"":"downstairs","":"downstairs","session_id":"4337ec24-7020-4d59-a71e-dfa1347f4768"} {"msg":"[0]R generation[0..12]: [142, 140, 141, 135, 143, 132, 130, 140, 141, 137, 142, 142]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:28.115131145Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1758,"":"downstairs","":"downstairs","session_id":"4337ec24-7020-4d59-a71e-dfa1347f4768"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:28.115149204Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1758,"":"downstairs","":"downstairs","session_id":"4337ec24-7020-4d59-a71e-dfa1347f4768"} {"msg":"[1]R flush_numbers[0..12]: [265, 259, 261, 245, 266, 241, 237, 259, 261, 249, 265, 265]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:28.115163223Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1758,"":"downstairs","":"downstairs","session_id":"4337ec24-7020-4d59-a71e-dfa1347f4768"} {"msg":"[1]R generation[0..12]: [142, 140, 141, 135, 143, 132, 130, 140, 141, 137, 142, 142]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:28.115176992Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1758,"":"downstairs","":"downstairs","session_id":"4337ec24-7020-4d59-a71e-dfa1347f4768"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:28.115190991Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1758,"":"downstairs","":"downstairs","session_id":"4337ec24-7020-4d59-a71e-dfa1347f4768"} {"msg":"[2]R flush_numbers[0..12]: [265, 259, 261, 245, 266, 241, 237, 259, 261, 249, 265, 265]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:28.1152047Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1758,"":"downstairs","":"downstairs","session_id":"4337ec24-7020-4d59-a71e-dfa1347f4768"} {"msg":"[2]R generation[0..12]: [142, 140, 141, 135, 143, 132, 130, 140, 141, 137, 142, 142]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:28.115218409Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1758,"":"downstairs","":"downstairs","session_id":"4337ec24-7020-4d59-a71e-dfa1347f4768"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:28.115232398Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1758,"":"downstairs","":"downstairs","session_id":"4337ec24-7020-4d59-a71e-dfa1347f4768"} {"msg":"Max found gen is 144","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:28.115245407Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1758,"":"downstairs","":"downstairs","session_id":"4337ec24-7020-4d59-a71e-dfa1347f4768"} {"msg":"Generation requested: 144 >= found:144","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:28.115258606Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1758,"":"downstairs","":"downstairs","session_id":"4337ec24-7020-4d59-a71e-dfa1347f4768"} {"msg":"Next flush: 267","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:28.115271395Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1758,"":"downstairs","":"downstairs","session_id":"4337ec24-7020-4d59-a71e-dfa1347f4768"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:28.115298623Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1758,"":"downstairs","":"downstairs","session_id":"4337ec24-7020-4d59-a71e-dfa1347f4768"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:28.115312792Z","hostname"Wait for a query_work_queue command to finish before sending IO :"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1758,"session_id":"4337ec24-7020-4d59-a71e-dfa1347f4768"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:28.115329971Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1758,"client":"0","":"downstairs","session_id":"4337ec24-7020-4d59-a71e-dfa1347f4768"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:28.11534364Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1758,"client":"1","":"downstairs","session_id":"4337ec24-7020-4d59-a71e-dfa1347f4768"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:28.115359399Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1758,"client":"2","":"downstairs","session_id":"4337ec24-7020-4d59-a71e-dfa1347f4768"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:28.115372718Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1758,"session_id":"4337ec24-7020-4d59-a71e-dfa1347f4768"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:28.115384927Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1758,"session_id":"4337ec24-7020-4d59-a71e-dfa1347f4768"} {"msg":"33335113-3d0b-4560-a1a6-1e70fa0aff0a is now active with session: 4337ec24-7020-4d59-a71e-dfa1347f4768","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:28.115397546Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1758,"session_id":"4337ec24-7020-4d59-a71e-dfa1347f4768"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:28.115409436Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1758,"session_id":"4337ec24-7020-4d59-a71e-dfa1347f4768"} {"msg":"The guest has finished waiting for activation with:144","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:28.115421485Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1758} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 306 len 32768 data: 10 9 11 10 9 13 11 14 02/30 Write block 181 len 16384 data: 9 9 10 13 03/30 Flush 04/30 Write block 568 len 32768 data: 11 13 11 14 14 13 14 13 05/30 Read block 125 len 24576 06/30 Read block 315 len 8192 07/30 Read block 317 len 4096 08/30 Flush 09/30 Write block 123 len 28672 data: 10 10 11 11 10 11 11 10/30 Read block 410 len 24576 11/30 Read block 88 len 16384 12/30 Write block 117 len 8192 data: 9 10 13/30 Write block 129 len 40960 data: 12 11 8 6 6 7 8 10 11 13 14/30 Flush 15/30 Write block 243 len 24576 data: 10 8 7 8 6 6 16/30 Read block 95 len 20480 17/30 Write block 424 len 32768 data: 13 11 9 12 12 13 11 13 18/30 Read block 121 len 16384 19/30 Read block 264 len 28672 20/30 Read block 239 len 20480 21/30 Write block 407 len 40960 data: 14 15 14 12 9 7 8 10 9 10 22/30 Write block 566 len 36864 data: 12 9 12 14 12 15 15 14 15 23/30 Write block 209 len 28672 data: 11 14 16 13 15 13 12 24/30 Read block 509 len 24576 25/30 Write block 205 len 16384 data: 15 15 10 13 26/30 Write block 347 len 24576 data: 13 11 10 13 9 12 27/30 Write block 432 len 28672 data: 13 12 9 12 10 9 11 28/30 Write block 504 len 40960 data: 11 10 11 12 10 10 11 11 12 10 29/30 Write block 352 len 16384 data: 13 12 15 15 30/30 Read block 252 len 32768 ---------------------------------------------------------------- Crucible gen:144 GIO:true work queues: Upstairs:0 downstairs:16 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1014 Acked Write 6 Done Done Done false 1015 Acked Read 5 Done Done Done false 1016 Acked Write 8 Done Done Done false 1017 Acked Read 4 Done Done Done false 1018 Acked Read 7 Done Done Done false 1019 Acked Read 5 Done Done Done false 1020 Acked Write 10 Done Done Done false 1021 Acked Write 9 Done Done Done false 1022 Acked Write 7 Done Done Done false 1023 Acked Read 6 Done Done Done false 1024 Acked Write 4 Done Done Done false 1025 Acked Write 6 Done Done Done false 1026 Acked Write 7 Done Done Done false 1027 Acked Write 10 Done Done Sent false 1028 Acked Write 4 Done Done Sent false 1029 Acked Read 8 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 3 4 Done 15 16 13 44 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1013 1013 1013 Downstairs last five completed: 1013 1012 1011 1010 1009 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:38:28.135404697Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1758} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 142 142 142 265 265 265 F F F 1 020-039 140 140 140 259 259 259 F F F 2 040-059 141 141 141 261 261 261 F F F 3 060-079 135 135 135 245 245 245 F F F 4 080-099 143 143 143 266 266 266 F F F 5 100-119 144 144 144 269 269 269 F F F 6 120-139 144 144 144 269 269 269 F F F 7 140-159 140 140 140 259 259 259 F F F 8 160-179 141 141 141 261 261 261 F F F 9 180-199 144 144 144 267 267 267 F F F 10 200-219 142 142 142 265 265 265 T T T 11 220-239 142 142 142 265 265 265 F F F 12 240-259 142 142 142 265 265 265 T T T 13 260-279 142 142 142 265 265 265 F F F 14 280-299 136 136 136 248 248 248 F F F 15 300-319 144 144 144 267 267 267 F F F 16 320-339 140 140 140 259 259 259 F F F 17 340-359 142 142 142 264 264 264 T T T 18 360-379 142 142 142 264 264 264 F F F 19 380-399 143 143 143 266 266 266 F F F 20 400-419 140 140 140 259 259 259 T T T 21 420-439 140 140 140 259 259 259 T T T 22 440-459 132 132 132 240 240 240 F F F 23 460-479 142 142 142 264 264 264 F F F 24 480-499 142 142 142 265 265 265 F F F 25 500-519 141 141 141 261 261 261 T T T 26 520-539 141 141 141 261 261 261 F F F 27 540-559 142 142 142 265 265 265 F F F 28 560-579 144 144 144 268 268 268 T T T 29 580-599 139 139 139 258 258 258 F F F Max gen: 144, Max flush: 269 A Difference in extent metadata is expected here On loop 72 Verifying data now No change after verify Loop: 72 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 142 142 142 265 265 265 F F F 1 020-039 140 140 140 259 259 259 F F F 2 040-059 141 141 141 261 261 261 F F F 3 060-079 135 135 135 245 245 245 F F F 4 080-099 143 143 143 266 266 266 F F F 5 100-119 144 144 144 269 269 269 F F F 6 120-139 144 144 144 269 269 269 F F F 7 140-159 140 140 140 259 259 259 F F F 8 160-179 141 141 141 261 261 261 F F F 9 180-199 144 144 144 267 267 267 F F F 10 200-219 145 145 145 270 270 270 F F F 11 220-239 142 142 142 265 265 265 F F F 12 240-259 145 145 145 270 270 270 F F F 13 260-279 142 142 142 265 265 265 F F F 14 280-299 136 136 136 248 248 248 F F F 15 300-319 144 144 144 267 267 267 F F F 16 320-339 140 140 140 259 259 259 F F F 17 340-359 145 145 145 270 270 270 F F F 18 360-379 142 142 142 264 264 264 F F F 19 380-399 143 143 143 266 266 266 F F F 20 400-419 145 145 145 270 270 270 F F F 21 420-439 145 145 145 270 270 270 F F F 22 440-459 132 132 132 240 240 240 F F F 23 460-479 142 142 142 264 264 264 F F F 24 480-499 142 142 142 265 265 265 F F F 25 500-519 145 145 145 270 270 270 F F F 26 520-539 141 141 141 261 261 261 F F F 27 540-559 142 142 142 265 265 265 F F F 28 560-579 145 145 145 270 270 270 F F F 29 580-599 139 139 139 258 258 258 F F F Max gen: 145, Max flush: 270 Begin loop 73 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:32.418568907Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1766,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:32.462455261Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1766,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:32.463167302Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1766} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"26e7d501b33f47379e53601720353c2983e7654b\",\n git_commit_timestamp: \"2024-12-12T20:16:40.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:32.463204159Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1766} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:32.463218518Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1766} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:32.463232927Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1766} {"msg":"Crucible 1b1f9707-cbdb-493c-a2d5-52e904ce12ae has session id: 80edb1fb-6898-4fcc-adb6-7c06657e4c05","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:32.463742632Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1766,"session_id":"80edb1fb-6898-4fcc-adb6-7c06657e4c05"} {"msg":"Upstairs opts: Upstairs UUID: 1b1f9707-cbdb-493c-a2d5-52e904ce12ae, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:32.463782719Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1766,"session_id":"80edb1fb-6898-4fcc-adb6-7c06657e4c05"} {"msg":"Crucible stats registered with UUID: 1b1f9707-cbdb-493c-a2d5-52e904ce12ae","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:32.463798868Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1766,"session_id":"80edb1fb-6898-4fcc-adb6-7c06657e4c05"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:32.495141407Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1766,"task":"crutest"} {"msg":"The guest has requested activation with gen:146","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:32.495173085Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1766} {"msg":"1b1f9707-cbdb-493c-a2d5-52e904ce12ae active request set","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:32.495249829Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1766,"session_id":"80edb1fb-6898-4fcc-adb6-7c06657e4c05"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:32.495271978Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1766,"client":"0","":"downstairs","session_id":"80edb1fb-6898-4fcc-adb6-7c06657e4c05"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:32.495304706Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1766,"client":"0","":"downstairs","session_id":"80edb1fb-6898-4fcc-adb6-7c06657e4c05"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:32.495322924Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1766,"client":"1","":"downstairs","session_id":"80edb1fb-6898-4fcc-adb6-7c06657e4c05"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:32.495340533Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1766,"client":"1","":"downstairs","session_id":"80edb1fb-6898-4fcc-adb6-7c06657e4c05"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:32.495356672Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1766,"client":"2","":"downstairs","session_id":"80edb1fb-6898-4fcc-adb6-7c06657e4c05"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:32.495373911Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1766,"client":"2","":"downstairs","session_id":"80edb1fb-6898-4fcc-adb6-7c06657e4c05"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:32.495400059Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1766,"":"io task","client":"0","":"downstairs","session_id":"80edb1fb-6898-4fcc-adb6-7c06657e4c05"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:32.495433627Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1766,"":"io task","client":"2","":"downstairs","session_id":"80edb1fb-6898-4fcc-adb6-7c06657e4c05"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:32.495465445Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1766,"":"io task","client":"1","":"downstairs","session_id":"80edb1fb-6898-4fcc-adb6-7c06657e4c05"} {"msg":"ds_connection connected from Ok(127.0.0.1:47932)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:32.495498622Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1766,"":"io task","client":"0","":"downstairs","session_id":"80edb1fb-6898-4fcc-adb6-7c06657e4c05"} {"msg":"ds_connection connected from Ok(127.0.0.1:43692)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:32.49552845Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1766,"":"io task","client":"2","":"downstairs","session_id":"80edb1fb-6898-4fcc-adb6-7c06657e4c05"} {"msg":"ds_connection connected from Ok(127.0.0.1:63629)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:32.495556868Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1766,"":"io task","client":"1","":"downstairs","session_id":"80edb1fb-6898-4fcc-adb6-7c06657e4c05"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:32.496371602Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1766,"client":"1","":"downstairs","session_id":"80edb1fb-6898-4fcc-adb6-7c06657e4c05"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:32.49640382Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1766,"client":"0","":"downstairs","session_id":"80edb1fb-6898-4fcc-adb6-7c06657e4c05"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:32.496419829Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1766,"client":"2","":"downstairs","session_id":"80edb1fb-6898-4fcc-adb6-7c06657e4c05"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:32.497031127Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1766,"client":"1","":"downstairs","session_id":"80edb1fb-6898-4fcc-adb6-7c06657e4c05"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:32.497062995Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1766,"client":"0","":"downstairs","session_id":"80edb1fb-6898-4fcc-adb6-7c06657e4c05"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:32.497078953Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1766,"client":"2","":"downstairs","session_id":"80edb1fb-6898-4fcc-adb6-7c06657e4c05"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:32.49756677Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1766,"session_id":"80edb1fb-6898-4fcc-adb6-7c06657e4c05"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:32.497677672Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1766,"session_id":"80edb1fb-6898-4fcc-adb6-7c06657e4c05"} {"msg":"[0]R flush_numbers[0..12]: [265, 259, 261, 245, 266, 269, 269, 259, 261, 267, 270, 265]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:32.497803293Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1766,"":"downstairs","":"downstairs","session_id":"80edb1fb-6898-4fcc-adb6-7c06657e4c05"} {"msg":"[0]R generation[0..12]: [142, 140, 141, 135, 143, 144, 144, 140, 141, 144, 145, 142]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:32.497822582Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1766,"":"downstairs","":"downstairs","session_id":"80edb1fb-6898-4fcc-adb6-7c06657e4c05"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:32.497836881Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1766,"":"downstairs","":"downstairs","session_id":"80edb1fb-6898-4fcc-adb6-7c06657e4c05"} {"msg":"[1]R flush_numbers[0..12]: [265, 259, 261, 245, 266, 269, 269, 259, 261, 267, 270, 265]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:32.49785064Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1766,"":"downstairs","":"downstairs","session_id":"80edb1fb-6898-4fcc-adb6-7c06657e4c05"} {"msg":"[1]R generation[0..12]: [142, 140, 141, 135, 143, 144, 144, 140, 141, 144, 145, 142]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:32.497864519Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1766,"":"downstairs","":"downstairs","session_id":"80edb1fb-6898-4fcc-adb6-7c06657e4c05"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:32.497878458Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1766,"":"downstairs","":"downstairs","session_id":"80edb1fb-6898-4fcc-adb6-7c06657e4c05"} {"msg":"[2]R flush_numbers[0..12]: [265, 259, 261, 245, 266, 269, 269, 259, 261, 267, 270, 265]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:32.497892067Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1766,"":"downstairs","":"downstairs","session_id":"80edb1fb-6898-4fcc-adb6-7c06657e4c05"} {"msg":"[2]R generation[0..12]: [142, 140, 141, 135, 143, 144, 144, 140, 141, 144, 145, 142]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:32.497905736Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1766,"":"downstairs","":"downstairs","session_id":"80edb1fb-6898-4fcc-adb6-7c06657e4c05"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:32.497919565Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1766,"":"downstairs","":"downstairs","session_id":"80edb1fb-6898-4fcc-adb6-7c06657e4c05"} {"msg":"Max found gen is 146","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:32.497935504Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1766,"":"downstairs","":"downstairs","session_id":"80edb1fb-6898-4fcc-adb6-7c06657e4c05"} {"msg":"Generation requested: 146 >= found:146","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:32.497948963Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1766,"":"downstairs","":"downstairs","session_id":"80edb1fb-6898-4fcc-adb6-7c06657e4c05"} {"msg":"Next flush: 271","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:32.497961853Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1766,"":"downstairs","":"downstairs","session_id":"80edb1fb-6898-4fcc-adb6-7c06657e4c05"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:32.497974712Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1766,"":"downstairs","":"downstairs","session_id":"80edb1fb-6898-4fcc-adb6-7c06657e4c05"} {"msg":"No downstairs reconciliation required","v":0,"Wait for a query_work_queue command to finish before sending IO name":"crucible","level":30,"time":"2024-12-12T20:38:32.498004859Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1766,"session_id":"80edb1fb-6898-4fcc-adb6-7c06657e4c05"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:32.498017099Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1766,"client":"0","":"downstairs","session_id":"80edb1fb-6898-4fcc-adb6-7c06657e4c05"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:32.498030168Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1766,"client":"1","":"downstairs","session_id":"80edb1fb-6898-4fcc-adb6-7c06657e4c05"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:32.498043007Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1766,"client":"2","":"downstairs","session_id":"80edb1fb-6898-4fcc-adb6-7c06657e4c05"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:32.498056096Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1766,"session_id":"80edb1fb-6898-4fcc-adb6-7c06657e4c05"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:32.498068275Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1766,"session_id":"80edb1fb-6898-4fcc-adb6-7c06657e4c05"} {"msg":"1b1f9707-cbdb-493c-a2d5-52e904ce12ae is now active with session: 80edb1fb-6898-4fcc-adb6-7c06657e4c05","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:32.498080954Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1766,"session_id":"80edb1fb-6898-4fcc-adb6-7c06657e4c05"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:32.498092764Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1766,"session_id":"80edb1fb-6898-4fcc-adb6-7c06657e4c05"} {"msg":"The guest has finished waiting for activation with:146","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:32.498104813Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1766} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 170 len 40960 data: 7 7 6 7 7 10 10 11 10 12 02/30 Write block 457 len 12288 data: 14 17 17 03/30 Read block 216 len 40960 04/30 Write block 297 len 16384 data: 11 8 8 10 05/30 Read block 275 len 8192 06/30 Read block 242 len 16384 07/30 Write block 313 len 24576 data: 15 15 17 17 15 15 08/30 Read block 102 len 8192 09/30 Read block 514 len 16384 10/30 Write block 49 len 32768 data: 12 14 13 11 14 14 18 21 11/30 Read block 437 len 24576 12/30 Read block 122 len 36864 13/30 Write block 359 len 40960 data: 11 11 12 11 11 14 15 16 15 11 14/30 Write block 333 len 20480 data: 13 13 12 10 11 15/30 Read block 211 len 4096 16/30 Read block 292 len 32768 17/30 Flush 18/30 Write block 513 len 4096 data: 11 19/30 Write block 8 len 24576 data: 10 11 11 12 10 8 20/30 Flush 21/30 Write block 230 len 16384 data: 14 12 12 12 22/30 Read block 115 len 8192 23/30 Read block 144 len 24576 24/30 Read block 557 len 40960 25/30 Write block 491 len 40960 data: 10 9 9 8 7 7 8 5 5 7 26/30 Read block 105 len 24576 27/30 Read block 28 len 40960 28/30 Write block 464 len 20480 data: 16 15 13 12 10 29/30 Read block 350 len 28672 30/30 Write block 197 len 28672 data: 9 7 8 8 9 11 12 ---------------------------------------------------------------- Crucible gen:146 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1020 Acked Write 4 Done Done Done false 1021 Acked Read 2 Done Done Done false 1022 Acked Read 6 Done Done Done false 1023 Acked Read 10 Done Done Done false 1024 Acked Write 10 Done Done Done false 1025 Acked Read 6 Done Done Done false 1026 Acked Read 10 Done Done Done false 1027 Acked Write 5 Done Done Done false 1028 Acked Read 7 Done Done Done false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 9 9 9 27 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1019 1019 1019 Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:38:32.518565222Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1766} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 146 146 146 272 272 272 F F F 1 020-039 140 140 140 259 259 259 F F F 2 040-059 146 146 146 271 271 271 F F F 3 060-079 135 135 135 245 245 245 F F F 4 080-099 143 143 143 266 266 266 F F F 5 100-119 144 144 144 269 269 269 F F F 6 120-139 144 144 144 269 269 269 F F F 7 140-159 140 140 140 259 259 259 F F F 8 160-179 146 146 146 271 271 271 F F F 9 180-199 144 144 144 267 267 267 T T T 10 200-219 145 145 145 270 270 270 T T T 11 220-239 142 142 142 265 265 265 T T T 12 240-259 145 145 145 270 270 270 F F F 13 260-279 142 142 142 265 265 265 F F F 14 280-299 146 146 146 271 271 271 F F F 15 300-319 146 146 146 271 271 271 F F F 16 320-339 146 146 146 271 271 271 F F F 17 340-359 146 146 146 271 271 271 F F F 18 360-379 146 146 146 271 271 271 F F F 19 380-399 143 143 143 266 266 266 F F F 20 400-419 145 145 145 270 270 270 F F F 21 420-439 145 145 145 270 270 270 F F F 22 440-459 146 146 146 271 271 271 F F F 23 460-479 142 142 142 264 264 264 T T T 24 480-499 142 142 142 265 265 265 T T T 25 500-519 146 146 146 272 272 272 T T T 26 520-539 141 141 141 261 261 261 F F F 27 540-559 142 142 142 265 265 265 F F F 28 560-579 145 145 145 270 270 270 F F F 29 580-599 139 139 139 258 258 258 F F F Max gen: 146, Max flush: 272 A Difference in extent metadata is expected here On loop 73 Verifying data now No change after verify Loop: 73 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 146 146 146 272 272 272 F F F 1 020-039 140 140 140 259 259 259 F F F 2 040-059 146 146 146 271 271 271 F F F 3 060-079 135 135 135 245 245 245 F F F 4 080-099 143 143 143 266 266 266 F F F 5 100-119 144 144 144 269 269 269 F F F 6 120-139 144 144 144 269 269 269 F F F 7 140-159 140 140 140 259 259 259 F F F 8 160-179 146 146 146 271 271 271 F F F 9 180-199 147 147 147 273 273 273 F F F 10 200-219 147 147 147 273 273 273 F F F 11 220-239 147 147 147 273 273 273 F F F 12 240-259 145 145 145 270 270 270 F F F 13 260-279 142 142 142 265 265 265 F F F 14 280-299 146 146 146 271 271 271 F F F 15 300-319 146 146 146 271 271 271 F F F 16 320-339 146 146 146 271 271 271 F F F 17 340-359 146 146 146 271 271 271 F F F 18 360-379 146 146 146 271 271 271 F F F 19 380-399 143 143 143 266 266 266 F F F 20 400-419 145 145 145 270 270 270 F F F 21 420-439 145 145 145 270 270 270 F F F 22 440-459 146 146 146 271 271 271 F F F 23 460-479 147 147 147 273 273 273 F F F 24 480-499 147 147 147 273 273 273 F F F 25 500-519 147 147 147 273 273 273 F F F 26 520-539 141 141 141 261 261 261 F F F 27 540-559 142 142 142 265 265 265 F F F 28 560-579 145 145 145 270 270 270 F F F 29 580-599 139 139 139 258 258 258 F F F Max gen: 147, Max flush: 273 Begin loop 74 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:36.793532637Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1774,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:36.838261843Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1774,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:36.838936946Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1774} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"26e7d501b33f47379e53601720353c2983e7654b\",\n git_commit_timestamp: \"2024-12-12T20:16:40.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:36.838972144Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1774} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:36.838986153Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1774} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:36.839000292Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1774} {"msg":"Crucible 353fc744-e3b4-4bd0-ba5a-f9398e5fe541 has session id: fd07bbc0-80a7-4d1a-aeda-9784025d8c03","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:36.839419613Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1774,"session_id":"fd07bbc0-80a7-4d1a-aeda-9784025d8c03"} {"msg":"Upstairs opts: Upstairs UUID: 353fc744-e3b4-4bd0-ba5a-f9398e5fe541, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:36.83946409Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1774,"session_id":"fd07bbc0-80a7-4d1a-aeda-9784025d8c03"} {"msg":"Crucible stats registered with UUID: 353fc744-e3b4-4bd0-ba5a-f9398e5fe541","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:36.839492318Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1774,"session_id":"fd07bbc0-80a7-4d1a-aeda-9784025d8c03"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:36.870955958Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1774,"task":"crutest"} {"msg":"The guest has requested activation with gen:148","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:36.870991006Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1774} {"msg":"353fc744-e3b4-4bd0-ba5a-f9398e5fe541 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:36.871061281Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1774,"session_id":"fd07bbc0-80a7-4d1a-aeda-9784025d8c03"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:36.8710844Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1774,"client":"0","":"downstairs","session_id":"fd07bbc0-80a7-4d1a-aeda-9784025d8c03"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:36.871114127Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1774,"client":"0","":"downstairs","session_id":"fd07bbc0-80a7-4d1a-aeda-9784025d8c03"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:36.871131946Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1774,"client":"1","":"downstairs","session_id":"fd07bbc0-80a7-4d1a-aeda-9784025d8c03"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:36.871149445Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1774,"client":"1","":"downstairs","session_id":"fd07bbc0-80a7-4d1a-aeda-9784025d8c03"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:36.871165784Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1774,"client":"2","":"downstairs","session_id":"fd07bbc0-80a7-4d1a-aeda-9784025d8c03"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:36.871197152Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1774,"client":"2","":"downstairs","session_id":"fd07bbc0-80a7-4d1a-aeda-9784025d8c03"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:36.87122688Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1774,"":"io task","client":"0","":"downstairs","session_id":"fd07bbc0-80a7-4d1a-aeda-9784025d8c03"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:36.871254528Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1774,"":"io task","client":"2","":"downstairs","session_id":"fd07bbc0-80a7-4d1a-aeda-9784025d8c03"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:36.871282466Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1774,"":"io task","client":"1","":"downstairs","session_id":"fd07bbc0-80a7-4d1a-aeda-9784025d8c03"} {"msg":"ds_connection connected from Ok(127.0.0.1:58016)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:36.871309604Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1774,"":"io task","client":"0","":"downstairs","session_id":"fd07bbc0-80a7-4d1a-aeda-9784025d8c03"} {"msg":"ds_connection connected from Ok(127.0.0.1:57727)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:36.871336112Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1774,"":"io task","client":"2","":"downstairs","session_id":"fd07bbc0-80a7-4d1a-aeda-9784025d8c03"} {"msg":"ds_connection connected from Ok(127.0.0.1:56063)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:36.87136934Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1774,"":"io task","client":"1","":"downstairs","session_id":"fd07bbc0-80a7-4d1a-aeda-9784025d8c03"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:36.872087511Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1774,"client":"1","":"downstairs","session_id":"fd07bbc0-80a7-4d1a-aeda-9784025d8c03"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:36.872116918Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1774,"client":"2","":"downstairs","session_id":"fd07bbc0-80a7-4d1a-aeda-9784025d8c03"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:36.872131868Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1774,"client":"0","":"downstairs","session_id":"fd07bbc0-80a7-4d1a-aeda-9784025d8c03"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:36.87267667Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1774,"client":"1","":"downstairs","session_id":"fd07bbc0-80a7-4d1a-aeda-9784025d8c03"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:36.872807111Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1774,"client":"2","":"downstairs","session_id":"fd07bbc0-80a7-4d1a-aeda-9784025d8c03"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:36.872942081Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1774,"client":"0","":"downstairs","session_id":"fd07bbc0-80a7-4d1a-aeda-9784025d8c03"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:36.873206323Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1774,"session_id":"fd07bbc0-80a7-4d1a-aeda-9784025d8c03"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:36.873318606Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1774,"session_id":"fd07bbc0-80a7-4d1a-aeda-9784025d8c03"} {"msg":"[0]R flush_numbers[0..12]: [272, 259, 271, 245, 266, 269, 269, 259, 271, 273, 273, 273]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:36.87353698Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1774,"":"downstairs","":"downstairs","session_id":"fd07bbc0-80a7-4d1a-aeda-9784025d8c03"} {"msg":"[0]R generation[0..12]: [146, 140, 146, 135, 143, 144, 144, 140, 146, 147, 147, 147]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:36.873556659Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1774,"":"downstairs","":"downstairs","session_id":"fd07bbc0-80a7-4d1a-aeda-9784025d8c03"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:36.873571268Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1774,"":"downstairs","":"downstairs","session_id":"fd07bbc0-80a7-4d1a-aeda-9784025d8c03"} {"msg":"[1]R flush_numbers[0..12]: [272, 259, 271, 245, 266, 269, 269, 259, 271, 273, 273, 273]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:36.873585267Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1774,"":"downstairs","":"downstairs","session_id":"fd07bbc0-80a7-4d1a-aeda-9784025d8c03"} {"msg":"[1]R generation[0..12]: [146, 140, 146, 135, 143, 144, 144, 140, 146, 147, 147, 147]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:36.873598866Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1774,"":"downstairs","":"downstairs","session_id":"fd07bbc0-80a7-4d1a-aeda-9784025d8c03"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:36.873616675Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1774,"":"downstairs","":"downstairs","session_id":"fd07bbc0-80a7-4d1a-aeda-9784025d8c03"} {"msg":"[2]R flush_numbers[0..12]: [272, 259, 271, 245, 266, 269, 269, 259, 271, 273, 273, 273]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:36.873630884Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1774,"":"downstairs","":"downstairs","session_id":"fd07bbc0-80a7-4d1a-aeda-9784025d8c03"} {"msg":"[2]R generation[0..12]: [146, 140, 146, 135, 143, 144, 144, 140, 146, 147, 147, 147]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:36.873644533Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1774,"":"downstairs","":"downstairs","session_id":"fd07bbc0-80a7-4d1a-aeda-9784025d8c03"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:36.873658302Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1774,"":"downstairs","":"downstairs","session_id":"fd07bbc0-80a7-4d1a-aeda-9784025d8c03"} {"msg":"Max found gen is 148","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:36.873671121Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1774,"":"downstairs","":"downstairs","session_id":"fd07bbc0-80a7-4d1a-aeda-9784025d8c03"} {"msg":"Generation requested: 148 >= found:148","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:36.87368424Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1774,"":"downstairs","":"downstairs","session_id":"fd07bbc0-80a7-4d1a-aeda-9784025d8c03"} {"msg":"Next flush: 274","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:36.873697209Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1774,"":"downstairs","":"downstairs","session_id":"fd07bbc0-80a7-4d1a-aeda-9784025d8c03"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:36.873709968Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1774,"":"downstairs","":"downstairs","session_id":"fd07bbc0-80a7-4d1a-aeda-9784025d8c03"} {"msg":"Wait for a query_work_queue command to finish before sending IO No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:36.873728977Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1774,"session_id":"fd07bbc0-80a7-4d1a-aeda-9784025d8c03"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:36.873751426Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1774,"client":"0","":"downstairs","session_id":"fd07bbc0-80a7-4d1a-aeda-9784025d8c03"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:36.873777484Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1774,"client":"1","":"downstairs","session_id":"fd07bbc0-80a7-4d1a-aeda-9784025d8c03"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:36.873802872Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1774,"client":"2","":"downstairs","session_id":"fd07bbc0-80a7-4d1a-aeda-9784025d8c03"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:36.873821591Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1774,"session_id":"fd07bbc0-80a7-4d1a-aeda-9784025d8c03"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:36.87383396Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1774,"session_id":"fd07bbc0-80a7-4d1a-aeda-9784025d8c03"} {"msg":"353fc744-e3b4-4bd0-ba5a-f9398e5fe541 is now active with session: fd07bbc0-80a7-4d1a-aeda-9784025d8c03","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:36.873846809Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1774,"session_id":"fd07bbc0-80a7-4d1a-aeda-9784025d8c03"} {"msg":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:36.873865708Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1774,"session_id":"fd07bbc0-80a7-4d1a-aeda-9784025d8c03"} {"msg":"The guest has finished waiting for activation with:148","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:36.873878597Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1774} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 140 len 20480 data: 7 7 10 11 11 02/30 Write block 331 len 40960 data: 15 13 14 14 13 11 12 11 9 11 03/30 Read block 351 len 12288 04/30 Flush 05/30 Write block 123 len 4096 data: 11 06/30 Read block 2 len 16384 07/30 Write block 464 len 4096 data: 17 08/30 Read block 204 len 20480 09/30 Read block 175 len 24576 10/30 Write block 454 len 28672 data: 13 12 13 15 18 18 16 11/30 Read block 435 len 24576 12/30 Read block 540 len 4096 13/30 Write block 344 len 36864 data: 16 15 15 14 12 11 14 10 14 14/30 Read block 529 len 36864 15/30 Read block 154 len 16384 16/30 Write block 403 len 4096 data: 13 17/30 Read block 154 len 4096 18/30 Write block 244 len 32768 data: 9 8 9 7 7 7 7 9 19/30 Write block 188 len 36864 data: 14 14 13 13 14 15 12 13 12 20/30 Write block 80 len 4096 data: 11 21/30 Read block 569 len 40960 22/30 Read block 378 len 4096 23/30 Flush 24/30 Write block 581 len 16384 data: 11 12 14 16 25/30 Flush 26/30 Write block 285 len 16384 data: 9 9 8 8 27/30 Read block 384 len 24576 28/30 Read block 197 len 4096 29/30 Write block 409 len 36864 data: 15 13 10 8 9 11 10 11 9 30/30 Read block 2 len 36864 ---------------------------------------------------------------- Crucible gen:148 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 4 Done Done Done false 1026 Acked Read 6 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Write 9 Sent Done Done false 1029 Acked Read 9 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 0 0 2 Done 3 5 5 13 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:38:36.892939233Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1774} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 146 146 146 272 272 272 F F F 1 020-039 140 140 140 259 259 259 F F F 2 040-059 146 146 146 271 271 271 F F F 3 060-079 135 135 135 245 245 245 F F F 4 080-099 148 148 148 275 275 275 F F F 5 100-119 144 144 144 269 269 269 F F F 6 120-139 148 148 148 275 275 275 F F F 7 140-159 148 148 148 274 274 274 F F F 8 160-179 146 146 146 271 271 271 F F F 9 180-199 148 148 148 275 275 275 F F F 10 200-219 147 147 147 273 273 273 F F F 11 220-239 147 147 147 273 273 273 F F F 12 240-259 148 148 148 275 275 275 F F F 13 260-279 142 142 142 265 265 265 F F F 14 280-299 146 146 146 271 271 271 T T T 15 300-319 146 146 146 271 271 271 F F F 16 320-339 148 148 148 274 274 274 F F F 17 340-359 148 148 148 275 275 275 F F F 18 360-379 146 146 146 271 271 271 F F F 19 380-399 143 143 143 266 266 266 F F F 20 400-419 148 148 148 275 275 275 T T T 21 420-439 145 145 145 270 270 270 F F F 22 440-459 148 148 148 275 275 275 F F F 23 460-479 148 148 148 275 275 275 F F F 24 480-499 147 147 147 273 273 273 F F F 25 500-519 147 147 147 273 273 273 F F F 26 520-539 141 141 141 261 261 261 F F F 27 540-559 142 142 142 265 265 265 F F F 28 560-579 145 145 145 270 270 270 F F F 29 580-599 148 148 148 276 276 276 F F F Max gen: 148, Max flush: 276 A Difference in extent metadata is expected here On loop 74 Verifying data now No change after verify Loop: 74 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 146 146 146 272 272 272 F F F 1 020-039 140 140 140 259 259 259 F F F 2 040-059 146 146 146 271 271 271 F F F 3 060-079 135 135 135 245 245 245 F F F 4 080-099 148 148 148 275 275 275 F F F 5 100-119 144 144 144 269 269 269 F F F 6 120-139 148 148 148 275 275 275 F F F 7 140-159 148 148 148 274 274 274 F F F 8 160-179 146 146 146 271 271 271 F F F 9 180-199 148 148 148 275 275 275 F F F 10 200-219 147 147 147 273 273 273 F F F 11 220-239 147 147 147 273 273 273 F F F 12 240-259 148 148 148 275 275 275 F F F 13 260-279 142 142 142 265 265 265 F F F 14 280-299 149 149 149 277 277 277 F F F 15 300-319 146 146 146 271 271 271 F F F 16 320-339 148 148 148 274 274 274 F F F 17 340-359 148 148 148 275 275 275 F F F 18 360-379 146 146 146 271 271 271 F F F 19 380-399 143 143 143 266 266 266 F F F 20 400-419 149 149 149 277 277 277 F F F 21 420-439 145 145 145 270 270 270 F F F 22 440-459 148 148 148 275 275 275 F F F 23 460-479 148 148 148 275 275 275 F F F 24 480-499 147 147 147 273 273 273 F F F 25 500-519 147 147 147 273 273 273 F F F 26 520-539 141 141 141 261 261 261 F F F 27 540-559 142 142 142 265 265 265 F F F 28 560-579 145 145 145 270 270 270 F F F 29 580-599 148 148 148 276 276 276 F F F Max gen: 149, Max flush: 277 Begin loop 75 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:41.153525969Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1782,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:41.197849383Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1782,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:41.198577053Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1782} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"26e7d501b33f47379e53601720353c2983e7654b\",\n git_commit_timestamp: \"2024-12-12T20:16:40.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:41.19861437Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1782} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:41.198628599Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1782} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:41.198647218Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1782} {"msg":"Crucible dc81e1e7-370a-4d02-957b-f566fbfe2c43 has session id: dc8aa3d1-bebc-449f-88c7-fdecb7566053","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:41.199065489Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1782,"session_id":"dc8aa3d1-bebc-449f-88c7-fdecb7566053"} {"msg":"Upstairs opts: Upstairs UUID: dc81e1e7-370a-4d02-957b-f566fbfe2c43, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:41.199114446Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1782,"session_id":"dc8aa3d1-bebc-449f-88c7-fdecb7566053"} {"msg":"Crucible stats registered with UUID: dc81e1e7-370a-4d02-957b-f566fbfe2c43","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:41.199131575Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1782,"session_id":"dc8aa3d1-bebc-449f-88c7-fdecb7566053"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:41.229768862Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1782,"task":"crutest"} {"msg":"The guest has requested activation with gen:150","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:41.22979708Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1782} {"msg":"dc81e1e7-370a-4d02-957b-f566fbfe2c43 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:41.229854106Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1782,"session_id":"dc8aa3d1-bebc-449f-88c7-fdecb7566053"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:41.229881914Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1782,"client":"0","":"downstairs","session_id":"dc8aa3d1-bebc-449f-88c7-fdecb7566053"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:41.229907152Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1782,"client":"0","":"downstairs","session_id":"dc8aa3d1-bebc-449f-88c7-fdecb7566053"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:41.229922031Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1782,"client":"1","":"downstairs","session_id":"dc8aa3d1-bebc-449f-88c7-fdecb7566053"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:41.22993635Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1782,"client":"1","":"downstairs","session_id":"dc8aa3d1-bebc-449f-88c7-fdecb7566053"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:41.229949939Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1782,"client":"2","":"downstairs","session_id":"dc8aa3d1-bebc-449f-88c7-fdecb7566053"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:41.229963938Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1782,"client":"2","":"downstairs","session_id":"dc8aa3d1-bebc-449f-88c7-fdecb7566053"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:41.229979297Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1782,"":"io task","client":"2","":"downstairs","session_id":"dc8aa3d1-bebc-449f-88c7-fdecb7566053"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:41.230010125Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1782,"":"io task","client":"0","":"downstairs","session_id":"dc8aa3d1-bebc-449f-88c7-fdecb7566053"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:41.230038353Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1782,"":"io task","client":"1","":"downstairs","session_id":"dc8aa3d1-bebc-449f-88c7-fdecb7566053"} {"msg":"ds_connection connected from Ok(127.0.0.1:45991)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:41.230068521Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1782,"":"io task","client":"0","":"downstairs","session_id":"dc8aa3d1-bebc-449f-88c7-fdecb7566053"} {"msg":"ds_connection connected from Ok(127.0.0.1:58717)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:41.230344072Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1782,"":"io task","client":"2","":"downstairs","session_id":"dc8aa3d1-bebc-449f-88c7-fdecb7566053"} {"msg":"ds_connection connected from Ok(127.0.0.1:54004)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:41.230463484Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1782,"":"io task","client":"1","":"downstairs","session_id":"dc8aa3d1-bebc-449f-88c7-fdecb7566053"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:41.230738265Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1782,"client":"0","":"downstairs","session_id":"dc8aa3d1-bebc-449f-88c7-fdecb7566053"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:41.230914383Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1782,"client":"1","":"downstairs","session_id":"dc8aa3d1-bebc-449f-88c7-fdecb7566053"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:41.230965819Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1782,"client":"2","":"downstairs","session_id":"dc8aa3d1-bebc-449f-88c7-fdecb7566053"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:41.231469225Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1782,"client":"0","":"downstairs","session_id":"dc8aa3d1-bebc-449f-88c7-fdecb7566053"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:41.231556969Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1782,"client":"1","":"downstairs","session_id":"dc8aa3d1-bebc-449f-88c7-fdecb7566053"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:41.231632534Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1782,"client":"2","":"downstairs","session_id":"dc8aa3d1-bebc-449f-88c7-fdecb7566053"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:41.232094611Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1782,"session_id":"dc8aa3d1-bebc-449f-88c7-fdecb7566053"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:41.23212053Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1782,"session_id":"dc8aa3d1-bebc-449f-88c7-fdecb7566053"} {"msg":"[0]R flush_numbers[0..12]: [272, 259, 271, 245, 275, 269, 275, 274, 271, 275, 273, 273]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:41.232210643Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1782,"":"downstairs","":"downstairs","session_id":"dc8aa3d1-bebc-449f-88c7-fdecb7566053"} {"msg":"[0]R generation[0..12]: [146, 140, 146, 135, 148, 144, 148, 148, 146, 148, 147, 147]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:41.232230362Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1782,"":"downstairs","":"downstairs","session_id":"dc8aa3d1-bebc-449f-88c7-fdecb7566053"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:41.232247961Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1782,"":"downstairs","":"downstairs","session_id":"dc8aa3d1-bebc-449f-88c7-fdecb7566053"} {"msg":"[1]R flush_numbers[0..12]: [272, 259, 271, 245, 275, 269, 275, 274, 271, 275, 273, 273]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:41.23226214Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1782,"":"downstairs","":"downstairs","session_id":"dc8aa3d1-bebc-449f-88c7-fdecb7566053"} {"msg":"[1]R generation[0..12]: [146, 140, 146, 135, 148, 144, 148, 148, 146, 148, 147, 147]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:41.232275899Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1782,"":"downstairs","":"downstairs","session_id":"dc8aa3d1-bebc-449f-88c7-fdecb7566053"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:41.232290108Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1782,"":"downstairs","":"downstairs","session_id":"dc8aa3d1-bebc-449f-88c7-fdecb7566053"} {"msg":"[2]R flush_numbers[0..12]: [272, 259, 271, 245, 275, 269, 275, 274, 271, 275, 273, 273]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:41.232303787Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1782,"":"downstairs","":"downstairs","session_id":"dc8aa3d1-bebc-449f-88c7-fdecb7566053"} {"msg":"[2]R generation[0..12]: [146, 140, 146, 135, 148, 144, 148, 148, 146, 148, 147, 147]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:41.232317346Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1782,"":"downstairs","":"downstairs","session_id":"dc8aa3d1-bebc-449f-88c7-fdecb7566053"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:41.232331225Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1782,"":"downstairs","":"downstairs","session_id":"dc8aa3d1-bebc-449f-88c7-fdecb7566053"} {"msg":"Max found gen is 150","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:41.232344074Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1782,"":"downstairs","":"downstairs","session_id":"dc8aa3d1-bebc-449f-88c7-fdecb7566053"} {"msg":"Generation requested: 150 >= found:150","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:41.232357104Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1782,"":"downstairs","":"downstairs","session_id":"dc8aa3d1-bebc-449f-88c7-fdecb7566053"} {"msg":"Next flush: 278","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:41.232370032Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1782,"":"downstairs","":"downstairs","session_id":"dc8aa3d1-bebc-449f-88c7-fdecb7566053"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:41.232382892Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1782,"":"downstairs","":"downstairs","session_id":"dc8aa3d1-bebc-449f-88c7-fdecb7566053"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:41.232395821Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1782,"session_id":"dc8aa3d1-bebc-449f-88c7-fdecb7566053"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:41.23240782Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1782,"client":"0","":"downstairs","session_id":"dc8aa3d1-bebc-449f-88c7-fdecb7566053"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:41.232420899Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1782,"client":"1","":"downstairs","session_id":"dc8aa3d1-bebc-449f-88c7-fdecb7566053"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:41.232435518Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1782,"client":"2","":"downstairs","session_id":"dc8aa3d1-bebc-449f-88c7-fdecb7566053"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:41.232448987Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1782,"session_id":"dc8aa3d1-bebc-449f-88c7-fdecb7566053"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:41.232461176Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1782,"session_id":"dc8aa3d1-bebc-449f-88c7-fdecb7566053"} {"msg":"dc81e1e7-370a-4d02-957b-f566fbfe2c43 is now active with session: dc8aa3d1-bebc-449f-88c7-fdecb7566053","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:41.232473825Z","hostname"Wait for a query_work_queue command to finish before sending IO :"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1782,"session_id":"dc8aa3d1-bebc-449f-88c7-fdecb7566053"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:41.232491834Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1782,"session_id":"dc8aa3d1-bebc-449f-88c7-fdecb7566053"} {"msg":"The guest has finished waiting for activation with:150","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:41.232503953Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1782} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 344 len 24576 data: 17 16 16 15 13 12 02/30 Read block 577 len 20480 03/30 Read block 570 len 4096 04/30 Write block 402 len 12288 data: 14 14 12 05/30 Read block 169 len 8192 06/30 Flush 07/30 Write block 163 len 32768 data: 11 12 10 10 7 7 8 8 08/30 Write block 390 len 24576 data: 12 11 9 9 12 14 09/30 Write block 20 len 36864 data: 10 9 11 13 16 14 12 13 13 10/30 Read block 557 len 8192 11/30 Write block 364 len 12288 data: 15 16 17 12/30 Read block 519 len 32768 13/30 Read block 253 len 36864 14/30 Read block 248 len 28672 15/30 Flush 16/30 Write block 387 len 32768 data: 13 12 10 13 12 10 10 13 17/30 Read block 96 len 4096 18/30 Read block 111 len 32768 19/30 Write block 326 len 32768 data: 9 8 9 12 12 16 14 15 20/30 Read block 150 len 40960 21/30 Read block 112 len 4096 22/30 Read block 581 len 24576 23/30 Read block 347 len 40960 24/30 Flush 25/30 Write block 521 len 12288 data: 19 17 21 26/30 Read block 118 len 12288 27/30 Write block 93 len 24576 data: 14 13 12 8 7 8 28/30 Write block 489 len 8192 data: 16 14 29/30 Read block 393 len 20480 30/30 Read block 185 len 32768 ---------------------------------------------------------------- Crucible gen:150 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 3 Done Done Done false 1025 Acked Read 3 Done Done Done false 1026 Acked Write 6 Done Done Done false 1027 Acked Write 2 Done Done Done false 1028 Acked Read 5 Done Done Done false 1029 Acked Read 8 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 6 5 5 16 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:38:41.252493675Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1782} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 146 146 146 272 272 272 F F F 1 020-039 150 150 150 279 279 279 F F F 2 040-059 146 146 146 271 271 271 F F F 3 060-079 135 135 135 245 245 245 F F F 4 080-099 148 148 148 275 275 275 T T T 5 100-119 144 144 144 269 269 269 F F F 6 120-139 148 148 148 275 275 275 F F F 7 140-159 148 148 148 274 274 274 F F F 8 160-179 150 150 150 279 279 279 F F F 9 180-199 148 148 148 275 275 275 F F F 10 200-219 147 147 147 273 273 273 F F F 11 220-239 147 147 147 273 273 273 F F F 12 240-259 148 148 148 275 275 275 F F F 13 260-279 142 142 142 265 265 265 F F F 14 280-299 149 149 149 277 277 277 F F F 15 300-319 146 146 146 271 271 271 F F F 16 320-339 150 150 150 280 280 280 F F F 17 340-359 150 150 150 278 278 278 F F F 18 360-379 150 150 150 279 279 279 F F F 19 380-399 150 150 150 280 280 280 F F F 20 400-419 150 150 150 278 278 278 F F F 21 420-439 145 145 145 270 270 270 F F F 22 440-459 148 148 148 275 275 275 F F F 23 460-479 148 148 148 275 275 275 F F F 24 480-499 147 147 147 273 273 273 T T T 25 500-519 147 147 147 273 273 273 F F F 26 520-539 141 141 141 261 261 261 T T T 27 540-559 142 142 142 265 265 265 F F F 28 560-579 145 145 145 270 270 270 F F F 29 580-599 148 148 148 276 276 276 F F F Max gen: 150, Max flush: 280 A Difference in extent metadata is expected here On loop 75 Verifying data now No change after verify Loop: 75 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 146 146 146 272 272 272 F F F 1 020-039 150 150 150 279 279 279 F F F 2 040-059 146 146 146 271 271 271 F F F 3 060-079 135 135 135 245 245 245 F F F 4 080-099 151 151 151 281 281 281 F F F 5 100-119 144 144 144 269 269 269 F F F 6 120-139 148 148 148 275 275 275 F F F 7 140-159 148 148 148 274 274 274 F F F 8 160-179 150 150 150 279 279 279 F F F 9 180-199 148 148 148 275 275 275 F F F 10 200-219 147 147 147 273 273 273 F F F 11 220-239 147 147 147 273 273 273 F F F 12 240-259 148 148 148 275 275 275 F F F 13 260-279 142 142 142 265 265 265 F F F 14 280-299 149 149 149 277 277 277 F F F 15 300-319 146 146 146 271 271 271 F F F 16 320-339 150 150 150 280 280 280 F F F 17 340-359 150 150 150 278 278 278 F F F 18 360-379 150 150 150 279 279 279 F F F 19 380-399 150 150 150 280 280 280 F F F 20 400-419 150 150 150 278 278 278 F F F 21 420-439 145 145 145 270 270 270 F F F 22 440-459 148 148 148 275 275 275 F F F 23 460-479 148 148 148 275 275 275 F F F 24 480-499 151 151 151 281 281 281 F F F 25 500-519 147 147 147 273 273 273 F F F 26 520-539 151 151 151 281 281 281 F F F 27 540-559 142 142 142 265 265 265 F F F 28 560-579 145 145 145 270 270 270 F F F 29 580-599 148 148 148 276 276 276 F F F Max gen: 151, Max flush: 281 Begin loop 76 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:45.509267525Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1790,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:38:45.552387342Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1790,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:45.552777355Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1790,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:45.584558153Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1790,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:45.585128024Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1790} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"26e7d501b33f47379e53601720353c2983e7654b\",\n git_commit_timestamp: \"2024-12-12T20:16:40.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:45.585158682Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1790} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:45.585171671Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1790} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:45.58518562Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1790} {"msg":"Crucible 2f1f8b8d-e73c-4079-b1d7-29bd95a6cade has session id: 409944c6-8264-4d00-a479-470719897ed9","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:45.585632449Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1790,"session_id":"409944c6-8264-4d00-a479-470719897ed9"} {"msg":"Upstairs opts: Upstairs UUID: 2f1f8b8d-e73c-4079-b1d7-29bd95a6cade, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:45.585675936Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1790,"session_id":"409944c6-8264-4d00-a479-470719897ed9"} {"msg":"Crucible stats registered with UUID: 2f1f8b8d-e73c-4079-b1d7-29bd95a6cade","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:45.585701235Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1790,"session_id":"409944c6-8264-4d00-a479-470719897ed9"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:45.616338712Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1790,"task":"crutest"} {"msg":"The guest has requested activation with gen:152","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:45.61636745Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1790} {"msg":"2f1f8b8d-e73c-4079-b1d7-29bd95a6cade active request set","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:45.616439605Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1790,"session_id":"409944c6-8264-4d00-a479-470719897ed9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:45.616458604Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1790,"client":"0","":"downstairs","session_id":"409944c6-8264-4d00-a479-470719897ed9"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:45.616482282Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1790,"client":"0","":"downstairs","session_id":"409944c6-8264-4d00-a479-470719897ed9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:45.616500391Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1790,"client":"1","":"downstairs","session_id":"409944c6-8264-4d00-a479-470719897ed9"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:45.61651523Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1790,"client":"1","":"downstairs","session_id":"409944c6-8264-4d00-a479-470719897ed9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:45.616528649Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1790,"client":"2","":"downstairs","session_id":"409944c6-8264-4d00-a479-470719897ed9"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:45.616542578Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1790,"client":"2","":"downstairs","session_id":"409944c6-8264-4d00-a479-470719897ed9"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:45.616565906Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1790,"":"io task","client":"2","":"downstairs","session_id":"409944c6-8264-4d00-a479-470719897ed9"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:45.616593134Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1790,"":"io task","client":"0","":"downstairs","session_id":"409944c6-8264-4d00-a479-470719897ed9"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:45.616619052Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1790,"":"io task","client":"1","":"downstairs","session_id":"409944c6-8264-4d00-a479-470719897ed9"} {"msg":"ds_connection connected from Ok(127.0.0.1:43266)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:45.616642441Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1790,"":"io task","client":"0","":"downstairs","session_id":"409944c6-8264-4d00-a479-470719897ed9"} {"msg":"ds_connection connected from Ok(127.0.0.1:40538)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:45.61665735Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1790,"":"io task","client":"2","":"downstairs","session_id":"409944c6-8264-4d00-a479-470719897ed9"} {"msg":"ds_connection connected from Ok(127.0.0.1:38418)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:45.616838137Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1790,"":"io task","client":"1","":"downstairs","session_id":"409944c6-8264-4d00-a479-470719897ed9"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:45.61738189Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1790,"client":"1","":"downstairs","session_id":"409944c6-8264-4d00-a479-470719897ed9"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:45.617537689Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1790,"client":"0","":"downstairs","session_id":"409944c6-8264-4d00-a479-470719897ed9"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:45.617576817Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1790,"client":"2","":"downstairs","session_id":"409944c6-8264-4d00-a479-470719897ed9"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:45.618078532Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1790,"client":"1","":"downstairs","session_id":"409944c6-8264-4d00-a479-470719897ed9"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:45.61811197Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1790,"client":"2","":"downstairs","session_id":"409944c6-8264-4d00-a479-470719897ed9"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:45.618360032Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1790,"client":"0","":"downstairs","session_id":"409944c6-8264-4d00-a479-470719897ed9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:45.618635093Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1790,"session_id":"409944c6-8264-4d00-a479-470719897ed9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:45.618740566Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1790,"session_id":"409944c6-8264-4d00-a479-470719897ed9"} {"msg":"[0]R flush_numbers[0..12]: [272, 279, 271, 245, 281, 269, 275, 274, 279, 275, 273, 273]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:45.618864668Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1790,"":"downstairs","":"downstairs","session_id":"409944c6-8264-4d00-a479-470719897ed9"} {"msg":"[0]R generation[0..12]: [146, 150, 146, 135, 151, 144, 148, 148, 150, 148, 147, 147]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:45.618885256Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1790,"":"downstairs","":"downstairs","session_id":"409944c6-8264-4d00-a479-470719897ed9"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:45.618900035Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1790,"":"downstairs","":"downstairs","session_id":"409944c6-8264-4d00-a479-470719897ed9"} {"msg":"[1]R flush_numbers[0..12]: [272, 279, 271, 245, 281, 269, 275, 274, 279, 275, 273, 273]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:45.618913934Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1790,"":"downstairs","":"downstairs","session_id":"409944c6-8264-4d00-a479-470719897ed9"} {"msg":"[1]R generation[0..12]: [146, 150, 146, 135, 151, 144, 148, 148, 150, 148, 147, 147]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:45.618927633Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1790,"":"downstairs","":"downstairs","session_id":"409944c6-8264-4d00-a479-470719897ed9"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:45.618942082Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1790,"":"downstairs","":"downstairs","session_id":"409944c6-8264-4d00-a479-470719897ed9"} {"msg":"[2]R flush_numbers[0..12]: [272, 279, 271, 245, 281, 269, 275, 274, 279, 275, 273, 273]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:45.618956151Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1790,"":"downstairs","":"downstairs","session_id":"409944c6-8264-4d00-a479-470719897ed9"} {"msg":"[2]R generation[0..12]: [146, 150, 146, 135, 151, 144, 148, 148, 150, 148, 147, 147]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:45.618969721Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1790,"":"downstairs","":"downstairs","session_id":"409944c6-8264-4d00-a479-470719897ed9"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:45.6189834Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1790,"":"downstairs","":"downstairs","session_id":"409944c6-8264-4d00-a479-470719897ed9"} {"msg":"Max found gen is 152","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:45.618996469Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1790,"":"downstairs","":"downstairs","session_id":"409944c6-8264-4d00-a479-470719897ed9"} {"msg":"Generation requested: 152 >= found:152","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:45.619013758Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1790,"":"downstairs","":"downstairs","session_id":"409944c6-8264-4d00-a479-470719897ed9Wait for a query_work_queue command to finish before sending IO "} {"msg":"Next flush: 282","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:45.619079283Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1790,"":"downstairs","":"downstairs","session_id":"409944c6-8264-4d00-a479-470719897ed9"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:45.619095782Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1790,"":"downstairs","":"downstairs","session_id":"409944c6-8264-4d00-a479-470719897ed9"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:45.619109111Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1790,"session_id":"409944c6-8264-4d00-a479-470719897ed9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:45.61912102Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1790,"client":"0","":"downstairs","session_id":"409944c6-8264-4d00-a479-470719897ed9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:45.619134509Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1790,"client":"1","":"downstairs","session_id":"409944c6-8264-4d00-a479-470719897ed9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:45.619147498Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1790,"client":"2","":"downstairs","session_id":"409944c6-8264-4d00-a479-470719897ed9"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:45.619160547Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1790,"session_id":"409944c6-8264-4d00-a479-470719897ed9"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:45.619172887Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1790,"session_id":"409944c6-8264-4d00-a479-470719897ed9"} {"msg":"2f1f8b8d-e73c-4079-b1d7-29bd95a6cade is now active with session: 409944c6-8264-4d00-a479-470719897ed9","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:45.619185706Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1790,"session_id":"409944c6-8264-4d00-a479-470719897ed9"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:45.619197525Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1790,"session_id":"409944c6-8264-4d00-a479-470719897ed9"} {"msg":"The guest has finished waiting for activation with:152","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:45.619209594Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1790} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 484 len 36864 data: 12 15 14 14 15 17 15 11 10 02/30 Read block 355 len 4096 03/30 Write block 379 len 20480 data: 14 14 14 16 14 04/30 Flush 05/30 Write block 280 len 32768 data: 9 11 10 9 9 10 10 9 06/30 Read block 110 len 4096 07/30 Read block 224 len 36864 08/30 Write block 9 len 40960 data: 12 12 13 11 9 10 11 9 9 10 09/30 Read block 85 len 40960 10/30 Read block 95 len 36864 11/30 Write block 468 len 36864 data: 11 11 11 9 7 9 9 10 10 12/30 Read block 285 len 4096 13/30 Read block 61 len 28672 14/30 Read block 192 len 8192 15/30 Write block 8 len 20480 data: 11 13 13 14 12 16/30 Read block 85 len 36864 17/30 Read block 141 len 4096 18/30 Read block 532 len 32768 19/30 Read block 51 len 32768 20/30 Read block 213 len 40960 21/30 Read block 139 len 4096 22/30 Read block 204 len 4096 23/30 Write block 227 len 36864 data: 11 12 14 15 13 13 13 12 12 24/30 Read block 229 len 16384 25/30 Flush 26/30 Write block 260 len 36864 data: 13 9 10 12 12 15 18 19 18 27/30 Write block 263 len 24576 data: 13 13 16 19 20 19 28/30 Write block 558 len 36864 data: 8 11 7 7 9 9 8 8 13 29/30 Write block 350 len 36864 data: 15 11 15 13 16 16 15 14 11 30/30 Write block 23 len 4096 data: 14 ---------------------------------------------------------------- Crucible gen:152 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 9 Done Done Done false 1026 Acked Write 6 Done Done Done false 1027 Acked Write 9 Sent Sent Sent false 1028 Acked Write 9 Sent Sent Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 3 3 9 Done 2 2 2 6 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:38:45.639824232Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1790} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 152 152 152 283 283 283 F F F 1 020-039 150 150 150 279 279 279 T T T 2 040-059 146 146 146 271 271 271 F F F 3 060-079 135 135 135 245 245 245 F F F 4 080-099 151 151 151 281 281 281 F F F 5 100-119 144 144 144 269 269 269 F F F 6 120-139 148 148 148 275 275 275 F F F 7 140-159 148 148 148 274 274 274 F F F 8 160-179 150 150 150 279 279 279 F F F 9 180-199 148 148 148 275 275 275 F F F 10 200-219 147 147 147 273 273 273 F F F 11 220-239 152 152 152 283 283 283 F F F 12 240-259 148 148 148 275 275 275 F F F 13 260-279 142 142 142 265 265 265 T T T 14 280-299 152 152 152 283 283 283 F F F 15 300-319 146 146 146 271 271 271 F F F 16 320-339 150 150 150 280 280 280 F F F 17 340-359 150 150 150 278 278 278 T T T 18 360-379 152 152 152 282 282 282 F F F 19 380-399 152 152 152 282 282 282 F F F 20 400-419 150 150 150 278 278 278 F F F 21 420-439 145 145 145 270 270 270 F F F 22 440-459 148 148 148 275 275 275 F F F 23 460-479 152 152 152 283 283 283 F F F 24 480-499 152 152 152 282 282 282 F F F 25 500-519 147 147 147 273 273 273 F F F 26 520-539 151 151 151 281 281 281 F F F 27 540-559 142 142 142 265 265 265 T T T 28 560-579 145 145 145 270 270 270 T T T 29 580-599 148 148 148 276 276 276 F F F Max gen: 152, Max flush: 283 A Difference in extent metadata is expected here On loop 76 Verifying data now No change after verify Loop: 76 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 152 152 152 283 283 283 F F F 1 020-039 153 153 153 284 284 284 F F F 2 040-059 146 146 146 271 271 271 F F F 3 060-079 135 135 135 245 245 245 F F F 4 080-099 151 151 151 281 281 281 F F F 5 100-119 144 144 144 269 269 269 F F F 6 120-139 148 148 148 275 275 275 F F F 7 140-159 148 148 148 274 274 274 F F F 8 160-179 150 150 150 279 279 279 F F F 9 180-199 148 148 148 275 275 275 F F F 10 200-219 147 147 147 273 273 273 F F F 11 220-239 152 152 152 283 283 283 F F F 12 240-259 148 148 148 275 275 275 F F F 13 260-279 153 153 153 284 284 284 F F F 14 280-299 152 152 152 283 283 283 F F F 15 300-319 146 146 146 271 271 271 F F F 16 320-339 150 150 150 280 280 280 F F F 17 340-359 153 153 153 284 284 284 F F F 18 360-379 152 152 152 282 282 282 F F F 19 380-399 152 152 152 282 282 282 F F F 20 400-419 150 150 150 278 278 278 F F F 21 420-439 145 145 145 270 270 270 F F F 22 440-459 148 148 148 275 275 275 F F F 23 460-479 152 152 152 283 283 283 F F F 24 480-499 152 152 152 282 282 282 F F F 25 500-519 147 147 147 273 273 273 F F F 26 520-539 151 151 151 281 281 281 F F F 27 540-559 153 153 153 284 284 284 F F F 28 560-579 153 153 153 284 284 284 F F F 29 580-599 148 148 148 276 276 276 F F F Max gen: 153, Max flush: 284 Begin loop 77 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:49.94070792Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1798,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:49.985619194Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1798,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:49.986268039Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1798} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"26e7d501b33f47379e53601720353c2983e7654b\",\n git_commit_timestamp: \"2024-12-12T20:16:40.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:49.986305006Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1798} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:49.986319605Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1798} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:49.986334444Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1798} {"msg":"Crucible d718640a-bc77-404f-83e0-22d3feb72f0a has session id: f0724437-66d0-422b-8cc4-f1e6473a84ed","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:49.986778844Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1798,"session_id":"f0724437-66d0-422b-8cc4-f1e6473a84ed"} {"msg":"Upstairs opts: Upstairs UUID: d718640a-bc77-404f-83e0-22d3feb72f0a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:49.986820711Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1798,"session_id":"f0724437-66d0-422b-8cc4-f1e6473a84ed"} {"msg":"Crucible stats registered with UUID: d718640a-bc77-404f-83e0-22d3feb72f0a","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:49.98683744Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1798,"session_id":"f0724437-66d0-422b-8cc4-f1e6473a84ed"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:50.018423942Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1798,"task":"crutest"} {"msg":"The guest has requested activation with gen:154","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:50.018467419Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1798} {"msg":"d718640a-bc77-404f-83e0-22d3feb72f0a active request set","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:50.018537524Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1798,"session_id":"f0724437-66d0-422b-8cc4-f1e6473a84ed"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:50.018572131Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1798,"client":"0","":"downstairs","session_id":"f0724437-66d0-422b-8cc4-f1e6473a84ed"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:50.018604109Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1798,"client":"0","":"downstairs","session_id":"f0724437-66d0-422b-8cc4-f1e6473a84ed"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:50.018636057Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1798,"client":"1","":"downstairs","session_id":"f0724437-66d0-422b-8cc4-f1e6473a84ed"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:50.018670065Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1798,"client":"1","":"downstairs","session_id":"f0724437-66d0-422b-8cc4-f1e6473a84ed"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:50.018701562Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1798,"client":"2","":"downstairs","session_id":"f0724437-66d0-422b-8cc4-f1e6473a84ed"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:50.01873613Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1798,"client":"2","":"downstairs","session_id":"f0724437-66d0-422b-8cc4-f1e6473a84ed"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:50.018769668Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1798,"":"io task","client":"0","":"downstairs","session_id":"f0724437-66d0-422b-8cc4-f1e6473a84ed"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:50.018804905Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1798,"":"io task","client":"2","":"downstairs","session_id":"f0724437-66d0-422b-8cc4-f1e6473a84ed"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:50.018838953Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1798,"":"io task","client":"1","":"downstairs","session_id":"f0724437-66d0-422b-8cc4-f1e6473a84ed"} {"msg":"ds_connection connected from Ok(127.0.0.1:62874)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:50.018875601Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1798,"":"io task","client":"1","":"downstairs","session_id":"f0724437-66d0-422b-8cc4-f1e6473a84ed"} {"msg":"ds_connection connected from Ok(127.0.0.1:50443)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:50.018971734Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1798,"":"io task","client":"2","":"downstairs","session_id":"f0724437-66d0-422b-8cc4-f1e6473a84ed"} {"msg":"ds_connection connected from Ok(127.0.0.1:62546)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:50.019014651Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1798,"":"io task","client":"0","":"downstairs","session_id":"f0724437-66d0-422b-8cc4-f1e6473a84ed"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:50.019482959Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1798,"client":"0","":"downstairs","session_id":"f0724437-66d0-422b-8cc4-f1e6473a84ed"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:50.019526785Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1798,"client":"1","":"downstairs","session_id":"f0724437-66d0-422b-8cc4-f1e6473a84ed"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:50.019686694Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1798,"client":"2","":"downstairs","session_id":"f0724437-66d0-422b-8cc4-f1e6473a84ed"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:50.020079958Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1798,"client":"1","":"downstairs","session_id":"f0724437-66d0-422b-8cc4-f1e6473a84ed"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:50.020139883Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1798,"client":"0","":"downstairs","session_id":"f0724437-66d0-422b-8cc4-f1e6473a84ed"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:50.020320511Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1798,"client":"2","":"downstairs","session_id":"f0724437-66d0-422b-8cc4-f1e6473a84ed"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:50.02062723Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1798,"session_id":"f0724437-66d0-422b-8cc4-f1e6473a84ed"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:50.020704644Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1798,"session_id":"f0724437-66d0-422b-8cc4-f1e6473a84ed"} {"msg":"[0]R flush_numbers[0..12]: [283, 284, 271, 245, 281, 269, 275, 274, 279, 275, 273, 283]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:50.020899241Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1798,"":"downstairs","":"downstairs","session_id":"f0724437-66d0-422b-8cc4-f1e6473a84ed"} {"msg":"[0]R generation[0..12]: [152, 153, 146, 135, 151, 144, 148, 148, 150, 148, 147, 152]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:50.020919309Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1798,"":"downstairs","":"downstairs","session_id":"f0724437-66d0-422b-8cc4-f1e6473a84ed"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:50.020935019Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1798,"":"downstairs","":"downstairs","session_id":"f0724437-66d0-422b-8cc4-f1e6473a84ed"} {"msg":"[1]R flush_numbers[0..12]: [283, 284, 271, 245, 281, 269, 275, 274, 279, 275, 273, 283]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:50.020949358Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1798,"":"downstairs","":"downstairs","session_id":"f0724437-66d0-422b-8cc4-f1e6473a84ed"} {"msg":"[1]R generation[0..12]: [152, 153, 146, 135, 151, 144, 148, 148, 150, 148, 147, 152]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:50.020963237Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1798,"":"downstairs","":"downstairs","session_id":"f0724437-66d0-422b-8cc4-f1e6473a84ed"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:50.020977165Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1798,"":"downstairs","":"downstairs","session_id":"f0724437-66d0-422b-8cc4-f1e6473a84ed"} {"msg":"[2]R flush_numbers[0..12]: [283, 284, 271, 245, 281, 269, 275, 274, 279, 275, 273, 283]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:50.021008453Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1798,"":"downstairs","":"downstairs","session_id":"f0724437-66d0-422b-8cc4-f1e6473a84ed"} {"msg":"[2]R generation[0..12]: [152, 153, 146, 135, 151, 144, 148, 148, 150, 148, 147, 152]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:50.021024562Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1798,"":"downstairs","":"downstairs","session_id":"f0724437-66d0-422b-8cc4-f1e6473a84ed"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:50.021038521Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1798,"":"downstairs","":"downstairs","session_id":"f0724437-66d0-422b-8cc4-f1e6473a84ed"} {"msg":"Max found gen is 154","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:50.02105144Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1798,"":"downstairs","":"downstairs","session_id":"f0724437-66d0-422b-8cc4-f1e6473a84ed"} {"msg":"Generation requested: 154 >= found:154","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:50.02106469Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1798,"":"downstairs","":"downstairs","session_id":"f0724437-66d0-422b-8cc4-f1e6473a84ed"} {"msg":"Next flush: 285","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-12-12T20:38:50.021077709Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1798,"":"downstairs","":"downstairs","session_id":"f0724437-66d0-422b-8cc4-f1e6473a84ed"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:50.021093528Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1798,"":"downstairs","":"downstairs","session_id":"f0724437-66d0-422b-8cc4-f1e6473a84ed"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:50.021106957Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1798,"session_id":"f0724437-66d0-422b-8cc4-f1e6473a84ed"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:50.021118986Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1798,"client":"0","":"downstairs","session_id":"f0724437-66d0-422b-8cc4-f1e6473a84ed"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:50.021132205Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1798,"client":"1","":"downstairs","session_id":"f0724437-66d0-422b-8cc4-f1e6473a84ed"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:50.021145294Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1798,"client":"2","":"downstairs","session_id":"f0724437-66d0-422b-8cc4-f1e6473a84ed"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:50.021158613Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1798,"session_id":"f0724437-66d0-422b-8cc4-f1e6473a84ed"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:50.021170712Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1798,"session_id":"f0724437-66d0-422b-8cc4-f1e6473a84ed"} {"msg":"d718640a-bc77-404f-83e0-22d3feb72f0a is now active with session: f0724437-66d0-422b-8cc4-f1e6473a84ed","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:50.021183401Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1798,"session_id":"f0724437-66d0-422b-8cc4-f1e6473a84ed"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:50.021195281Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1798,"session_id":"f0724437-66d0-422b-8cc4-f1e6473a84ed"} {"msg":"The guest has finished waiting for activation with:154","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:50.02120738Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1798} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 234 len 32768 data: 13 13 14 13 12 13 12 12 02/30 Read block 193 len 16384 03/30 Write block 292 len 8192 data: 11 12 04/30 Read block 109 len 12288 05/30 Write block 377 len 40960 data: 9 11 15 15 15 17 15 13 14 16 06/30 Flush 07/30 Write block 219 len 28672 data: 12 12 9 8 11 10 11 08/30 Write block 360 len 4096 data: 12 09/30 Read block 365 len 40960 10/30 Write block 74 len 40960 data: 8 10 8 13 12 11 12 10 7 8 11/30 Write block 110 len 20480 data: 11 12 14 12 10 12/30 Write block 454 len 8192 data: 14 13 13/30 Read block 404 len 32768 14/30 Read block 221 len 36864 15/30 Read block 412 len 40960 16/30 Write block 122 len 8192 data: 11 12 17/30 Write block 545 len 24576 data: 9 9 9 9 10 10 18/30 Flush 19/30 Write block 299 len 36864 data: 9 11 9 8 8 7 9 11 10 20/30 Read block 78 len 28672 21/30 Write block 435 len 4096 data: 13 22/30 Read block 221 len 20480 23/30 Read block 214 len 32768 24/30 Read block 505 len 4096 25/30 Read block 462 len 16384 26/30 Write block 238 len 16384 data: 13 14 13 13 27/30 Read block 595 len 16384 28/30 Read block 482 len 8192 29/30 Write block 582 len 4096 data: 13 30/30 Write block 226 len 8192 data: 14 12 ---------------------------------------------------------------- Crucible gen:154 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1018 Acked Write 9 Done Done Done false 1019 Acked Read 7 Done Done Done false 1020 Acked Write 1 Done Done Done false 1021 Acked Read 5 Done Done Done false 1022 Acked Read 8 Done Done Done false 1023 Acked Read 1 Done Done Done false 1024 Acked Read 4 Done Done Done false 1025 Acked Write 4 Done Done Done false 1026 Acked Read 4 Done Done Done false 1027 Acked Read 2 Done Done Done false 1028 Acked Write 1 Sent Sent Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 10 10 10 30 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1017 1017 1017 Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:38:50.040618871Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1798} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 152 152 152 283 283 283 F F F 1 020-039 153 153 153 284 284 284 F F F 2 040-059 146 146 146 271 271 271 F F F 3 060-079 154 154 154 286 286 286 F F F 4 080-099 154 154 154 286 286 286 F F F 5 100-119 154 154 154 286 286 286 F F F 6 120-139 154 154 154 286 286 286 F F F 7 140-159 148 148 148 274 274 274 F F F 8 160-179 150 150 150 279 279 279 F F F 9 180-199 148 148 148 275 275 275 F F F 10 200-219 154 154 154 286 286 286 F F F 11 220-239 154 154 154 286 286 286 T T T 12 240-259 154 154 154 285 285 285 T T T 13 260-279 153 153 153 284 284 284 F F F 14 280-299 154 154 154 285 285 285 T T T 15 300-319 146 146 146 271 271 271 T T T 16 320-339 150 150 150 280 280 280 F F F 17 340-359 153 153 153 284 284 284 F F F 18 360-379 154 154 154 286 286 286 F F F 19 380-399 154 154 154 285 285 285 F F F 20 400-419 150 150 150 278 278 278 F F F 21 420-439 145 145 145 270 270 270 T T T 22 440-459 154 154 154 286 286 286 F F F 23 460-479 152 152 152 283 283 283 F F F 24 480-499 152 152 152 282 282 282 F F F 25 500-519 147 147 147 273 273 273 F F F 26 520-539 151 151 151 281 281 281 F F F 27 540-559 154 154 154 286 286 286 F F F 28 560-579 153 153 153 284 284 284 F F F 29 580-599 148 148 148 276 276 276 T T T Max gen: 154, Max flush: 286 A Difference in extent metadata is expected here On loop 77 Verifying data now No change after verify Loop: 77 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 152 152 152 283 283 283 F F F 1 020-039 153 153 153 284 284 284 F F F 2 040-059 146 146 146 271 271 271 F F F 3 060-079 154 154 154 286 286 286 F F F 4 080-099 154 154 154 286 286 286 F F F 5 100-119 154 154 154 286 286 286 F F F 6 120-139 154 154 154 286 286 286 F F F 7 140-159 148 148 148 274 274 274 F F F 8 160-179 150 150 150 279 279 279 F F F 9 180-199 148 148 148 275 275 275 F F F 10 200-219 154 154 154 286 286 286 F F F 11 220-239 155 155 155 287 287 287 F F F 12 240-259 155 155 155 287 287 287 F F F 13 260-279 153 153 153 284 284 284 F F F 14 280-299 155 155 155 287 287 287 F F F 15 300-319 155 155 155 287 287 287 F F F 16 320-339 150 150 150 280 280 280 F F F 17 340-359 153 153 153 284 284 284 F F F 18 360-379 154 154 154 286 286 286 F F F 19 380-399 154 154 154 285 285 285 F F F 20 400-419 150 150 150 278 278 278 F F F 21 420-439 155 155 155 287 287 287 F F F 22 440-459 154 154 154 286 286 286 F F F 23 460-479 152 152 152 283 283 283 F F F 24 480-499 152 152 152 282 282 282 F F F 25 500-519 147 147 147 273 273 273 F F F 26 520-539 151 151 151 281 281 281 F F F 27 540-559 154 154 154 286 286 286 F F F 28 560-579 153 153 153 284 284 284 F F F 29 580-599 155 155 155 287 287 287 F F F Max gen: 155, Max flush: 287 Begin loop 78 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:54.316710389Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1806,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:54.361196732Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1806,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:54.361994206Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1806} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"26e7d501b33f47379e53601720353c2983e7654b\",\n git_commit_timestamp: \"2024-12-12T20:16:40.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:54.362031104Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1806} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:54.362044953Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1806} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:54.362063372Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1806} {"msg":"Crucible 5b9cc8a3-ff41-48b4-8b56-d0fa4ccc5cf3 has session id: 466f32a8-d184-41fd-a5ae-fece557f0a62","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:54.362508441Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1806,"session_id":"466f32a8-d184-41fd-a5ae-fece557f0a62"} {"msg":"Upstairs opts: Upstairs UUID: 5b9cc8a3-ff41-48b4-8b56-d0fa4ccc5cf3, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:54.362544309Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1806,"session_id":"466f32a8-d184-41fd-a5ae-fece557f0a62"} {"msg":"Crucible stats registered with UUID: 5b9cc8a3-ff41-48b4-8b56-d0fa4ccc5cf3","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:54.362559138Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1806,"session_id":"466f32a8-d184-41fd-a5ae-fece557f0a62"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:54.393297568Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1806,"task":"crutest"} {"msg":"The guest has requested activation with gen:156","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:54.393341105Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1806} {"msg":"5b9cc8a3-ff41-48b4-8b56-d0fa4ccc5cf3 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:54.393402311Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1806,"session_id":"466f32a8-d184-41fd-a5ae-fece557f0a62"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:54.393431129Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1806,"client":"0","":"downstairs","session_id":"466f32a8-d184-41fd-a5ae-fece557f0a62"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:54.393513113Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1806,"client":"0","":"downstairs","session_id":"466f32a8-d184-41fd-a5ae-fece557f0a62"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:54.39356161Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1806,"client":"1","":"downstairs","session_id":"466f32a8-d184-41fd-a5ae-fece557f0a62"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:54.393592118Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1806,"client":"1","":"downstairs","session_id":"466f32a8-d184-41fd-a5ae-fece557f0a62"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:54.393608037Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1806,"client":"2","":"downstairs","session_id":"466f32a8-d184-41fd-a5ae-fece557f0a62"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:54.393622816Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1806,"client":"2","":"downstairs","session_id":"466f32a8-d184-41fd-a5ae-fece557f0a62"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:54.393636265Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1806,"":"io task","client":"2","":"downstairs","session_id":"466f32a8-d184-41fd-a5ae-fece557f0a62"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:54.393654243Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1806,"":"io task","client":"0","":"downstairs","session_id":"466f32a8-d184-41fd-a5ae-fece557f0a62"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:54.393668313Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1806,"":"io task","client":"1","":"downstairs","session_id":"466f32a8-d184-41fd-a5ae-fece557f0a62"} {"msg":"ds_connection connected from Ok(127.0.0.1:56870)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:54.393682152Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1806,"":"io task","client":"2","":"downstairs","session_id":"466f32a8-d184-41fd-a5ae-fece557f0a62"} {"msg":"ds_connection connected from Ok(127.0.0.1:48635)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:54.39369918Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1806,"":"io task","client":"1","":"downstairs","session_id":"466f32a8-d184-41fd-a5ae-fece557f0a62"} {"msg":"ds_connection connected from Ok(127.0.0.1:51486)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:54.393885738Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1806,"":"io task","client":"0","":"downstairs","session_id":"466f32a8-d184-41fd-a5ae-fece557f0a62"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:54.394487276Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1806,"client":"0","":"downstairs","session_id":"466f32a8-d184-41fd-a5ae-fece557f0a62"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:54.394528603Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1806,"client":"1","":"downstairs","session_id":"466f32a8-d184-41fd-a5ae-fece557f0a62"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:54.394560321Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1806,"client":"2","":"downstairs","session_id":"466f32a8-d184-41fd-a5ae-fece557f0a62"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:54.395125392Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1806,"client":"1","":"downstairs","session_id":"466f32a8-d184-41fd-a5ae-fece557f0a62"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:54.395171879Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1806,"client":"0","":"downstairs","session_id":"466f32a8-d184-41fd-a5ae-fece557f0a62"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:54.395200537Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1806,"client":"2","":"downstairs","session_id":"466f32a8-d184-41fd-a5ae-fece557f0a62"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:54.39573889Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1806,"session_id":"466f32a8-d184-41fd-a5ae-fece557f0a62"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:54.395777227Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1806,"session_id":"466f32a8-d184-41fd-a5ae-fece557f0a62"} {"msg":"[0]R flush_numbers[0..12]: [283, 284, 271, 286, 286, 286, 286, 274, 279, 275, 286, 287]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:54.395866831Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1806,"":"downstairs","":"downstairs","session_id":"466f32a8-d184-41fd-a5ae-fece557f0a62"} {"msg":"[0]R generation[0..12]: [152, 153, 146, 154, 154, 154, 154, 148, 150, 148, 154, 155]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:54.395896549Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1806,"":"downstairs","":"downstairs","session_id":"466f32a8-d184-41fd-a5ae-fece557f0a62"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:54.395916607Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1806,"":"downstairs","":"downstairs","session_id":"466f32a8-d184-41fd-a5ae-fece557f0a62"} {"msg":"[1]R flush_numbers[0..12]: [283, 284, 271, 286, 286, 286, 286, 274, 279, 275, 286, 287]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:54.395931716Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1806,"":"downstairs","":"downstairs","session_id":"466f32a8-d184-41fd-a5ae-fece557f0a62"} {"msg":"[1]R generation[0..12]: [152, 153, 146, 154, 154, 154, 154, 148, 150, 148, 154, 155]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:54.395945945Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1806,"":"downstairs","":"downstairs","session_id":"466f32a8-d184-41fd-a5ae-fece557f0a62"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:54.395960074Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1806,"":"downstairs","":"downstairs","session_id":"466f32a8-d184-41fd-a5ae-fece557f0a62"} {"msg":"[2]R flush_numbers[0..12]: [283, 284, 271, 286, 286, 286, 286, 274, 279, 275, 286, 287]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:54.395973814Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1806,"":"downstairs","":"downstairs","session_id":"466f32a8-d184-41fd-a5ae-fece557f0a62"} {"msg":"[2]R generation[0..12]: [152, 153, 146, 154, 154, 154, 154, 148, 150, 148, 154, 155]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:54.395987322Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1806,"":"downstairs","":"downstairs","session_id":"466f32a8-d184-41fd-a5ae-fece557f0a62"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:54.396001372Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1806,"":"downstairs","":"downstairs","session_id":"466f32a8-d184-41fd-a5ae-fece557f0a62"} {"msg":"Max found gen is 156","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:54.396014611Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1806,"":"downstairs","":"downstairs","session_id":"466f32a8-d184-41fd-a5ae-fece557f0a62"} {"msg":"Generation requested: 156 >= found:156","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:54.39602785Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1806,"":"downstairs","":"downstairs","session_id":"466f32a8-d184-41fd-a5ae-fece557f0a62"} {"msg":"Next flush: 288","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:54.396041059Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1806,"":"downstairs","":"downstairs","session_id":"466f32a8-d184-41fd-a5ae-fece557f0a62"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:54.396053888Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1806,"":"downstairs","":"downstairs","session_id":"466f32a8-d184-41fd-a5ae-fece557f0a62"} {"Wait for a query_work_queue command to finish before sending IO msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:54.396071157Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1806,"session_id":"466f32a8-d184-41fd-a5ae-fece557f0a62"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:54.396083226Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1806,"client":"0","":"downstairs","session_id":"466f32a8-d184-41fd-a5ae-fece557f0a62"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:54.396096435Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1806,"client":"1","":"downstairs","session_id":"466f32a8-d184-41fd-a5ae-fece557f0a62"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:54.396111544Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1806,"client":"2","":"downstairs","session_id":"466f32a8-d184-41fd-a5ae-fece557f0a62"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:54.396124933Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1806,"session_id":"466f32a8-d184-41fd-a5ae-fece557f0a62"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:54.396137252Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1806,"session_id":"466f32a8-d184-41fd-a5ae-fece557f0a62"} {"msg":"5b9cc8a3-ff41-48b4-8b56-d0fa4ccc5cf3 is now active with session: 466f32a8-d184-41fd-a5ae-fece557f0a62","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:54.396149841Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1806,"session_id":"466f32a8-d184-41fd-a5ae-fece557f0a62"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:54.39616249Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1806,"session_id":"466f32a8-d184-41fd-a5ae-fece557f0a62"} {"msg":"The guest has finished waiting for activation with:156","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:54.39617474Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1806} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 262 len 36864 data: 11 14 14 17 20 21 20 11 12 02/30 Read block 326 len 36864 03/30 Read block 497 len 24576 04/30 Write block 278 len 8192 data: 13 11 05/30 Write block 366 len 4096 data: 18 06/30 Write block 249 len 32768 data: 8 8 10 10 13 13 13 11 07/30 Write block 7 len 12288 data: 9 12 14 08/30 Read block 70 len 24576 09/30 Write block 308 len 8192 data: 12 11 10/30 Write block 528 len 40960 data: 10 10 5 6 7 7 7 6 10 10 11/30 Read block 426 len 8192 12/30 Read block 172 len 4096 13/30 Write block 151 len 16384 data: 11 13 13 12 14/30 Write block 569 len 16384 data: 15 13 16 16 15/30 Flush 16/30 Write block 572 len 16384 data: 17 15 16 14 17/30 Read block 454 len 24576 18/30 Flush 19/30 Write block 114 len 36864 data: 11 12 11 10 11 9 12 12 12 20/30 Write block 109 len 16384 data: 10 12 13 15 21/30 Read block 244 len 8192 22/30 Read block 189 len 4096 23/30 Read block 46 len 8192 24/30 Write block 443 len 36864 data: 9 9 9 12 15 15 18 17 18 25/30 Read block 379 len 8192 26/30 Read block 465 len 20480 27/30 Read block 274 len 24576 28/30 Write block 328 len 4096 data: 10 29/30 Write block 429 len 36864 data: 14 12 14 14 13 10 14 11 10 30/30 Write block 487 len 16384 data: 15 16 18 16 ---------------------------------------------------------------- Crucible gen:156 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1018 Acked Write 9 Done Done Done false 1019 Acked Write 4 Done Done Done false 1020 Acked Read 2 Done Done Done false 1021 Acked Read 1 Done Done Done false 1022 Acked Read 2 Done Done Done false 1023 Acked Write 9 Done Done Done false 1024 Acked Read 2 Done Done Done false 1025 Acked Read 5 Done Done Done false 1026 Acked Read 6 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Write 9 Sent Sent Sent false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 10 10 10 30 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1017 1017 1017 Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:38:54.414493477Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1806} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 156 156 156 288 288 288 F F F 1 020-039 153 153 153 284 284 284 F F F 2 040-059 146 146 146 271 271 271 F F F 3 060-079 154 154 154 286 286 286 F F F 4 080-099 154 154 154 286 286 286 F F F 5 100-119 154 154 154 286 286 286 T T T 6 120-139 154 154 154 286 286 286 T T T 7 140-159 156 156 156 288 288 288 F F F 8 160-179 150 150 150 279 279 279 F F F 9 180-199 148 148 148 275 275 275 F F F 10 200-219 154 154 154 286 286 286 F F F 11 220-239 155 155 155 287 287 287 F F F 12 240-259 156 156 156 288 288 288 F F F 13 260-279 156 156 156 288 288 288 F F F 14 280-299 155 155 155 287 287 287 F F F 15 300-319 156 156 156 288 288 288 F F F 16 320-339 150 150 150 280 280 280 T T T 17 340-359 153 153 153 284 284 284 F F F 18 360-379 156 156 156 288 288 288 F F F 19 380-399 154 154 154 285 285 285 F F F 20 400-419 150 150 150 278 278 278 F F F 21 420-439 155 155 155 287 287 287 T T T 22 440-459 154 154 154 286 286 286 T T T 23 460-479 152 152 152 283 283 283 F F F 24 480-499 152 152 152 282 282 282 T T T 25 500-519 147 147 147 273 273 273 F F F 26 520-539 156 156 156 288 288 288 F F F 27 540-559 154 154 154 286 286 286 F F F 28 560-579 156 156 156 289 289 289 F F F 29 580-599 155 155 155 287 287 287 F F F Max gen: 156, Max flush: 289 A Difference in extent metadata is expected here On loop 78 Verifying data now No change after verify Loop: 78 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 156 156 156 288 288 288 F F F 1 020-039 153 153 153 284 284 284 F F F 2 040-059 146 146 146 271 271 271 F F F 3 060-079 154 154 154 286 286 286 F F F 4 080-099 154 154 154 286 286 286 F F F 5 100-119 157 157 157 290 290 290 F F F 6 120-139 157 157 157 290 290 290 F F F 7 140-159 156 156 156 288 288 288 F F F 8 160-179 150 150 150 279 279 279 F F F 9 180-199 148 148 148 275 275 275 F F F 10 200-219 154 154 154 286 286 286 F F F 11 220-239 155 155 155 287 287 287 F F F 12 240-259 156 156 156 288 288 288 F F F 13 260-279 156 156 156 288 288 288 F F F 14 280-299 155 155 155 287 287 287 F F F 15 300-319 156 156 156 288 288 288 F F F 16 320-339 157 157 157 290 290 290 F F F 17 340-359 153 153 153 284 284 284 F F F 18 360-379 156 156 156 288 288 288 F F F 19 380-399 154 154 154 285 285 285 F F F 20 400-419 150 150 150 278 278 278 F F F 21 420-439 157 157 157 290 290 290 F F F 22 440-459 157 157 157 290 290 290 F F F 23 460-479 152 152 152 283 283 283 F F F 24 480-499 157 157 157 290 290 290 F F F 25 500-519 147 147 147 273 273 273 F F F 26 520-539 156 156 156 288 288 288 F F F 27 540-559 154 154 154 286 286 286 F F F 28 560-579 156 156 156 289 289 289 F F F 29 580-599 155 155 155 287 287 287 F F F Max gen: 157, Max flush: 290 Begin loop 79 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:58.69007125Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1814,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:58.734783647Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1814,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:58.735412893Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1814} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"26e7d501b33f47379e53601720353c2983e7654b\",\n git_commit_timestamp: \"2024-12-12T20:16:40.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:58.73544955Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1814} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:58.73546406Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1814} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:58.735478468Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1814} {"msg":"Crucible b6197dc2-4aaf-4731-9aa6-ef5f2246086d has session id: bbe78e69-70d5-4682-8bda-22e0682fc146","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:58.735856473Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1814,"session_id":"bbe78e69-70d5-4682-8bda-22e0682fc146"} {"msg":"Upstairs opts: Upstairs UUID: b6197dc2-4aaf-4731-9aa6-ef5f2246086d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:58.735904899Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1814,"session_id":"bbe78e69-70d5-4682-8bda-22e0682fc146"} {"msg":"Crucible stats registered with UUID: b6197dc2-4aaf-4731-9aa6-ef5f2246086d","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:58.735931187Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1814,"session_id":"bbe78e69-70d5-4682-8bda-22e0682fc146"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:58.767333102Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1814,"task":"crutest"} {"msg":"The guest has requested activation with gen:158","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:58.76736524Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1814} {"msg":"b6197dc2-4aaf-4731-9aa6-ef5f2246086d active request set","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:58.767445004Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1814,"session_id":"bbe78e69-70d5-4682-8bda-22e0682fc146"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:58.767466213Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1814,"client":"0","":"downstairs","session_id":"bbe78e69-70d5-4682-8bda-22e0682fc146"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:58.767494591Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1814,"client":"0","":"downstairs","session_id":"bbe78e69-70d5-4682-8bda-22e0682fc146"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:58.767515959Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1814,"client":"1","":"downstairs","session_id":"bbe78e69-70d5-4682-8bda-22e0682fc146"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:58.767533828Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1814,"client":"1","":"downstairs","session_id":"bbe78e69-70d5-4682-8bda-22e0682fc146"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:58.767550187Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1814,"client":"2","":"downstairs","session_id":"bbe78e69-70d5-4682-8bda-22e0682fc146"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:58.767567536Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1814,"client":"2","":"downstairs","session_id":"bbe78e69-70d5-4682-8bda-22e0682fc146"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:58.767584445Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1814,"":"io task","client":"2","":"downstairs","session_id":"bbe78e69-70d5-4682-8bda-22e0682fc146"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:58.767614283Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1814,"":"io task","client":"0","":"downstairs","session_id":"bbe78e69-70d5-4682-8bda-22e0682fc146"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:58.76764531Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1814,"":"io task","client":"1","":"downstairs","session_id":"bbe78e69-70d5-4682-8bda-22e0682fc146"} {"msg":"ds_connection connected from Ok(127.0.0.1:53440)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:58.767714976Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1814,"":"io task","client":"0","":"downstairs","session_id":"bbe78e69-70d5-4682-8bda-22e0682fc146"} {"msg":"ds_connection connected from Ok(127.0.0.1:47136)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:58.767847947Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1814,"":"io task","client":"1","":"downstairs","session_id":"bbe78e69-70d5-4682-8bda-22e0682fc146"} {"msg":"ds_connection connected from Ok(127.0.0.1:53784)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:58.767897393Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1814,"":"io task","client":"2","":"downstairs","session_id":"bbe78e69-70d5-4682-8bda-22e0682fc146"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:58.768449765Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1814,"client":"2","":"downstairs","session_id":"bbe78e69-70d5-4682-8bda-22e0682fc146"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:58.768480743Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1814,"client":"0","":"downstairs","session_id":"bbe78e69-70d5-4682-8bda-22e0682fc146"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:58.768499842Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1814,"client":"1","":"downstairs","session_id":"bbe78e69-70d5-4682-8bda-22e0682fc146"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:58.769094441Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1814,"client":"2","":"downstairs","session_id":"bbe78e69-70d5-4682-8bda-22e0682fc146"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:58.769130838Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1814,"client":"1","":"downstairs","session_id":"bbe78e69-70d5-4682-8bda-22e0682fc146"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:58.769159376Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1814,"client":"0","":"downstairs","session_id":"bbe78e69-70d5-4682-8bda-22e0682fc146"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:58.769642433Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1814,"session_id":"bbe78e69-70d5-4682-8bda-22e0682fc146"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:58.769761804Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1814,"session_id":"bbe78e69-70d5-4682-8bda-22e0682fc146"} {"msg":"[0]R flush_numbers[0..12]: [288, 284, 271, 286, 286, 290, 290, 288, 279, 275, 286, 287]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:58.76982177Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1814,"":"downstairs","":"downstairs","session_id":"bbe78e69-70d5-4682-8bda-22e0682fc146"} {"msg":"[0]R generation[0..12]: [156, 153, 146, 154, 154, 157, 157, 156, 150, 148, 154, 155]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:58.769841109Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1814,"":"downstairs","":"downstairs","session_id":"bbe78e69-70d5-4682-8bda-22e0682fc146"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:58.769855798Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1814,"":"downstairs","":"downstairs","session_id":"bbe78e69-70d5-4682-8bda-22e0682fc146"} {"msg":"[1]R flush_numbers[0..12]: [288, 284, 271, 286, 286, 290, 290, 288, 279, 275, 286, 287]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:58.769869947Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1814,"":"downstairs","":"downstairs","session_id":"bbe78e69-70d5-4682-8bda-22e0682fc146"} {"msg":"[1]R generation[0..12]: [156, 153, 146, 154, 154, 157, 157, 156, 150, 148, 154, 155]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:58.769883766Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1814,"":"downstairs","":"downstairs","session_id":"bbe78e69-70d5-4682-8bda-22e0682fc146"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:58.769897745Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1814,"":"downstairs","":"downstairs","session_id":"bbe78e69-70d5-4682-8bda-22e0682fc146"} {"msg":"[2]R flush_numbers[0..12]: [288, 284, 271, 286, 286, 290, 290, 288, 279, 275, 286, 287]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:58.769911254Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1814,"":"downstairs","":"downstairs","session_id":"bbe78e69-70d5-4682-8bda-22e0682fc146"} {"msg":"[2]R generation[0..12]: [156, 153, 146, 154, 154, 157, 157, 156, 150, 148, 154, 155]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:58.769927873Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1814,"":"downstairs","":"downstairs","session_id":"bbe78e69-70d5-4682-8bda-22e0682fc146"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:58.769953351Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1814,"":"downstairs","":"downstairs","session_id":"bbe78e69-70d5-4682-8bda-22e0682fc146"} {"msg":"Max found gen is 158","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:58.769980369Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1814,"":"downstairs","":"downstairs","session_id":"bbe78e69-70d5-4682-8bda-22e0682fc146"} {"msg":"Generation requested: 158 >= found:158","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:58.770002538Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1814,"":"downstairs","":"downstairs","session_id":"bbe78e69-70d5-4682-8bda-22e0682fc146"} {"msg":"Next flush: 291","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:58.770016377Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1814,"":"downstairs","":"downstairs","session_id":"bbe78e69-70d5-4682-8bda-22e0682fc146"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:58.770029656Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1814,"":"Wait for a query_work_queue command to finish before sending IO downstairs","":"downstairs","session_id":"bbe78e69-70d5-4682-8bda-22e0682fc146"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:58.770048115Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1814,"session_id":"bbe78e69-70d5-4682-8bda-22e0682fc146"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:58.770059934Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1814,"client":"0","":"downstairs","session_id":"bbe78e69-70d5-4682-8bda-22e0682fc146"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:58.770073463Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1814,"client":"1","":"downstairs","session_id":"bbe78e69-70d5-4682-8bda-22e0682fc146"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:58.770086522Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1814,"client":"2","":"downstairs","session_id":"bbe78e69-70d5-4682-8bda-22e0682fc146"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:58.770099771Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1814,"session_id":"bbe78e69-70d5-4682-8bda-22e0682fc146"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:58.77011193Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1814,"session_id":"bbe78e69-70d5-4682-8bda-22e0682fc146"} {"msg":"b6197dc2-4aaf-4731-9aa6-ef5f2246086d is now active with session: bbe78e69-70d5-4682-8bda-22e0682fc146","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:58.770124639Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1814,"session_id":"bbe78e69-70d5-4682-8bda-22e0682fc146"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:58.770136359Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1814,"session_id":"bbe78e69-70d5-4682-8bda-22e0682fc146"} {"msg":"The guest has finished waiting for activation with:158","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:38:58.770148408Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1814} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 474 len 24576 data: 10 11 11 9 10 10 02/30 Write block 272 len 20480 data: 14 13 15 13 11 03/30 Flush 04/30 Write block 140 len 24576 data: 8 8 11 12 12 12 05/30 Read block 231 len 32768 06/30 Read block 590 len 4096 07/30 Read block 518 len 40960 08/30 Read block 586 len 4096 09/30 Read block 228 len 40960 10/30 Write block 88 len 36864 data: 16 14 15 12 9 15 14 13 9 11/30 Read block 233 len 12288 12/30 Read block 43 len 24576 13/30 Flush 14/30 Write block 291 len 16384 data: 11 12 13 12 15/30 Write block 65 len 40960 data: 8 10 9 9 9 10 9 10 10 9 16/30 Write block 297 len 4096 data: 12 17/30 Read block 69 len 12288 18/30 Read block 340 len 36864 19/30 Write block 504 len 12288 data: 12 11 12 20/30 Flush 21/30 Write block 537 len 16384 data: 11 8 7 11 22/30 Read block 199 len 8192 23/30 Write block 125 len 28672 data: 12 12 11 12 13 12 9 24/30 Write block 118 len 40960 data: 12 10 13 13 13 13 11 13 13 12 25/30 Read block 494 len 24576 26/30 Read block 211 len 40960 27/30 Write block 316 len 40960 data: 18 16 16 12 13 13 13 11 9 7 28/30 Write block 356 len 16384 data: 16 15 12 12 29/30 Read block 165 len 16384 30/30 Read block 338 len 12288 ---------------------------------------------------------------- Crucible gen:158 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1020 Acked Write 4 Done Done Done false 1021 Acked Read 2 Done Done Done false 1022 Acked Write 7 Done Done Done false 1023 Acked Write 10 Done Done Done false 1024 Acked Read 6 Done Done Done false 1025 Acked Read 10 Done Done Done false 1026 Acked Write 10 Done Done Done false 1027 Acked Write 4 Done Done Done false 1028 Acked Read 4 Done Done Done false 1029 Acked Read 3 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 10 10 9 29 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1019 1019 1019 Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:38:58.790190115Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1814} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 156 156 156 288 288 288 F F F 1 020-039 153 153 153 284 284 284 F F F 2 040-059 146 146 146 271 271 271 F F F 3 060-079 158 158 158 293 293 293 F F F 4 080-099 158 158 158 292 292 292 F F F 5 100-119 157 157 157 290 290 290 T T T 6 120-139 157 157 157 290 290 290 T T T 7 140-159 158 158 158 292 292 292 F F F 8 160-179 150 150 150 279 279 279 F F F 9 180-199 148 148 148 275 275 275 F F F 10 200-219 154 154 154 286 286 286 F F F 11 220-239 155 155 155 287 287 287 F F F 12 240-259 156 156 156 288 288 288 F F F 13 260-279 158 158 158 291 291 291 F F F 14 280-299 158 158 158 293 293 293 F F F 15 300-319 156 156 156 288 288 288 T T T 16 320-339 157 157 157 290 290 290 T T T 17 340-359 153 153 153 284 284 284 T T T 18 360-379 156 156 156 288 288 288 F F F 19 380-399 154 154 154 285 285 285 F F F 20 400-419 150 150 150 278 278 278 F F F 21 420-439 157 157 157 290 290 290 F F F 22 440-459 157 157 157 290 290 290 F F F 23 460-479 158 158 158 291 291 291 F F F 24 480-499 157 157 157 290 290 290 F F F 25 500-519 158 158 158 293 293 293 F F F 26 520-539 156 156 156 288 288 288 T T T 27 540-559 154 154 154 286 286 286 T T T 28 560-579 156 156 156 289 289 289 F F F 29 580-599 155 155 155 287 287 287 F F F Max gen: 158, Max flush: 293 A Difference in extent metadata is expected here On loop 79 Verifying data now No change after verify Loop: 79 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 156 156 156 288 288 288 F F F 1 020-039 153 153 153 284 284 284 F F F 2 040-059 146 146 146 271 271 271 F F F 3 060-079 158 158 158 293 293 293 F F F 4 080-099 158 158 158 292 292 292 F F F 5 100-119 159 159 159 294 294 294 F F F 6 120-139 159 159 159 294 294 294 F F F 7 140-159 158 158 158 292 292 292 F F F 8 160-179 150 150 150 279 279 279 F F F 9 180-199 148 148 148 275 275 275 F F F 10 200-219 154 154 154 286 286 286 F F F 11 220-239 155 155 155 287 287 287 F F F 12 240-259 156 156 156 288 288 288 F F F 13 260-279 158 158 158 291 291 291 F F F 14 280-299 158 158 158 293 293 293 F F F 15 300-319 159 159 159 294 294 294 F F F 16 320-339 159 159 159 294 294 294 F F F 17 340-359 159 159 159 294 294 294 F F F 18 360-379 156 156 156 288 288 288 F F F 19 380-399 154 154 154 285 285 285 F F F 20 400-419 150 150 150 278 278 278 F F F 21 420-439 157 157 157 290 290 290 F F F 22 440-459 157 157 157 290 290 290 F F F 23 460-479 158 158 158 291 291 291 F F F 24 480-499 157 157 157 290 290 290 F F F 25 500-519 158 158 158 293 293 293 F F F 26 520-539 159 159 159 294 294 294 F F F 27 540-559 159 159 159 294 294 294 F F F 28 560-579 156 156 156 289 289 289 F F F 29 580-599 155 155 155 287 287 287 F F F Max gen: 159, Max flush: 294 Begin loop 80 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:03.078880434Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1822,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:03.122888639Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1822,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:03.123534955Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1822} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"26e7d501b33f47379e53601720353c2983e7654b\",\n git_commit_timestamp: \"2024-12-12T20:16:40.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:03.123570492Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1822} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:03.123585411Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1822} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:03.12360048Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1822} {"msg":"Crucible e509ac19-590a-470c-9ece-7adf25370620 has session id: 1b26f2f7-74c7-4a3c-9395-a702f0027e97","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:03.124093596Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1822,"session_id":"1b26f2f7-74c7-4a3c-9395-a702f0027e97"} {"msg":"Upstairs opts: Upstairs UUID: e509ac19-590a-470c-9ece-7adf25370620, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:03.124140273Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1822,"session_id":"1b26f2f7-74c7-4a3c-9395-a702f0027e97"} {"msg":"Crucible stats registered with UUID: e509ac19-590a-470c-9ece-7adf25370620","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:03.124170321Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1822,"session_id":"1b26f2f7-74c7-4a3c-9395-a702f0027e97"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:03.155521259Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1822,"task":"crutest"} {"msg":"The guest has requested activation with gen:160","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:03.155555287Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1822} {"msg":"e509ac19-590a-470c-9ece-7adf25370620 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:03.155636061Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1822,"session_id":"1b26f2f7-74c7-4a3c-9395-a702f0027e97"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:03.155658059Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1822,"client":"0","":"downstairs","session_id":"1b26f2f7-74c7-4a3c-9395-a702f0027e97"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:03.155683987Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1822,"client":"0","":"downstairs","session_id":"1b26f2f7-74c7-4a3c-9395-a702f0027e97"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:03.155698397Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1822,"client":"1","":"downstairs","session_id":"1b26f2f7-74c7-4a3c-9395-a702f0027e97"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:03.155712456Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1822,"client":"1","":"downstairs","session_id":"1b26f2f7-74c7-4a3c-9395-a702f0027e97"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:03.155725705Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1822,"client":"2","":"downstairs","session_id":"1b26f2f7-74c7-4a3c-9395-a702f0027e97"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:03.155739624Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1822,"client":"2","":"downstairs","session_id":"1b26f2f7-74c7-4a3c-9395-a702f0027e97"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:03.155752743Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1822,"":"io task","client":"2","":"downstairs","session_id":"1b26f2f7-74c7-4a3c-9395-a702f0027e97"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:03.155766522Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1822,"":"io task","client":"0","":"downstairs","session_id":"1b26f2f7-74c7-4a3c-9395-a702f0027e97"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:03.155780321Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1822,"":"io task","client":"1","":"downstairs","session_id":"1b26f2f7-74c7-4a3c-9395-a702f0027e97"} {"msg":"ds_connection connected from Ok(127.0.0.1:53059)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:03.155882804Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1822,"":"io task","client":"2","":"downstairs","session_id":"1b26f2f7-74c7-4a3c-9395-a702f0027e97"} {"msg":"ds_connection connected from Ok(127.0.0.1:34169)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:03.156105858Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1822,"":"io task","client":"1","":"downstairs","session_id":"1b26f2f7-74c7-4a3c-9395-a702f0027e97"} {"msg":"ds_connection connected from Ok(127.0.0.1:44909)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:03.156154305Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1822,"":"io task","client":"0","":"downstairs","session_id":"1b26f2f7-74c7-4a3c-9395-a702f0027e97"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:03.156734635Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1822,"client":"0","":"downstairs","session_id":"1b26f2f7-74c7-4a3c-9395-a702f0027e97"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:03.156774942Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1822,"client":"2","":"downstairs","session_id":"1b26f2f7-74c7-4a3c-9395-a702f0027e97"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:03.156966479Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1822,"client":"1","":"downstairs","session_id":"1b26f2f7-74c7-4a3c-9395-a702f0027e97"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:03.157427717Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1822,"client":"2","":"downstairs","session_id":"1b26f2f7-74c7-4a3c-9395-a702f0027e97"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:03.157458495Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1822,"client":"0","":"downstairs","session_id":"1b26f2f7-74c7-4a3c-9395-a702f0027e97"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:03.157599866Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1822,"client":"1","":"downstairs","session_id":"1b26f2f7-74c7-4a3c-9395-a702f0027e97"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:03.15797379Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1822,"session_id":"1b26f2f7-74c7-4a3c-9395-a702f0027e97"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:03.158053144Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1822,"session_id":"1b26f2f7-74c7-4a3c-9395-a702f0027e97"} {"msg":"[0]R flush_numbers[0..12]: [288, 284, 271, 293, 292, 294, 294, 292, 279, 275, 286, 287]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:03.158175826Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1822,"":"downstairs","":"downstairs","session_id":"1b26f2f7-74c7-4a3c-9395-a702f0027e97"} {"msg":"[0]R generation[0..12]: [156, 153, 146, 158, 158, 159, 159, 158, 150, 148, 154, 155]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:03.158195744Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1822,"":"downstairs","":"downstairs","session_id":"1b26f2f7-74c7-4a3c-9395-a702f0027e97"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:03.158210773Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1822,"":"downstairs","":"downstairs","session_id":"1b26f2f7-74c7-4a3c-9395-a702f0027e97"} {"msg":"[1]R flush_numbers[0..12]: [288, 284, 271, 293, 292, 294, 294, 292, 279, 275, 286, 287]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:03.158224862Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1822,"":"downstairs","":"downstairs","session_id":"1b26f2f7-74c7-4a3c-9395-a702f0027e97"} {"msg":"[1]R generation[0..12]: [156, 153, 146, 158, 158, 159, 159, 158, 150, 148, 154, 155]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:03.158238801Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1822,"":"downstairs","":"downstairs","session_id":"1b26f2f7-74c7-4a3c-9395-a702f0027e97"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:03.15825275Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1822,"":"downstairs","":"downstairs","session_id":"1b26f2f7-74c7-4a3c-9395-a702f0027e97"} {"msg":"[2]R flush_numbers[0..12]: [288, 284, 271, 293, 292, 294, 294, 292, 279, 275, 286, 287]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:03.158270029Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1822,"":"downstairs","":"downstairs","session_id":"1b26f2f7-74c7-4a3c-9395-a702f0027e97"} {"msg":"[2]R generation[0..12]: [156, 153, 146, 158, 158, 159, 159, 158, 150, 148, 154, 155]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:03.158284488Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1822,"":"downstairs","":"downstairs","session_id":"1b26f2f7-74c7-4a3c-9395-a702f0027e97"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:03.158298777Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1822,"":"downstairs","":"downstairs","session_id":"1b26f2f7-74c7-4a3c-9395-a702f0027e97"} {"msg":"Max found gen is 160","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:03.158311816Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1822,"":"downstairs","":"downstairs","session_id":"1b26f2f7-74c7-4a3c-9395-a702f0027e97"} {"msg":"Generation requested: 160 >= found:160","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:03.158325285Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1822,"":"downstairs","":"downstairs","session_id":"1b26f2f7-74c7-4a3c-9395-a702f0027e97"} {"msg":"Next flush: 295","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:03.158338364Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1822,"":"downstairs","":"downstairs","session_id":"1b26f2f7-74c7-4a3c-9395-a702f0027e97"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-12-12T20:39:03.158351504Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1822,"":"downstairs","":"downstairs","session_id":"1b26f2f7-74c7-4a3c-9395-a702f0027e97"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:03.158367522Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1822,"session_id":"1b26f2f7-74c7-4a3c-9395-a702f0027e97"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:03.158379832Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1822,"client":"0","":"downstairs","session_id":"1b26f2f7-74c7-4a3c-9395-a702f0027e97"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:03.158393161Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1822,"client":"1","":"downstairs","session_id":"1b26f2f7-74c7-4a3c-9395-a702f0027e97"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:03.15840643Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1822,"client":"2","":"downstairs","session_id":"1b26f2f7-74c7-4a3c-9395-a702f0027e97"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:03.158419719Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1822,"session_id":"1b26f2f7-74c7-4a3c-9395-a702f0027e97"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:03.158432128Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1822,"session_id":"1b26f2f7-74c7-4a3c-9395-a702f0027e97"} {"msg":"e509ac19-590a-470c-9ece-7adf25370620 is now active with session: 1b26f2f7-74c7-4a3c-9395-a702f0027e97","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:03.158444977Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1822,"session_id":"1b26f2f7-74c7-4a3c-9395-a702f0027e97"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:03.158457116Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1822,"session_id":"1b26f2f7-74c7-4a3c-9395-a702f0027e97"} {"msg":"The guest has finished waiting for activation with:160","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:03.158471695Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1822} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 543 len 12288 data: 10 11 10 03/30 Write block 56 len 40960 data: 22 22 19 18 17 11 11 12 10 9 04/30 Read block 345 len 16384 05/30 Flush 06/30 Write block 57 len 16384 data: 23 20 19 18 07/30 Write block 129 len 8192 data: 14 13 08/30 Read block 578 len 12288 09/30 Write block 206 len 4096 data: 16 10/30 Write block 272 len 36864 data: 15 14 16 14 12 11 14 12 10 11/30 Write block 121 len 24576 data: 14 14 14 12 14 14 12/30 Write block 120 len 20480 data: 14 15 15 15 13 13/30 Write block 42 len 20480 data: 10 10 10 12 10 14/30 Flush 15/30 Write block 378 len 4096 data: 12 16/30 Write block 433 len 20480 data: 14 11 15 12 11 17/30 Flush 18/30 Write block 475 len 32768 data: 12 12 10 11 11 8 9 9 19/30 Write block 215 len 28672 data: 13 10 12 11 13 13 10 20/30 Write block 16 len 20480 data: 10 10 11 8 11 21/30 Write block 138 len 24576 data: 14 8 9 9 12 13 22/30 Read block 270 len 36864 23/30 Read block 421 len 12288 24/30 Write block 470 len 36864 data: 12 10 8 10 11 13 13 11 12 25/30 Read block 541 len 8192 26/30 Read block 449 len 24576 27/30 Write block 462 len 40960 data: 20 18 18 16 14 13 12 12 13 11 28/30 Read block 76 len 40960 29/30 Write block 203 len 12288 data: 13 14 16 30/30 Read block 463 len 12288 ---------------------------------------------------------------- Crucible gen:160 GIO:true work queues: Upstairs:0 downstairs:13 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1017 Acked Write 8 Done Done Done false 1018 Acked Write 7 Done Done Done false 1019 Acked Write 5 Done Done Done false 1020 Acked Write 6 Done Done Done false 1021 Acked Read 9 Done Done Done false 1022 Acked Read 3 Done Done Done false 1023 Acked Write 9 Done Done Done false 1024 Acked Read 2 Done Done Done false 1025 Acked Read 6 Done Done Done false 1026 Acked Write 10 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Write 3 Done Done Done false 1029 Acked Read 3 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 0 1 Done 13 12 13 38 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1016 1016 1016 Downstairs last five completed: 1016 1015 1014 1013 1012 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:03.17711067Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1822} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 156 156 156 288 288 288 T T T 1 020-039 153 153 153 284 284 284 T T T 2 040-059 160 160 160 297 297 297 F F F 3 060-079 160 160 160 297 297 297 F F F 4 080-099 158 158 158 292 292 292 F F F 5 100-119 159 159 159 294 294 294 F F F 6 120-139 160 160 160 297 297 297 T T T 7 140-159 158 158 158 292 292 292 T T T 8 160-179 150 150 150 279 279 279 F F F 9 180-199 148 148 148 275 275 275 F F F 10 200-219 160 160 160 297 297 297 T T T 11 220-239 155 155 155 287 287 287 T T T 12 240-259 156 156 156 288 288 288 F F F 13 260-279 160 160 160 297 297 297 F F F 14 280-299 160 160 160 297 297 297 F F F 15 300-319 159 159 159 294 294 294 F F F 16 320-339 159 159 159 294 294 294 F F F 17 340-359 159 159 159 294 294 294 F F F 18 360-379 160 160 160 298 298 298 F F F 19 380-399 154 154 154 285 285 285 F F F 20 400-419 150 150 150 278 278 278 F F F 21 420-439 160 160 160 298 298 298 F F F 22 440-459 157 157 157 290 290 290 F F F 23 460-479 158 158 158 291 291 291 T T T 24 480-499 157 157 157 290 290 290 T T T 25 500-519 158 158 158 293 293 293 F F F 26 520-539 159 159 159 294 294 294 F F F 27 540-559 160 160 160 296 296 296 F F F 28 560-579 156 156 156 289 289 289 F F F 29 580-599 155 155 155 287 287 287 F F F Max gen: 160, Max flush: 298 A Difference in extent metadata is expected here On loop 80 Verifying data now No change after verify Loop: 80 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 161 161 161 299 299 299 F F F 1 020-039 161 161 161 299 299 299 F F F 2 040-059 160 160 160 297 297 297 F F F 3 060-079 160 160 160 297 297 297 F F F 4 080-099 158 158 158 292 292 292 F F F 5 100-119 159 159 159 294 294 294 F F F 6 120-139 161 161 161 299 299 299 F F F 7 140-159 161 161 161 299 299 299 F F F 8 160-179 150 150 150 279 279 279 F F F 9 180-199 148 148 148 275 275 275 F F F 10 200-219 161 161 161 299 299 299 F F F 11 220-239 161 161 161 299 299 299 F F F 12 240-259 156 156 156 288 288 288 F F F 13 260-279 160 160 160 297 297 297 F F F 14 280-299 160 160 160 297 297 297 F F F 15 300-319 159 159 159 294 294 294 F F F 16 320-339 159 159 159 294 294 294 F F F 17 340-359 159 159 159 294 294 294 F F F 18 360-379 160 160 160 298 298 298 F F F 19 380-399 154 154 154 285 285 285 F F F 20 400-419 150 150 150 278 278 278 F F F 21 420-439 160 160 160 298 298 298 F F F 22 440-459 157 157 157 290 290 290 F F F 23 460-479 161 161 161 299 299 299 F F F 24 480-499 161 161 161 299 299 299 F F F 25 500-519 158 158 158 293 293 293 F F F 26 520-539 159 159 159 294 294 294 F F F 27 540-559 160 160 160 296 296 296 F F F 28 560-579 156 156 156 289 289 289 F F F 29 580-599 155 155 155 287 287 287 F F F Max gen: 161, Max flush: 299 Begin loop 81 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:07.460433589Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1830,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:07.505032234Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1830,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:07.505716897Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1830} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"26e7d501b33f47379e53601720353c2983e7654b\",\n git_commit_timestamp: \"2024-12-12T20:16:40.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:07.505754074Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1830} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:07.505773083Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1830} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:07.505788142Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1830} {"msg":"Crucible 94fdfc68-41d3-4e93-be68-2562c2b8fe97 has session id: 489b2d32-a110-442f-afa3-3cf5d0dcafd2","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:07.508305228Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1830,"session_id":"489b2d32-a110-442f-afa3-3cf5d0dcafd2"} {"msg":"Upstairs opts: Upstairs UUID: 94fdfc68-41d3-4e93-be68-2562c2b8fe97, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:07.508339316Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1830,"session_id":"489b2d32-a110-442f-afa3-3cf5d0dcafd2"} {"msg":"Crucible stats registered with UUID: 94fdfc68-41d3-4e93-be68-2562c2b8fe97","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:07.508354825Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1830,"session_id":"489b2d32-a110-442f-afa3-3cf5d0dcafd2"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:07.539221077Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1830,"task":"crutest"} {"msg":"The guest has requested activation with gen:162","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:07.539252414Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1830} {"msg":"94fdfc68-41d3-4e93-be68-2562c2b8fe97 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:07.539284502Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1830,"session_id":"489b2d32-a110-442f-afa3-3cf5d0dcafd2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:07.539301311Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1830,"client":"0","":"downstairs","session_id":"489b2d32-a110-442f-afa3-3cf5d0dcafd2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:07.539330039Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1830,"client":"0","":"downstairs","session_id":"489b2d32-a110-442f-afa3-3cf5d0dcafd2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:07.539348168Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1830,"client":"1","":"downstairs","session_id":"489b2d32-a110-442f-afa3-3cf5d0dcafd2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:07.539366047Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1830,"client":"1","":"downstairs","session_id":"489b2d32-a110-442f-afa3-3cf5d0dcafd2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:07.539396424Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1830,"client":"2","":"downstairs","session_id":"489b2d32-a110-442f-afa3-3cf5d0dcafd2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:07.539425733Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1830,"client":"2","":"downstairs","session_id":"489b2d32-a110-442f-afa3-3cf5d0dcafd2"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:07.539450471Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1830,"":"io task","client":"2","":"downstairs","session_id":"489b2d32-a110-442f-afa3-3cf5d0dcafd2"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:07.539471269Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1830,"":"io task","client":"0","":"downstairs","session_id":"489b2d32-a110-442f-afa3-3cf5d0dcafd2"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:07.539498548Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1830,"":"io task","client":"1","":"downstairs","session_id":"489b2d32-a110-442f-afa3-3cf5d0dcafd2"} {"msg":"ds_connection connected from Ok(127.0.0.1:54084)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:07.539586202Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1830,"":"io task","client":"2","":"downstairs","session_id":"489b2d32-a110-442f-afa3-3cf5d0dcafd2"} {"msg":"ds_connection connected from Ok(127.0.0.1:40661)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:07.539624739Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1830,"":"io task","client":"0","":"downstairs","session_id":"489b2d32-a110-442f-afa3-3cf5d0dcafd2"} {"msg":"ds_connection connected from Ok(127.0.0.1:37275)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:07.539656617Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1830,"":"io task","client":"1","":"downstairs","session_id":"489b2d32-a110-442f-afa3-3cf5d0dcafd2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:07.540368177Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1830,"client":"0","":"downstairs","session_id":"489b2d32-a110-442f-afa3-3cf5d0dcafd2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:07.540409555Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1830,"client":"2","":"downstairs","session_id":"489b2d32-a110-442f-afa3-3cf5d0dcafd2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:07.540486369Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1830,"client":"1","":"downstairs","session_id":"489b2d32-a110-442f-afa3-3cf5d0dcafd2"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:07.541035831Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1830,"client":"0","":"downstairs","session_id":"489b2d32-a110-442f-afa3-3cf5d0dcafd2"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:07.541191071Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1830,"client":"1","":"downstairs","session_id":"489b2d32-a110-442f-afa3-3cf5d0dcafd2"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:07.541226308Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1830,"client":"2","":"downstairs","session_id":"489b2d32-a110-442f-afa3-3cf5d0dcafd2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:07.541525718Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1830,"session_id":"489b2d32-a110-442f-afa3-3cf5d0dcafd2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:07.541771911Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1830,"session_id":"489b2d32-a110-442f-afa3-3cf5d0dcafd2"} {"msg":"[0]R flush_numbers[0..12]: [299, 299, 297, 297, 292, 294, 299, 299, 279, 275, 299, 299]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:07.541839906Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1830,"":"downstairs","":"downstairs","session_id":"489b2d32-a110-442f-afa3-3cf5d0dcafd2"} {"msg":"[0]R generation[0..12]: [161, 161, 160, 160, 158, 159, 161, 161, 150, 148, 161, 161]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:07.541863704Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1830,"":"downstairs","":"downstairs","session_id":"489b2d32-a110-442f-afa3-3cf5d0dcafd2"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:07.541879053Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1830,"":"downstairs","":"downstairs","session_id":"489b2d32-a110-442f-afa3-3cf5d0dcafd2"} {"msg":"[1]R flush_numbers[0..12]: [299, 299, 297, 297, 292, 294, 299, 299, 279, 275, 299, 299]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:07.541893322Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1830,"":"downstairs","":"downstairs","session_id":"489b2d32-a110-442f-afa3-3cf5d0dcafd2"} {"msg":"[1]R generation[0..12]: [161, 161, 160, 160, 158, 159, 161, 161, 150, 148, 161, 161]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:07.541907042Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1830,"":"downstairs","":"downstairs","session_id":"489b2d32-a110-442f-afa3-3cf5d0dcafd2"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:07.541921081Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1830,"":"downstairs","":"downstairs","session_id":"489b2d32-a110-442f-afa3-3cf5d0dcafd2"} {"msg":"[2]R flush_numbers[0..12]: [299, 299, 297, 297, 292, 294, 299, 299, 279, 275, 299, 299]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:07.54193602Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1830,"":"downstairs","":"downstairs","session_id":"489b2d32-a110-442f-afa3-3cf5d0dcafd2"} {"msg":"[2]R generation[0..12]: [161, 161, 160, 160, 158, 159, 161, 161, 150, 148, 161, 161]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:07.541961438Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1830,"":"downstairs","":"downstairs","session_id":"489b2d32-a110-442f-afa3-3cf5d0dcafd2"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:07.541990776Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1830,"":"downstairs","":"downstairs","session_id":"489b2d32-a110-442f-afa3-3cf5d0dcafd2"} {"msg":"Max found gen is 162","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:07.542012004Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1830,"":"downstairs","":"downstairs","session_id":"489b2d32-a110-442f-afa3-3cf5d0dcafd2"} {"msg":"Generation requested: 162 >= found:162","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:07.542037282Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1830,"":"downstairs","":"downstairs","session_id":"489b2d32-a110-442f-afa3-3cf5d0dcafd2"} Wait for a query_work_queue command to finish before sending IO {"msg":"Next flush: 300","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:07.542058291Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1830,"":"downstairs","":"downstairs","session_id":"489b2d32-a110-442f-afa3-3cf5d0dcafd2"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:07.542085629Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1830,"":"downstairs","":"downstairs","session_id":"489b2d32-a110-442f-afa3-3cf5d0dcafd2"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:07.542100508Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1830,"session_id":"489b2d32-a110-442f-afa3-3cf5d0dcafd2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:07.542122016Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1830,"client":"0","":"downstairs","session_id":"489b2d32-a110-442f-afa3-3cf5d0dcafd2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:07.542144965Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1830,"client":"1","":"downstairs","session_id":"489b2d32-a110-442f-afa3-3cf5d0dcafd2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:07.542158834Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1830,"client":"2","":"downstairs","session_id":"489b2d32-a110-442f-afa3-3cf5d0dcafd2"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:07.542174033Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1830,"session_id":"489b2d32-a110-442f-afa3-3cf5d0dcafd2"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:07.542196791Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1830,"session_id":"489b2d32-a110-442f-afa3-3cf5d0dcafd2"} {"msg":"94fdfc68-41d3-4e93-be68-2562c2b8fe97 is now active with session: 489b2d32-a110-442f-afa3-3cf5d0dcafd2","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:07.54221463Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1830,"session_id":"489b2d32-a110-442f-afa3-3cf5d0dcafd2"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:07.542227219Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1830,"session_id":"489b2d32-a110-442f-afa3-3cf5d0dcafd2"} {"msg":"The guest has finished waiting for activation with:162","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:07.542239499Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1830} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 218 len 32768 data: 12 14 14 11 9 12 11 12 03/30 Read block 189 len 24576 04/30 Read block 299 len 36864 05/30 Write block 340 len 36864 data: 12 9 12 14 18 17 17 16 14 06/30 Write block 489 len 32768 data: 19 17 12 11 10 9 8 8 07/30 Read block 359 len 8192 08/30 Write block 553 len 36864 data: 9 10 11 11 11 9 12 8 8 09/30 Write block 366 len 24576 data: 19 16 12 11 9 7 10/30 Read block 149 len 16384 11/30 Write block 589 len 32768 data: 17 21 19 16 13 8 7 5 12/30 Read block 503 len 36864 13/30 Read block 573 len 28672 14/30 Read block 449 len 16384 15/30 Flush 16/30 Write block 585 len 12288 data: 17 19 19 17/30 Read block 384 len 24576 18/30 Read block 589 len 28672 19/30 Read block 403 len 32768 20/30 Write block 232 len 16384 data: 14 14 14 14 21/30 Read block 576 len 4096 22/30 Write block 309 len 8192 data: 12 10 23/30 Write block 137 len 8192 data: 12 15 24/30 Flush 25/30 Write block 232 len 40960 data: 15 15 15 15 15 14 14 15 14 14 26/30 Read block 261 len 28672 27/30 Read block 390 len 24576 28/30 Write block 454 len 20480 data: 15 14 14 16 19 29/30 Read block 387 len 8192 30/30 Write block 554 len 4096 data: 11 ---------------------------------------------------------------- Crucible gen:162 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 10 Done Done Done false 1025 Acked Read 7 Done Done Done false 1026 Acked Read 6 Done Done Done false 1027 Acked Write 5 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:07.562366371Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1830} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 161 161 161 299 299 299 F F F 1 020-039 161 161 161 299 299 299 F F F 2 040-059 160 160 160 297 297 297 F F F 3 060-079 160 160 160 297 297 297 F F F 4 080-099 158 158 158 292 292 292 F F F 5 100-119 159 159 159 294 294 294 F F F 6 120-139 162 162 162 302 302 302 F F F 7 140-159 161 161 161 299 299 299 F F F 8 160-179 150 150 150 279 279 279 F F F 9 180-199 148 148 148 275 275 275 F F F 10 200-219 162 162 162 301 301 301 F F F 11 220-239 162 162 162 302 302 302 T T T 12 240-259 156 156 156 288 288 288 T T T 13 260-279 160 160 160 297 297 297 F F F 14 280-299 160 160 160 297 297 297 F F F 15 300-319 162 162 162 302 302 302 F F F 16 320-339 159 159 159 294 294 294 F F F 17 340-359 162 162 162 301 301 301 F F F 18 360-379 162 162 162 301 301 301 F F F 19 380-399 154 154 154 285 285 285 F F F 20 400-419 150 150 150 278 278 278 F F F 21 420-439 160 160 160 298 298 298 F F F 22 440-459 157 157 157 290 290 290 T T T 23 460-479 161 161 161 299 299 299 F F F 24 480-499 162 162 162 301 301 301 F F F 25 500-519 158 158 158 293 293 293 F F F 26 520-539 159 159 159 294 294 294 F F F 27 540-559 162 162 162 301 301 301 T T T 28 560-579 162 162 162 301 301 301 F F F 29 580-599 162 162 162 302 302 302 F F F Max gen: 162, Max flush: 302 A Difference in extent metadata is expected here On loop 81 Verifying data now No change after verify Loop: 81 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 161 161 161 299 299 299 F F F 1 020-039 161 161 161 299 299 299 F F F 2 040-059 160 160 160 297 297 297 F F F 3 060-079 160 160 160 297 297 297 F F F 4 080-099 158 158 158 292 292 292 F F F 5 100-119 159 159 159 294 294 294 F F F 6 120-139 162 162 162 302 302 302 F F F 7 140-159 161 161 161 299 299 299 F F F 8 160-179 150 150 150 279 279 279 F F F 9 180-199 148 148 148 275 275 275 F F F 10 200-219 162 162 162 301 301 301 F F F 11 220-239 163 163 163 303 303 303 F F F 12 240-259 163 163 163 303 303 303 F F F 13 260-279 160 160 160 297 297 297 F F F 14 280-299 160 160 160 297 297 297 F F F 15 300-319 162 162 162 302 302 302 F F F 16 320-339 159 159 159 294 294 294 F F F 17 340-359 162 162 162 301 301 301 F F F 18 360-379 162 162 162 301 301 301 F F F 19 380-399 154 154 154 285 285 285 F F F 20 400-419 150 150 150 278 278 278 F F F 21 420-439 160 160 160 298 298 298 F F F 22 440-459 163 163 163 303 303 303 F F F 23 460-479 161 161 161 299 299 299 F F F 24 480-499 162 162 162 301 301 301 F F F 25 500-519 158 158 158 293 293 293 F F F 26 520-539 159 159 159 294 294 294 F F F 27 540-559 163 163 163 303 303 303 F F F 28 560-579 162 162 162 301 301 301 F F F 29 580-599 162 162 162 302 302 302 F F F Max gen: 163, Max flush: 303 Begin loop 82 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:11.830669985Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1838,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:11.874709948Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1838,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:11.875602697Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1838} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"26e7d501b33f47379e53601720353c2983e7654b\",\n git_commit_timestamp: \"2024-12-12T20:16:40.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:11.875639534Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1838} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:11.875653703Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1838} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:11.875668162Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1838} {"msg":"Crucible ac34cc1a-2a69-4c24-b1cc-c46dc4bd291c has session id: 52e21f8c-a12b-4c4e-be72-29a5051da77c","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:11.876097193Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1838,"session_id":"52e21f8c-a12b-4c4e-be72-29a5051da77c"} {"msg":"Upstairs opts: Upstairs UUID: ac34cc1a-2a69-4c24-b1cc-c46dc4bd291c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:11.87614491Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1838,"session_id":"52e21f8c-a12b-4c4e-be72-29a5051da77c"} {"msg":"Crucible stats registered with UUID: ac34cc1a-2a69-4c24-b1cc-c46dc4bd291c","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:11.876163478Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1838,"session_id":"52e21f8c-a12b-4c4e-be72-29a5051da77c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:11.907447111Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1838,"task":"crutest"} {"msg":"The guest has requested activation with gen:164","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:11.907484748Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1838} {"msg":"ac34cc1a-2a69-4c24-b1cc-c46dc4bd291c active request set","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:11.907566343Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1838,"session_id":"52e21f8c-a12b-4c4e-be72-29a5051da77c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:11.907589681Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1838,"client":"0","":"downstairs","session_id":"52e21f8c-a12b-4c4e-be72-29a5051da77c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:11.907617049Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1838,"client":"0","":"downstairs","session_id":"52e21f8c-a12b-4c4e-be72-29a5051da77c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:11.907638668Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1838,"client":"1","":"downstairs","session_id":"52e21f8c-a12b-4c4e-be72-29a5051da77c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:11.907656657Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1838,"client":"1","":"downstairs","session_id":"52e21f8c-a12b-4c4e-be72-29a5051da77c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:11.907672916Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1838,"client":"2","":"downstairs","session_id":"52e21f8c-a12b-4c4e-be72-29a5051da77c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:11.907690124Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1838,"client":"2","":"downstairs","session_id":"52e21f8c-a12b-4c4e-be72-29a5051da77c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:11.907706523Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1838,"":"io task","client":"2","":"downstairs","session_id":"52e21f8c-a12b-4c4e-be72-29a5051da77c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:11.907732011Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1838,"":"io task","client":"0","":"downstairs","session_id":"52e21f8c-a12b-4c4e-be72-29a5051da77c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:11.907764729Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1838,"":"io task","client":"1","":"downstairs","session_id":"52e21f8c-a12b-4c4e-be72-29a5051da77c"} {"msg":"ds_connection connected from Ok(127.0.0.1:35293)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:11.907796977Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1838,"":"io task","client":"2","":"downstairs","session_id":"52e21f8c-a12b-4c4e-be72-29a5051da77c"} {"msg":"ds_connection connected from Ok(127.0.0.1:36689)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:11.907830765Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1838,"":"io task","client":"0","":"downstairs","session_id":"52e21f8c-a12b-4c4e-be72-29a5051da77c"} {"msg":"ds_connection connected from Ok(127.0.0.1:39454)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:11.907867052Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1838,"":"io task","client":"1","":"downstairs","session_id":"52e21f8c-a12b-4c4e-be72-29a5051da77c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:11.908599741Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1838,"client":"0","":"downstairs","session_id":"52e21f8c-a12b-4c4e-be72-29a5051da77c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:11.908627799Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1838,"client":"1","":"downstairs","session_id":"52e21f8c-a12b-4c4e-be72-29a5051da77c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:11.908759821Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1838,"client":"2","":"downstairs","session_id":"52e21f8c-a12b-4c4e-be72-29a5051da77c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:11.909220469Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1838,"client":"1","":"downstairs","session_id":"52e21f8c-a12b-4c4e-be72-29a5051da77c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:11.909262496Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1838,"client":"0","":"downstairs","session_id":"52e21f8c-a12b-4c4e-be72-29a5051da77c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:11.909394917Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1838,"client":"2","":"downstairs","session_id":"52e21f8c-a12b-4c4e-be72-29a5051da77c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:11.909709955Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1838,"session_id":"52e21f8c-a12b-4c4e-be72-29a5051da77c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:11.909797479Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1838,"session_id":"52e21f8c-a12b-4c4e-be72-29a5051da77c"} {"msg":"[0]R flush_numbers[0..12]: [299, 299, 297, 297, 292, 294, 302, 299, 279, 275, 301, 303]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:11.909957848Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1838,"":"downstairs","":"downstairs","session_id":"52e21f8c-a12b-4c4e-be72-29a5051da77c"} {"msg":"[0]R generation[0..12]: [161, 161, 160, 160, 158, 159, 162, 161, 150, 148, 162, 163]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:11.909977026Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1838,"":"downstairs","":"downstairs","session_id":"52e21f8c-a12b-4c4e-be72-29a5051da77c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:11.909991785Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1838,"":"downstairs","":"downstairs","session_id":"52e21f8c-a12b-4c4e-be72-29a5051da77c"} {"msg":"[1]R flush_numbers[0..12]: [299, 299, 297, 297, 292, 294, 302, 299, 279, 275, 301, 303]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:11.910005895Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1838,"":"downstairs","":"downstairs","session_id":"52e21f8c-a12b-4c4e-be72-29a5051da77c"} {"msg":"[1]R generation[0..12]: [161, 161, 160, 160, 158, 159, 162, 161, 150, 148, 162, 163]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:11.910019734Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1838,"":"downstairs","":"downstairs","session_id":"52e21f8c-a12b-4c4e-be72-29a5051da77c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:11.910033573Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1838,"":"downstairs","":"downstairs","session_id":"52e21f8c-a12b-4c4e-be72-29a5051da77c"} {"msg":"[2]R flush_numbers[0..12]: [299, 299, 297, 297, 292, 294, 302, 299, 279, 275, 301, 303]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:11.910047482Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1838,"":"downstairs","":"downstairs","session_id":"52e21f8c-a12b-4c4e-be72-29a5051da77c"} {"msg":"[2]R generation[0..12]: [161, 161, 160, 160, 158, 159, 162, 161, 150, 148, 162, 163]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:11.910061471Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1838,"":"downstairs","":"downstairs","session_id":"52e21f8c-a12b-4c4e-be72-29a5051da77c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:11.91007534Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1838,"":"downstairs","":"downstairs","session_id":"52e21f8c-a12b-4c4e-be72-29a5051da77c"} {"msg":"Max found gen is 164","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:11.910088369Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1838,"":"downstairs","":"downstairs","session_id":"52e21f8c-a12b-4c4e-be72-29a5051da77c"} {"msg":"Generation requested: 164 >= found:164","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:11.910105998Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1838,"":"downstairs","":"downstairs","session_id":"52e21f8c-a12b-4c4e-be72-29a5051da77c"} {"msg":"Next flush: 304","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:11.910119457Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1838,"":"downstairs","":"downstairs","session_id":"52e21f8c-a12b-4c4e-be72-29a5051da77c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:11.910132336Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1838,"":"downstairs","":"downstairs","session_id":"52e21f8c-a12b-4c4e-be72-29a5051da77c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:11.910146225Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1838,"session_id":"52e21f8c-a12b-4c4e-be72-29a5051da77c"} Wait for a query_work_queue command to finish before sending IO {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:11.910161974Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1838,"client":"0","":"downstairs","session_id":"52e21f8c-a12b-4c4e-be72-29a5051da77c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:11.910175463Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1838,"client":"1","":"downstairs","session_id":"52e21f8c-a12b-4c4e-be72-29a5051da77c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:11.910188492Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1838,"client":"2","":"downstairs","session_id":"52e21f8c-a12b-4c4e-be72-29a5051da77c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:11.910201541Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1838,"session_id":"52e21f8c-a12b-4c4e-be72-29a5051da77c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:11.91021377Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1838,"session_id":"52e21f8c-a12b-4c4e-be72-29a5051da77c"} {"msg":"ac34cc1a-2a69-4c24-b1cc-c46dc4bd291c is now active with session: 52e21f8c-a12b-4c4e-be72-29a5051da77c","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:11.910226469Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1838,"session_id":"52e21f8c-a12b-4c4e-be72-29a5051da77c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:11.910238348Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1838,"session_id":"52e21f8c-a12b-4c4e-be72-29a5051da77c"} {"msg":"The guest has finished waiting for activation with:164","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:11.910250298Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1838} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 449 len 40960 data: 19 18 19 17 13 16 15 15 17 20 02/30 Read block 110 len 36864 03/30 Read block 291 len 8192 04/30 Read block 53 len 32768 05/30 Write block 207 len 16384 data: 11 14 12 15 06/30 Flush 07/30 Write block 416 len 28672 data: 12 10 11 11 11 11 12 08/30 Read block 148 len 32768 09/30 Write block 504 len 16384 data: 13 12 13 13 10/30 Read block 40 len 24576 11/30 Write block 547 len 28672 data: 10 10 11 11 9 8 10 12/30 Read block 171 len 24576 13/30 Write block 419 len 16384 data: 12 12 12 13 14/30 Read block 290 len 4096 15/30 Read block 398 len 4096 16/30 Write block 65 len 28672 data: 10 11 10 10 10 11 10 17/30 Flush 18/30 Write block 37 len 4096 data: 12 19/30 Read block 198 len 8192 20/30 Read block 328 len 12288 21/30 Flush 22/30 Write block 151 len 32768 data: 12 14 14 13 11 11 11 9 23/30 Read block 396 len 8192 24/30 Read block 349 len 20480 25/30 Read block 231 len 12288 26/30 Flush 27/30 Write block 513 len 4096 data: 12 28/30 Read block 371 len 24576 29/30 Read block 29 len 4096 30/30 Write block 9 len 40960 data: 15 14 15 13 10 11 12 11 11 12 ---------------------------------------------------------------- Crucible gen:164 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 1 Done Done Done false 1027 Acked Read 6 Done Done Done false 1028 Acked Read 1 Done Done Done false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:11.930295245Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1838} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 161 161 161 299 299 299 T T T 1 020-039 164 164 164 306 306 306 F F F 2 040-059 160 160 160 297 297 297 F F F 3 060-079 164 164 164 305 305 305 F F F 4 080-099 158 158 158 292 292 292 F F F 5 100-119 159 159 159 294 294 294 F F F 6 120-139 162 162 162 302 302 302 F F F 7 140-159 164 164 164 307 307 307 F F F 8 160-179 150 150 150 279 279 279 F F F 9 180-199 148 148 148 275 275 275 F F F 10 200-219 164 164 164 304 304 304 F F F 11 220-239 163 163 163 303 303 303 F F F 12 240-259 163 163 163 303 303 303 F F F 13 260-279 160 160 160 297 297 297 F F F 14 280-299 160 160 160 297 297 297 F F F 15 300-319 162 162 162 302 302 302 F F F 16 320-339 159 159 159 294 294 294 F F F 17 340-359 162 162 162 301 301 301 F F F 18 360-379 162 162 162 301 301 301 F F F 19 380-399 154 154 154 285 285 285 F F F 20 400-419 164 164 164 305 305 305 F F F 21 420-439 164 164 164 305 305 305 F F F 22 440-459 164 164 164 304 304 304 F F F 23 460-479 161 161 161 299 299 299 F F F 24 480-499 162 162 162 301 301 301 F F F 25 500-519 164 164 164 305 305 305 T T T 26 520-539 159 159 159 294 294 294 F F F 27 540-559 164 164 164 305 305 305 F F F 28 560-579 162 162 162 301 301 301 F F F 29 580-599 162 162 162 302 302 302 F F F Max gen: 164, Max flush: 307 A Difference in extent metadata is expected here On loop 82 Verifying data now No change after verify Loop: 82 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 165 165 165 308 308 308 F F F 1 020-039 164 164 164 306 306 306 F F F 2 040-059 160 160 160 297 297 297 F F F 3 060-079 164 164 164 305 305 305 F F F 4 080-099 158 158 158 292 292 292 F F F 5 100-119 159 159 159 294 294 294 F F F 6 120-139 162 162 162 302 302 302 F F F 7 140-159 164 164 164 307 307 307 F F F 8 160-179 150 150 150 279 279 279 F F F 9 180-199 148 148 148 275 275 275 F F F 10 200-219 164 164 164 304 304 304 F F F 11 220-239 163 163 163 303 303 303 F F F 12 240-259 163 163 163 303 303 303 F F F 13 260-279 160 160 160 297 297 297 F F F 14 280-299 160 160 160 297 297 297 F F F 15 300-319 162 162 162 302 302 302 F F F 16 320-339 159 159 159 294 294 294 F F F 17 340-359 162 162 162 301 301 301 F F F 18 360-379 162 162 162 301 301 301 F F F 19 380-399 154 154 154 285 285 285 F F F 20 400-419 164 164 164 305 305 305 F F F 21 420-439 164 164 164 305 305 305 F F F 22 440-459 164 164 164 304 304 304 F F F 23 460-479 161 161 161 299 299 299 F F F 24 480-499 162 162 162 301 301 301 F F F 25 500-519 165 165 165 308 308 308 F F F 26 520-539 159 159 159 294 294 294 F F F 27 540-559 164 164 164 305 305 305 F F F 28 560-579 162 162 162 301 301 301 F F F 29 580-599 162 162 162 302 302 302 F F F Max gen: 165, Max flush: 308 Begin loop 83 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:16.18251514Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1846,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:16.226920568Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1846,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:16.227593001Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1846} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"26e7d501b33f47379e53601720353c2983e7654b\",\n git_commit_timestamp: \"2024-12-12T20:16:40.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:16.227646108Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1846} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:16.227670906Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1846} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:16.227692134Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1846} {"msg":"Crucible a2ba4836-6d0e-402f-8203-da7478670ecc has session id: 7606b08c-ca99-4e39-9aa0-36086422fefc","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:16.228106876Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1846,"session_id":"7606b08c-ca99-4e39-9aa0-36086422fefc"} {"msg":"Upstairs opts: Upstairs UUID: a2ba4836-6d0e-402f-8203-da7478670ecc, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:16.228145153Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1846,"session_id":"7606b08c-ca99-4e39-9aa0-36086422fefc"} {"msg":"Crucible stats registered with UUID: a2ba4836-6d0e-402f-8203-da7478670ecc","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:16.228161512Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1846,"session_id":"7606b08c-ca99-4e39-9aa0-36086422fefc"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:16.258852616Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1846,"task":"crutest"} {"msg":"The guest has requested activation with gen:166","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:16.258891283Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1846} {"msg":"a2ba4836-6d0e-402f-8203-da7478670ecc active request set","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:16.258961238Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1846,"session_id":"7606b08c-ca99-4e39-9aa0-36086422fefc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:16.258996216Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1846,"client":"0","":"downstairs","session_id":"7606b08c-ca99-4e39-9aa0-36086422fefc"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:16.259026394Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1846,"client":"0","":"downstairs","session_id":"7606b08c-ca99-4e39-9aa0-36086422fefc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:16.259045442Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1846,"client":"1","":"downstairs","session_id":"7606b08c-ca99-4e39-9aa0-36086422fefc"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:16.259086329Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1846,"client":"1","":"downstairs","session_id":"7606b08c-ca99-4e39-9aa0-36086422fefc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:16.259119947Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1846,"client":"2","":"downstairs","session_id":"7606b08c-ca99-4e39-9aa0-36086422fefc"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:16.259154135Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1846,"client":"2","":"downstairs","session_id":"7606b08c-ca99-4e39-9aa0-36086422fefc"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:16.259186403Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1846,"":"io task","client":"2","":"downstairs","session_id":"7606b08c-ca99-4e39-9aa0-36086422fefc"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:16.25922064Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1846,"":"io task","client":"0","":"downstairs","session_id":"7606b08c-ca99-4e39-9aa0-36086422fefc"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:16.259252568Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1846,"":"io task","client":"1","":"downstairs","session_id":"7606b08c-ca99-4e39-9aa0-36086422fefc"} {"msg":"ds_connection connected from Ok(127.0.0.1:58758)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:16.259284596Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1846,"":"io task","client":"2","":"downstairs","session_id":"7606b08c-ca99-4e39-9aa0-36086422fefc"} {"msg":"ds_connection connected from Ok(127.0.0.1:34967)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:16.259461574Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1846,"":"io task","client":"1","":"downstairs","session_id":"7606b08c-ca99-4e39-9aa0-36086422fefc"} {"msg":"ds_connection connected from Ok(127.0.0.1:45544)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:16.25951367Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1846,"":"io task","client":"0","":"downstairs","session_id":"7606b08c-ca99-4e39-9aa0-36086422fefc"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:16.260037244Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1846,"client":"2","":"downstairs","session_id":"7606b08c-ca99-4e39-9aa0-36086422fefc"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:16.260067152Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1846,"client":"1","":"downstairs","session_id":"7606b08c-ca99-4e39-9aa0-36086422fefc"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:16.26008714Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1846,"client":"0","":"downstairs","session_id":"7606b08c-ca99-4e39-9aa0-36086422fefc"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:16.260713197Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1846,"client":"1","":"downstairs","session_id":"7606b08c-ca99-4e39-9aa0-36086422fefc"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:16.260742565Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1846,"client":"2","":"downstairs","session_id":"7606b08c-ca99-4e39-9aa0-36086422fefc"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:16.260765904Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1846,"client":"0","":"downstairs","session_id":"7606b08c-ca99-4e39-9aa0-36086422fefc"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:16.261303257Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1846,"session_id":"7606b08c-ca99-4e39-9aa0-36086422fefc"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:16.261340794Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1846,"session_id":"7606b08c-ca99-4e39-9aa0-36086422fefc"} {"msg":"[0]R flush_numbers[0..12]: [308, 306, 297, 305, 292, 294, 302, 307, 279, 275, 304, 303]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:16.261367272Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1846,"":"downstairs","":"downstairs","session_id":"7606b08c-ca99-4e39-9aa0-36086422fefc"} {"msg":"[0]R generation[0..12]: [165, 164, 160, 164, 158, 159, 162, 164, 150, 148, 164, 163]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:16.261383601Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1846,"":"downstairs","":"downstairs","session_id":"7606b08c-ca99-4e39-9aa0-36086422fefc"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:16.2613982Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1846,"":"downstairs","":"downstairs","session_id":"7606b08c-ca99-4e39-9aa0-36086422fefc"} {"msg":"[1]R flush_numbers[0..12]: [308, 306, 297, 305, 292, 294, 302, 307, 279, 275, 304, 303]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:16.261412479Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1846,"":"downstairs","":"downstairs","session_id":"7606b08c-ca99-4e39-9aa0-36086422fefc"} {"msg":"[1]R generation[0..12]: [165, 164, 160, 164, 158, 159, 162, 164, 150, 148, 164, 163]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:16.261426418Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1846,"":"downstairs","":"downstairs","session_id":"7606b08c-ca99-4e39-9aa0-36086422fefc"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:16.261440387Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1846,"":"downstairs","":"downstairs","session_id":"7606b08c-ca99-4e39-9aa0-36086422fefc"} {"msg":"[2]R flush_numbers[0..12]: [308, 306, 297, 305, 292, 294, 302, 307, 279, 275, 304, 303]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:16.261454216Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1846,"":"downstairs","":"downstairs","session_id":"7606b08c-ca99-4e39-9aa0-36086422fefc"} {"msg":"[2]R generation[0..12]: [165, 164, 160, 164, 158, 159, 162, 164, 150, 148, 164, 163]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:16.261470835Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1846,"":"downstairs","":"downstairs","session_id":"7606b08c-ca99-4e39-9aa0-36086422fefc"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:16.261485234Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1846,"":"downstairs","":"downstairs","session_id":"7606b08c-ca99-4e39-9aa0-36086422fefc"} {"msg":"Max found gen is 166","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:16.261498203Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1846,"":"downstairs","":"downstairs","session_id":"7606b08c-ca99-4e39-9aa0-36086422fefc"} {"msg":"Generation requested: 166 >= found:166","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:16.261511462Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1846,"":"downstairs","":"downstairs","session_id":"7606b08c-ca99-4e39-9aa0-36086422fefc"} {"msg":"Next flush: 309","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:16.261524372Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1846,"":"downstairs","":"downstairs","session_id":"7606b08c-ca99-4e39-9aa0-36086422fefc"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:16.26154661Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1846,"":"downstairs","":"downstairs","session_id":"7606b08c-ca99-4e39-9aa0-36086422fefc"} {"msgWait for a query_work_queue command to finish before sending IO ":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:16.261577278Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1846,"session_id":"7606b08c-ca99-4e39-9aa0-36086422fefc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:16.261599066Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1846,"client":"0","":"downstairs","session_id":"7606b08c-ca99-4e39-9aa0-36086422fefc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:16.261613645Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1846,"client":"1","":"downstairs","session_id":"7606b08c-ca99-4e39-9aa0-36086422fefc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:16.261626894Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1846,"client":"2","":"downstairs","session_id":"7606b08c-ca99-4e39-9aa0-36086422fefc"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:16.261639913Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1846,"session_id":"7606b08c-ca99-4e39-9aa0-36086422fefc"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:16.261652123Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1846,"session_id":"7606b08c-ca99-4e39-9aa0-36086422fefc"} {"msg":"a2ba4836-6d0e-402f-8203-da7478670ecc is now active with session: 7606b08c-ca99-4e39-9aa0-36086422fefc","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:16.261664902Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1846,"session_id":"7606b08c-ca99-4e39-9aa0-36086422fefc"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:16.261676851Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1846,"session_id":"7606b08c-ca99-4e39-9aa0-36086422fefc"} {"msg":"The guest has finished waiting for activation with:166","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:16.26168897Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1846} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 120 len 12288 data: 15 16 16 02/30 Write block 476 len 8192 data: 14 12 03/30 Read block 585 len 36864 04/30 Read block 31 len 12288 05/30 Read block 520 len 36864 06/30 Read block 121 len 8192 07/30 Write block 277 len 20480 data: 12 15 13 11 12 08/30 Write block 494 len 20480 data: 10 9 9 9 6 09/30 Read block 248 len 40960 10/30 Read block 270 len 24576 11/30 Read block 38 len 28672 12/30 Write block 516 len 28672 data: 9 12 17 20 20 20 18 13/30 Read block 57 len 36864 14/30 Write block 552 len 12288 data: 9 11 12 15/30 Read block 240 len 36864 16/30 Read block 261 len 20480 17/30 Flush 18/30 Write block 263 len 4096 data: 15 19/30 Read block 109 len 24576 20/30 Read block 329 len 4096 21/30 Read block 492 len 16384 22/30 Write block 211 len 36864 data: 17 14 16 14 14 11 13 13 15 23/30 Read block 151 len 20480 24/30 Read block 324 len 8192 25/30 Read block 403 len 40960 26/30 Read block 405 len 16384 27/30 Read block 223 len 32768 28/30 Read block 260 len 24576 29/30 Read block 355 len 40960 30/30 Write block 8 len 24576 data: 13 16 15 16 14 11 ---------------------------------------------------------------- Crucible gen:166 GIO:true work queues: Upstairs:0 downstairs:13 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1017 Acked Write 1 Done Done Done false 1018 Acked Read 6 Done Done Done false 1019 Acked Read 1 Done Done Done false 1020 Acked Read 4 Done Done Done false 1021 Acked Write 9 Done Done Done false 1022 Acked Read 5 Done Done Done false 1023 Acked Read 2 Done Done Done false 1024 Acked Read 10 Done Done Done false 1025 Acked Read 4 Done Done Done false 1026 Acked Read 8 Done Done Done false 1027 Acked Read 6 Done Done Done false 1028 Acked Read 10 Done Done Done false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 12 12 12 36 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1016 1016 1016 Downstairs last five completed: 1016 1015 1014 1013 1012 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:16.282115072Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1846} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 165 165 165 308 308 308 T T T 1 020-039 164 164 164 306 306 306 F F F 2 040-059 160 160 160 297 297 297 F F F 3 060-079 164 164 164 305 305 305 F F F 4 080-099 158 158 158 292 292 292 F F F 5 100-119 159 159 159 294 294 294 F F F 6 120-139 166 166 166 309 309 309 F F F 7 140-159 164 164 164 307 307 307 F F F 8 160-179 150 150 150 279 279 279 F F F 9 180-199 148 148 148 275 275 275 F F F 10 200-219 164 164 164 304 304 304 T T T 11 220-239 163 163 163 303 303 303 F F F 12 240-259 163 163 163 303 303 303 F F F 13 260-279 166 166 166 309 309 309 T T T 14 280-299 166 166 166 309 309 309 F F F 15 300-319 162 162 162 302 302 302 F F F 16 320-339 159 159 159 294 294 294 F F F 17 340-359 162 162 162 301 301 301 F F F 18 360-379 162 162 162 301 301 301 F F F 19 380-399 154 154 154 285 285 285 F F F 20 400-419 164 164 164 305 305 305 F F F 21 420-439 164 164 164 305 305 305 F F F 22 440-459 164 164 164 304 304 304 F F F 23 460-479 166 166 166 309 309 309 F F F 24 480-499 166 166 166 309 309 309 F F F 25 500-519 166 166 166 309 309 309 F F F 26 520-539 166 166 166 309 309 309 F F F 27 540-559 166 166 166 309 309 309 F F F 28 560-579 162 162 162 301 301 301 F F F 29 580-599 162 162 162 302 302 302 F F F Max gen: 166, Max flush: 309 A Difference in extent metadata is expected here On loop 83 Verifying data now No change after verify Loop: 83 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 167 167 167 310 310 310 F F F 1 020-039 164 164 164 306 306 306 F F F 2 040-059 160 160 160 297 297 297 F F F 3 060-079 164 164 164 305 305 305 F F F 4 080-099 158 158 158 292 292 292 F F F 5 100-119 159 159 159 294 294 294 F F F 6 120-139 166 166 166 309 309 309 F F F 7 140-159 164 164 164 307 307 307 F F F 8 160-179 150 150 150 279 279 279 F F F 9 180-199 148 148 148 275 275 275 F F F 10 200-219 167 167 167 310 310 310 F F F 11 220-239 163 163 163 303 303 303 F F F 12 240-259 163 163 163 303 303 303 F F F 13 260-279 167 167 167 310 310 310 F F F 14 280-299 166 166 166 309 309 309 F F F 15 300-319 162 162 162 302 302 302 F F F 16 320-339 159 159 159 294 294 294 F F F 17 340-359 162 162 162 301 301 301 F F F 18 360-379 162 162 162 301 301 301 F F F 19 380-399 154 154 154 285 285 285 F F F 20 400-419 164 164 164 305 305 305 F F F 21 420-439 164 164 164 305 305 305 F F F 22 440-459 164 164 164 304 304 304 F F F 23 460-479 166 166 166 309 309 309 F F F 24 480-499 166 166 166 309 309 309 F F F 25 500-519 166 166 166 309 309 309 F F F 26 520-539 166 166 166 309 309 309 F F F 27 540-559 166 166 166 309 309 309 F F F 28 560-579 162 162 162 301 301 301 F F F 29 580-599 162 162 162 302 302 302 F F F Max gen: 167, Max flush: 310 Begin loop 84 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:20.543002948Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:20.58937131Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:20.589976339Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"26e7d501b33f47379e53601720353c2983e7654b\",\n git_commit_timestamp: \"2024-12-12T20:16:40.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:20.590044854Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:20.590067522Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:20.590091411Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854} {"msg":"Crucible 9818006f-7b32-4b58-b8ab-7abcd7b6e969 has session id: 7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:20.59054192Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"Upstairs opts: Upstairs UUID: 9818006f-7b32-4b58-b8ab-7abcd7b6e969, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:20.590591636Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"Crucible stats registered with UUID: 9818006f-7b32-4b58-b8ab-7abcd7b6e969","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:20.590611055Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:20.621937395Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"task":"crutest"} {"msg":"The guest has requested activation with gen:168","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:20.621965283Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854} {"msg":"9818006f-7b32-4b58-b8ab-7abcd7b6e969 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:20.622041138Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:20.622063406Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"client":"0","":"downstairs","session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:20.622091404Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"client":"0","":"downstairs","session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:20.622108613Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"client":"1","":"downstairs","session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:20.622126022Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"client":"1","":"downstairs","session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:20.622142751Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"client":"2","":"downstairs","session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:20.622172349Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"client":"2","":"downstairs","session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:20.622202357Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"":"io task","client":"2","":"downstairs","session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:20.622232884Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"":"io task","client":"0","":"downstairs","session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:20.622264272Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"":"io task","client":"1","":"downstairs","session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"ds_connection connected from Ok(127.0.0.1:56991)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:20.622485697Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"":"io task","client":"2","":"downstairs","session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"ds_connection connected from Ok(127.0.0.1:46705)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:20.622531184Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"":"io task","client":"0","":"downstairs","session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"ds_connection connected from Ok(127.0.0.1:34399)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:20.622567032Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"":"io task","client":"1","":"downstairs","session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:20.623004861Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"client":"0","":"downstairs","session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:20.623152111Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"client":"2","":"downstairs","session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:20.62331813Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"client":"1","":"downstairs","session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:20.623631068Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"client":"0","":"downstairs","session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:20.623664656Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"client":"2","":"downstairs","session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:20.623933987Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"client":"1","":"downstairs","session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:20.624272024Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:20.624297962Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"[0]R flush_numbers[0..12]: [310, 306, 297, 305, 292, 294, 309, 307, 279, 275, 310, 303]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:20.624532966Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"":"downstairs","":"downstairs","session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"[0]R generation[0..12]: [167, 164, 160, 164, 158, 159, 166, 164, 150, 148, 167, 163]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:20.624560894Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"":"downstairs","":"downstairs","session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:20.624576663Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"":"downstairs","":"downstairs","session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"[1]R flush_numbers[0..12]: [310, 306, 297, 305, 292, 294, 309, 307, 279, 275, 310, 303]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:20.624594622Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"":"downstairs","":"downstairs","session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"[1]R generation[0..12]: [167, 164, 160, 164, 158, 159, 166, 164, 150, 148, 167, 163]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:20.624608911Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"":"downstairs","":"downstairs","session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:20.624623Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"":"downstairs","":"downstairs","session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"[2]R flush_numbers[0..12]: [310, 306, 297, 305, 292, 294, 309, 307, 279, 275, 310, 303]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:20.624636879Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"":"downstairs","":"downstairs","session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"[2]R generation[0..12]: [167, 164, 160, 164, 158, 159, 166, 164, 150, 148, 167, 163]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:20.624650438Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"":"downstairs","":"downstairs","session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:20.624664197Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"":"downstairs","":"downstairs","session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"Max found gen is 168","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:20.624677056Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"":"downstairs","":"downstairs","session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"Generation requested: 168 >= found:168","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:20.624690135Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"":"downstairs","":"downstairs","session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"Next flush: 311","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:20.624703214Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"":"downstairs","":"downstairs","session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:20.624716223Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"":"downstairs","":"downstairs","session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":Wait for a query_work_queue command to finish before sending IO 30,"time":"2024-12-12T20:39:20.624734542Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:20.624747151Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"client":"0","":"downstairs","session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:20.62476033Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"client":"1","":"downstairs","session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:20.624773689Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"client":"2","":"downstairs","session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:20.624786678Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:20.624800587Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"9818006f-7b32-4b58-b8ab-7abcd7b6e969 is now active with session: 7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:20.624813177Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:20.624824976Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"The guest has finished waiting for activation with:168","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:20.624837215Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 491 len 36864 data: 13 12 11 11 10 10 10 7 6 02/30 Flush 03/30 Write block 237 len 4096 data: 15 04/30 Write block 501 len 36864 data: 9 10 11 14 13 14 14 11 11 05/30 Read block 572 len 16384 06/30 Read block 60 len 8192 07/30 Write block 105 len 12288 data: 9 9 12 08/30 Write block 259 len 12288 data: 11 14 10 09/30 Flush 10/30 Write block 254 len 8192 data: 14 14 11/30 Read block 23 len 24576 12/30 Write block 142 len 20480 data: 13 14 13 13 13 13/30 Flush 14/30 Write block 334 len 12288 data: 15 14 12 15/30 Read block 344 len 8192 16/30 Read block 498 len 36864 17/30 Read block 362 len 4096 18/30 Write block 162 len 36864 data: 12 12 13 11 11 8 8 9 9 19/30 Write block 400 len 36864 data: 15 17 15 15 13 12 13 15 16 20/30 Flush 21/30 Write block 173 len 32768 data: 8 8 11 11 12 11 13 10 22/30 Flush 23/30 Write block 161 len 40960 data: 9 13 13 14 12 12 9 9 10 10 24/30 Read block 14 len 36864 25/30 Read block 438 len 12288 26/30 Read block 530 len 4096 27/30 Write block 265 len 28672 data: 18 21 22 21 12 13 12 28/30 Write block 186 len 40960 data: 13 12 15 15 14 14 15 16 13 14 29/30 Write block 191 len 12288 data: 15 16 17 30/30 Write block 166 len 8192 data: 13 10 ---------------------------------------------------------------- Crucible gen:168 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 10 Done Done Done false 1023 Acked Read 9 Done Done Done false 1024 Acked Read 3 Done Done Done false 1025 Acked Read 1 Done Done Done false 1026 Acked Write 7 Done Done Done false 1027 Acked Write 10 Sent Sent Sent false 1028 Acked Write 3 Sent Sent Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 3 3 9 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1021 1021 1021 Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:20.643424713Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854} Wrote out file "/tmp/test_repair_verify.data" {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:20.644034501Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:20.644075598Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:20.644101996Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:20.644124145Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:20.644144373Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:20.644162872Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:20.644185501Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:20.644210719Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:20.644228698Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:20.644248676Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:20.644261525Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:20.644273515Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:20.644285804Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:20.644297953Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:20.644309892Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:20.644321841Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:20.644333771Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:20.64434596Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:20.644361319Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:20.644373918Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:20.644386207Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:20.644398526Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:20.644410835Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:20.644423084Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:20.644435293Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:20.644447533Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:20.644459812Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:20.644472051Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:20.64448436Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:20.644496389Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:20.644508519Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:20.644520398Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:20.644532397Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:20.644544416Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:20.644556335Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:20.644570844Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:20.644583353Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:20.644595333Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:20.644607302Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:20.644619321Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:20.64463141Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:20.644643429Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:20.644655378Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:20.644667357Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:20.644679437Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:20.644691456Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:20.644703395Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:20.644715354Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:20.644727293Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:20.644739372Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:20.644751352Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:20.644763251Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:20.64477781Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:20.644790139Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:20.644802128Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:20.644814117Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:20.644826077Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:20.644838196Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:20.644850275Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:20.644862344Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:20.644874193Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:20.644886342Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:20.644898341Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:20.644910441Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:20.64492237Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:20.644934349Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:20.644946428Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:20.644958668Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:20.644970996Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:20.644984536Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:20.645024253Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:20.645038322Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:20.645050731Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:20.6450628Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:20.645074929Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:20.645086948Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:20.645099068Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:20.645111077Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:20.645123056Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:20.645134995Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:20.645146835Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:20.645158714Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:20.645170563Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:20.645182662Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:20.645194501Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:20.64520641Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:20.64521847Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:20.645232429Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:20.645244788Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:20.645256867Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:20.645269096Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:20.645281225Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:20.645293344Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:20.645305493Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:20.645317473Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:20.645329502Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:20.645341581Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:20.64535382Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:20.645365999Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:20.645377899Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:20.645390008Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:20.645401987Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:20.645414016Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:20.645426005Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:20.645439484Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:20.645451463Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:20.645463363Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:20.645475362Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:20.645487311Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:20.64549936Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:20.645511299Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:20.645523189Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:20.645535268Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:20.645547207Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:20.645559166Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:20.645571095Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:20.645583074Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:20.645594924Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:20.645607013Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:20.645618902Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:20.645630941Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:20.64564445Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:20.645656529Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:20.645668508Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:20.645680348Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1854,"session_id":"7ad4d425-f5d7-43f8-9884-3bdf4e8d9de7"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 167 167 167 310 310 310 F F F 1 020-039 164 164 164 306 306 306 F F F 2 040-059 160 160 160 297 297 297 F F F 3 060-079 164 164 164 305 305 305 F F F 4 080-099 158 158 158 292 292 292 F F F 5 100-119 168 168 168 312 312 312 F F F 6 120-139 166 166 166 309 309 309 F F F 7 140-159 168 168 168 313 313 313 F F F 8 160-179 168 168 168 315 315 315 T T T 9 180-199 168 168 168 315 315 315 T T T 10 200-219 167 167 167 310 310 310 F F F 11 220-239 168 168 168 312 312 312 F F F 12 240-259 168 168 168 313 313 313 F F F 13 260-279 168 168 168 312 312 312 T T T 14 280-299 166 166 166 309 309 309 F F F 15 300-319 162 162 162 302 302 302 F F F 16 320-339 168 168 168 314 314 314 F F F 17 340-359 162 162 162 301 301 301 F F F 18 360-379 162 162 162 301 301 301 F F F 19 380-399 154 154 154 285 285 285 F F F 20 400-419 168 168 168 314 314 314 F F F 21 420-439 164 164 164 305 305 305 F F F 22 440-459 164 164 164 304 304 304 F F F 23 460-479 166 166 166 309 309 309 F F F 24 480-499 168 168 168 311 311 311 F F F 25 500-519 168 168 168 312 312 312 F F F 26 520-539 166 166 166 309 309 309 F F F 27 540-559 166 166 166 309 309 309 F F F 28 560-579 162 162 162 301 301 301 F F F 29 580-599 162 162 162 302 302 302 F F F Max gen: 168, Max flush: 315 A Difference in extent metadata is expected here On loop 84 Verifying data now No change after verify Loop: 84 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 167 167 167 310 310 310 F F F 1 020-039 164 164 164 306 306 306 F F F 2 040-059 160 160 160 297 297 297 F F F 3 060-079 164 164 164 305 305 305 F F F 4 080-099 158 158 158 292 292 292 F F F 5 100-119 168 168 168 312 312 312 F F F 6 120-139 166 166 166 309 309 309 F F F 7 140-159 168 168 168 313 313 313 F F F 8 160-179 169 169 169 316 316 316 F F F 9 180-199 169 169 169 316 316 316 F F F 10 200-219 167 167 167 310 310 310 F F F 11 220-239 168 168 168 312 312 312 F F F 12 240-259 168 168 168 313 313 313 F F F 13 260-279 169 169 169 316 316 316 F F F 14 280-299 166 166 166 309 309 309 F F F 15 300-319 162 162 162 302 302 302 F F F 16 320-339 168 168 168 314 314 314 F F F 17 340-359 162 162 162 301 301 301 F F F 18 360-379 162 162 162 301 301 301 F F F 19 380-399 154 154 154 285 285 285 F F F 20 400-419 168 168 168 314 314 314 F F F 21 420-439 164 164 164 305 305 305 F F F 22 440-459 164 164 164 304 304 304 F F F 23 460-479 166 166 166 309 309 309 F F F 24 480-499 168 168 168 311 311 311 F F F 25 500-519 168 168 168 312 312 312 F F F 26 520-539 166 166 166 309 309 309 F F F 27 540-559 166 166 166 309 309 309 F F F 28 560-579 162 162 162 301 301 301 F F F 29 580-599 162 162 162 302 302 302 F F F Max gen: 169, Max flush: 316 Begin loop 85 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:24.990413052Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1862,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:25.033426906Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1862,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:25.033816779Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1862,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:25.065391102Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1862,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:25.065988301Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1862} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"26e7d501b33f47379e53601720353c2983e7654b\",\n git_commit_timestamp: \"2024-12-12T20:16:40.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:25.066021749Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1862} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:25.066035917Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1862} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:25.066050436Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1862} {"msg":"Crucible 74b78b13-bffc-41cd-9e30-c227a417e224 has session id: 264ccec3-b1dc-4db6-baf7-67baf183e9f0","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:25.066573871Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1862,"session_id":"264ccec3-b1dc-4db6-baf7-67baf183e9f0"} {"msg":"Upstairs opts: Upstairs UUID: 74b78b13-bffc-41cd-9e30-c227a417e224, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:25.066610838Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1862,"session_id":"264ccec3-b1dc-4db6-baf7-67baf183e9f0"} {"msg":"Crucible stats registered with UUID: 74b78b13-bffc-41cd-9e30-c227a417e224","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:25.066627037Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1862,"session_id":"264ccec3-b1dc-4db6-baf7-67baf183e9f0"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:25.098000713Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1862,"task":"crutest"} {"msg":"The guest has requested activation with gen:170","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:25.098032221Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1862} {"msg":"74b78b13-bffc-41cd-9e30-c227a417e224 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:25.098097447Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1862,"session_id":"264ccec3-b1dc-4db6-baf7-67baf183e9f0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:25.098119145Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1862,"client":"0","":"downstairs","session_id":"264ccec3-b1dc-4db6-baf7-67baf183e9f0"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:25.098147103Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1862,"client":"0","":"downstairs","session_id":"264ccec3-b1dc-4db6-baf7-67baf183e9f0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:25.098164672Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1862,"client":"1","":"downstairs","session_id":"264ccec3-b1dc-4db6-baf7-67baf183e9f0"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:25.098182191Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1862,"client":"1","":"downstairs","session_id":"264ccec3-b1dc-4db6-baf7-67baf183e9f0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:25.09819874Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1862,"client":"2","":"downstairs","session_id":"264ccec3-b1dc-4db6-baf7-67baf183e9f0"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:25.098216458Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1862,"client":"2","":"downstairs","session_id":"264ccec3-b1dc-4db6-baf7-67baf183e9f0"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:25.098232767Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1862,"":"io task","client":"2","":"downstairs","session_id":"264ccec3-b1dc-4db6-baf7-67baf183e9f0"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:25.098249656Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1862,"":"io task","client":"0","":"downstairs","session_id":"264ccec3-b1dc-4db6-baf7-67baf183e9f0"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:25.098266505Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1862,"":"io task","client":"1","":"downstairs","session_id":"264ccec3-b1dc-4db6-baf7-67baf183e9f0"} {"msg":"ds_connection connected from Ok(127.0.0.1:56975)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:25.098453222Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1862,"":"io task","client":"1","":"downstairs","session_id":"264ccec3-b1dc-4db6-baf7-67baf183e9f0"} {"msg":"ds_connection connected from Ok(127.0.0.1:58397)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:25.098496799Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1862,"":"io task","client":"2","":"downstairs","session_id":"264ccec3-b1dc-4db6-baf7-67baf183e9f0"} {"msg":"ds_connection connected from Ok(127.0.0.1:46872)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:25.098577604Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1862,"":"io task","client":"0","":"downstairs","session_id":"264ccec3-b1dc-4db6-baf7-67baf183e9f0"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:25.09920301Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1862,"client":"2","":"downstairs","session_id":"264ccec3-b1dc-4db6-baf7-67baf183e9f0"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:25.099247287Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1862,"client":"1","":"downstairs","session_id":"264ccec3-b1dc-4db6-baf7-67baf183e9f0"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:25.099463092Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1862,"client":"0","":"downstairs","session_id":"264ccec3-b1dc-4db6-baf7-67baf183e9f0"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:25.099760022Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1862,"client":"1","":"downstairs","session_id":"264ccec3-b1dc-4db6-baf7-67baf183e9f0"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:25.0999316Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1862,"client":"2","":"downstairs","session_id":"264ccec3-b1dc-4db6-baf7-67baf183e9f0"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:25.100172724Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1862,"client":"0","":"downstairs","session_id":"264ccec3-b1dc-4db6-baf7-67baf183e9f0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:25.100361201Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1862,"session_id":"264ccec3-b1dc-4db6-baf7-67baf183e9f0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:25.100456604Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1862,"session_id":"264ccec3-b1dc-4db6-baf7-67baf183e9f0"} {"msg":"[0]R flush_numbers[0..12]: [310, 306, 297, 305, 292, 312, 309, 313, 316, 316, 310, 312]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:25.100775702Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1862,"":"downstairs","":"downstairs","session_id":"264ccec3-b1dc-4db6-baf7-67baf183e9f0"} {"msg":"[0]R generation[0..12]: [167, 164, 160, 164, 158, 168, 166, 168, 169, 169, 167, 168]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:25.100795171Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1862,"":"downstairs","":"downstairs","session_id":"264ccec3-b1dc-4db6-baf7-67baf183e9f0"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:25.10080993Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1862,"":"downstairs","":"downstairs","session_id":"264ccec3-b1dc-4db6-baf7-67baf183e9f0"} {"msg":"[1]R flush_numbers[0..12]: [310, 306, 297, 305, 292, 312, 309, 313, 316, 316, 310, 312]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:25.100824099Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1862,"":"downstairs","":"downstairs","session_id":"264ccec3-b1dc-4db6-baf7-67baf183e9f0"} {"msg":"[1]R generation[0..12]: [167, 164, 160, 164, 158, 168, 166, 168, 169, 169, 167, 168]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:25.100837978Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1862,"":"downstairs","":"downstairs","session_id":"264ccec3-b1dc-4db6-baf7-67baf183e9f0"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:25.100852037Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1862,"":"downstairs","":"downstairs","session_id":"264ccec3-b1dc-4db6-baf7-67baf183e9f0"} {"msg":"[2]R flush_numbers[0..12]: [310, 306, 297, 305, 292, 312, 309, 313, 316, 316, 310, 312]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:25.100869195Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1862,"":"downstairs","":"downstairs","session_id":"264ccec3-b1dc-4db6-baf7-67baf183e9f0"} {"msg":"[2]R generation[0..12]: [167, 164, 160, 164, 158, 168, 166, 168, 169, 169, 167, 168]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:25.100883414Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1862,"":"downstairs","":"downstairs","session_id":"264ccec3-b1dc-4db6-baf7-67baf183e9f0"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:25.100897254Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1862,"":"downstairs","":"downstairs","session_id":"264ccec3-b1dc-4db6-baf7-67baf183e9f0"} {"msg":"Max found gen is 170","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:25.100910033Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1862,"":"downstairs","":"downstairs","session_id":"264ccec3-b1dc-4db6-baf7-67baf183e9f0"} {"msg":"Generation requested: 170 >= found:170","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:25.100923162Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1862,"":"downstairs","":"downstairs","session_id":"264ccec3-b1dc-4db6-baf7-67baf183e9f0"} Wait for a query_work_queue command to finish before sending IO {"msg":"Next flush: 317","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:25.100940891Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1862,"":"downstairs","":"downstairs","session_id":"264ccec3-b1dc-4db6-baf7-67baf183e9f0"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:25.10095419Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1862,"":"downstairs","":"downstairs","session_id":"264ccec3-b1dc-4db6-baf7-67baf183e9f0"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:25.100967319Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1862,"session_id":"264ccec3-b1dc-4db6-baf7-67baf183e9f0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:25.100979358Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1862,"client":"0","":"downstairs","session_id":"264ccec3-b1dc-4db6-baf7-67baf183e9f0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:25.100992837Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1862,"client":"1","":"downstairs","session_id":"264ccec3-b1dc-4db6-baf7-67baf183e9f0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:25.101006136Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1862,"client":"2","":"downstairs","session_id":"264ccec3-b1dc-4db6-baf7-67baf183e9f0"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:25.101019505Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1862,"session_id":"264ccec3-b1dc-4db6-baf7-67baf183e9f0"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:25.101032124Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1862,"session_id":"264ccec3-b1dc-4db6-baf7-67baf183e9f0"} {"msg":"74b78b13-bffc-41cd-9e30-c227a417e224 is now active with session: 264ccec3-b1dc-4db6-baf7-67baf183e9f0","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:25.101044843Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1862,"session_id":"264ccec3-b1dc-4db6-baf7-67baf183e9f0"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:25.101056813Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1862,"session_id":"264ccec3-b1dc-4db6-baf7-67baf183e9f0"} {"msg":"The guest has finished waiting for activation with:170","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:25.101069042Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1862} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 374 len 12288 data: 10 8 9 02/30 Flush 03/30 Write block 121 len 8192 data: 17 17 04/30 Write block 272 len 8192 data: 16 15 05/30 Write block 457 len 36864 data: 18 21 19 17 19 21 19 19 17 06/30 Write block 45 len 32768 data: 13 11 10 10 13 15 14 12 07/30 Write block 43 len 24576 data: 11 11 14 12 11 11 08/30 Write block 265 len 16384 data: 19 22 23 22 09/30 Read block 311 len 40960 10/30 Write block 284 len 28672 data: 10 11 11 10 9 9 8 11/30 Write block 322 len 12288 data: 14 12 10 12/30 Write block 220 len 28672 data: 15 12 10 13 12 13 15 13/30 Write block 591 len 24576 data: 20 17 14 9 8 6 14/30 Write block 403 len 16384 data: 16 14 13 14 15/30 Read block 378 len 28672 16/30 Write block 319 len 12288 data: 13 14 14 17/30 Flush 18/30 Flush 19/30 Write block 524 len 20480 data: 20 17 10 11 11 20/30 Write block 428 len 24576 data: 13 15 13 15 15 15 21/30 Flush 22/30 Write block 29 len 36864 data: 13 11 12 12 13 14 16 14 13 23/30 Read block 340 len 28672 24/30 Write block 332 len 24576 data: 15 16 16 15 13 13 25/30 Read block 489 len 24576 26/30 Write block 75 len 28672 data: 11 9 14 13 12 13 11 27/30 Read block 370 len 16384 28/30 Write block 409 len 16384 data: 16 14 11 9 29/30 Read block 286 len 36864 30/30 Write block 577 len 8192 data: 10 11 ---------------------------------------------------------------- Crucible gen:170 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 9 Done Done Done false 1022 Acked Read 7 Done Done Done false 1023 Acked Write 6 Done Done Done false 1024 Acked Read 6 Done Done Done false 1025 Acked Write 7 Done Done Done false 1026 Acked Read 4 Done Done Done false 1027 Acked Write 4 Done Done Done false 1028 Acked Read 9 Done Done Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 2 4 Done 8 8 7 23 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1020 1020 1020 Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:25.119339062Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1862} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 167 167 167 310 310 310 F F F 1 020-039 164 164 164 306 306 306 T T T 2 040-059 170 170 170 318 318 318 F F F 3 060-079 164 164 164 305 305 305 T T T 4 080-099 158 158 158 292 292 292 T T T 5 100-119 168 168 168 312 312 312 F F F 6 120-139 170 170 170 318 318 318 F F F 7 140-159 168 168 168 313 313 313 F F F 8 160-179 169 169 169 316 316 316 F F F 9 180-199 169 169 169 316 316 316 F F F 10 200-219 167 167 167 310 310 310 F F F 11 220-239 170 170 170 318 318 318 F F F 12 240-259 168 168 168 313 313 313 F F F 13 260-279 170 170 170 318 318 318 F F F 14 280-299 170 170 170 318 318 318 F F F 15 300-319 170 170 170 318 318 318 F F F 16 320-339 170 170 170 318 318 318 T T T 17 340-359 162 162 162 301 301 301 F F F 18 360-379 170 170 170 317 317 317 F F F 19 380-399 154 154 154 285 285 285 F F F 20 400-419 170 170 170 318 318 318 T T T 21 420-439 170 170 170 320 320 320 F F F 22 440-459 170 170 170 318 318 318 F F F 23 460-479 170 170 170 318 318 318 F F F 24 480-499 168 168 168 311 311 311 F F F 25 500-519 168 168 168 312 312 312 F F F 26 520-539 170 170 170 320 320 320 F F F 27 540-559 166 166 166 309 309 309 F F F 28 560-579 162 162 162 301 301 301 T T T 29 580-599 170 170 170 318 318 318 F F F Max gen: 170, Max flush: 320 A Difference in extent metadata is expected here On loop 85 Verifying data now No change after verify Loop: 85 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 167 167 167 310 310 310 F F F 1 020-039 171 171 171 321 321 321 F F F 2 040-059 170 170 170 318 318 318 F F F 3 060-079 171 171 171 321 321 321 F F F 4 080-099 171 171 171 321 321 321 F F F 5 100-119 168 168 168 312 312 312 F F F 6 120-139 170 170 170 318 318 318 F F F 7 140-159 168 168 168 313 313 313 F F F 8 160-179 169 169 169 316 316 316 F F F 9 180-199 169 169 169 316 316 316 F F F 10 200-219 167 167 167 310 310 310 F F F 11 220-239 170 170 170 318 318 318 F F F 12 240-259 168 168 168 313 313 313 F F F 13 260-279 170 170 170 318 318 318 F F F 14 280-299 170 170 170 318 318 318 F F F 15 300-319 170 170 170 318 318 318 F F F 16 320-339 171 171 171 321 321 321 F F F 17 340-359 162 162 162 301 301 301 F F F 18 360-379 170 170 170 317 317 317 F F F 19 380-399 154 154 154 285 285 285 F F F 20 400-419 171 171 171 321 321 321 F F F 21 420-439 170 170 170 320 320 320 F F F 22 440-459 170 170 170 318 318 318 F F F 23 460-479 170 170 170 318 318 318 F F F 24 480-499 168 168 168 311 311 311 F F F 25 500-519 168 168 168 312 312 312 F F F 26 520-539 170 170 170 320 320 320 F F F 27 540-559 166 166 166 309 309 309 F F F 28 560-579 171 171 171 321 321 321 F F F 29 580-599 170 170 170 318 318 318 F F F Max gen: 171, Max flush: 321 Begin loop 86 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:29.422557659Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1870,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:29.465436712Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1870,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:29.465826606Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1870,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:29.497314264Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1870,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:29.497893314Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1870} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"26e7d501b33f47379e53601720353c2983e7654b\",\n git_commit_timestamp: \"2024-12-12T20:16:40.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:29.497937101Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1870} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:29.49795502Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1870} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:29.497969729Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1870} {"msg":"Crucible 3d5b54b8-cd22-4a11-b147-32f000e14c14 has session id: 68b88b45-908f-4b94-9e24-a4f963b4578c","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:29.498443056Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1870,"session_id":"68b88b45-908f-4b94-9e24-a4f963b4578c"} {"msg":"Upstairs opts: Upstairs UUID: 3d5b54b8-cd22-4a11-b147-32f000e14c14, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:29.498488024Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1870,"session_id":"68b88b45-908f-4b94-9e24-a4f963b4578c"} {"msg":"Crucible stats registered with UUID: 3d5b54b8-cd22-4a11-b147-32f000e14c14","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:29.498509612Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1870,"session_id":"68b88b45-908f-4b94-9e24-a4f963b4578c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:29.529938155Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1870,"task":"crutest"} {"msg":"The guest has requested activation with gen:172","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:29.529967583Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1870} {"msg":"3d5b54b8-cd22-4a11-b147-32f000e14c14 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:29.530040198Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1870,"session_id":"68b88b45-908f-4b94-9e24-a4f963b4578c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:29.530062026Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1870,"client":"0","":"downstairs","session_id":"68b88b45-908f-4b94-9e24-a4f963b4578c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:29.530090594Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1870,"client":"0","":"downstairs","session_id":"68b88b45-908f-4b94-9e24-a4f963b4578c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:29.530108193Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1870,"client":"1","":"downstairs","session_id":"68b88b45-908f-4b94-9e24-a4f963b4578c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:29.530126392Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1870,"client":"1","":"downstairs","session_id":"68b88b45-908f-4b94-9e24-a4f963b4578c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:29.530139701Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1870,"client":"2","":"downstairs","session_id":"68b88b45-908f-4b94-9e24-a4f963b4578c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:29.530217676Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1870,"client":"2","":"downstairs","session_id":"68b88b45-908f-4b94-9e24-a4f963b4578c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:29.530256963Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1870,"":"io task","client":"2","":"downstairs","session_id":"68b88b45-908f-4b94-9e24-a4f963b4578c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:29.530453969Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1870,"":"io task","client":"0","":"downstairs","session_id":"68b88b45-908f-4b94-9e24-a4f963b4578c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:29.530488507Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1870,"":"io task","client":"1","":"downstairs","session_id":"68b88b45-908f-4b94-9e24-a4f963b4578c"} {"msg":"ds_connection connected from Ok(127.0.0.1:41024)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:29.530514245Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1870,"":"io task","client":"2","":"downstairs","session_id":"68b88b45-908f-4b94-9e24-a4f963b4578c"} {"msg":"ds_connection connected from Ok(127.0.0.1:38344)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:29.530532074Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1870,"":"io task","client":"0","":"downstairs","session_id":"68b88b45-908f-4b94-9e24-a4f963b4578c"} {"msg":"ds_connection connected from Ok(127.0.0.1:60085)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:29.530555102Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1870,"":"io task","client":"1","":"downstairs","session_id":"68b88b45-908f-4b94-9e24-a4f963b4578c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:29.530940036Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1870,"client":"1","":"downstairs","session_id":"68b88b45-908f-4b94-9e24-a4f963b4578c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:29.531130653Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1870,"client":"0","":"downstairs","session_id":"68b88b45-908f-4b94-9e24-a4f963b4578c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:29.531203537Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1870,"client":"2","":"downstairs","session_id":"68b88b45-908f-4b94-9e24-a4f963b4578c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:29.531636048Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1870,"client":"1","":"downstairs","session_id":"68b88b45-908f-4b94-9e24-a4f963b4578c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:29.53174377Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1870,"client":"2","":"downstairs","session_id":"68b88b45-908f-4b94-9e24-a4f963b4578c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:29.532029831Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1870,"client":"0","":"downstairs","session_id":"68b88b45-908f-4b94-9e24-a4f963b4578c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:29.532168391Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1870,"session_id":"68b88b45-908f-4b94-9e24-a4f963b4578c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:29.53232874Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1870,"session_id":"68b88b45-908f-4b94-9e24-a4f963b4578c"} {"msg":"[0]R flush_numbers[0..12]: [310, 321, 318, 321, 321, 312, 318, 313, 316, 316, 310, 318]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:29.532541065Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1870,"":"downstairs","":"downstairs","session_id":"68b88b45-908f-4b94-9e24-a4f963b4578c"} {"msg":"[0]R generation[0..12]: [167, 171, 170, 171, 171, 168, 170, 168, 169, 169, 167, 170]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:29.532571603Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1870,"":"downstairs","":"downstairs","session_id":"68b88b45-908f-4b94-9e24-a4f963b4578c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:29.532588042Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1870,"":"downstairs","":"downstairs","session_id":"68b88b45-908f-4b94-9e24-a4f963b4578c"} {"msg":"[1]R flush_numbers[0..12]: [310, 321, 318, 321, 321, 312, 318, 313, 316, 316, 310, 318]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:29.532602821Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1870,"":"downstairs","":"downstairs","session_id":"68b88b45-908f-4b94-9e24-a4f963b4578c"} {"msg":"[1]R generation[0..12]: [167, 171, 170, 171, 171, 168, 170, 168, 169, 169, 167, 170]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:29.53261708Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1870,"":"downstairs","":"downstairs","session_id":"68b88b45-908f-4b94-9e24-a4f963b4578c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:29.532631319Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1870,"":"downstairs","":"downstairs","session_id":"68b88b45-908f-4b94-9e24-a4f963b4578c"} {"msg":"[2]R flush_numbers[0..12]: [310, 321, 318, 321, 321, 312, 318, 313, 316, 316, 310, 318]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:29.532645148Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1870,"":"downstairs","":"downstairs","session_id":"68b88b45-908f-4b94-9e24-a4f963b4578c"} {"msg":"[2]R generation[0..12]: [167, 171, 170, 171, 171, 168, 170, 168, 169, 169, 167, 170]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:29.532658877Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1870,"":"downstairs","":"downstairs","session_id":"68b88b45-908f-4b94-9e24-a4f963b4578c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:29.532672496Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1870,"":"downstairs","":"downstairs","session_id":"68b88b45-908f-4b94-9e24-a4f963b4578c"} {"msg":"Max found gen is 172","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:29.532685365Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1870,"":"downstairs","":"downstairs","session_id":"68b88b45-908f-4b94-9e24-a4f963b4578c"} {"msg":"Generation requested: 172 >= found:172","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:29.532698644Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1870,"":"downstairs","":"downstairs","session_id":"68b88b45-908f-4b94-9e24-a4f963b4578c"} {"msg":"Next flush: 322","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO 2024-12-12T20:39:29.532711473Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1870,"":"downstairs","":"downstairs","session_id":"68b88b45-908f-4b94-9e24-a4f963b4578c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:29.532728212Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1870,"":"downstairs","":"downstairs","session_id":"68b88b45-908f-4b94-9e24-a4f963b4578c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:29.532741271Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1870,"session_id":"68b88b45-908f-4b94-9e24-a4f963b4578c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:29.53275619Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1870,"client":"0","":"downstairs","session_id":"68b88b45-908f-4b94-9e24-a4f963b4578c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:29.53277002Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1870,"client":"1","":"downstairs","session_id":"68b88b45-908f-4b94-9e24-a4f963b4578c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:29.532783179Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1870,"client":"2","":"downstairs","session_id":"68b88b45-908f-4b94-9e24-a4f963b4578c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:29.532796368Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1870,"session_id":"68b88b45-908f-4b94-9e24-a4f963b4578c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:29.532809477Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1870,"session_id":"68b88b45-908f-4b94-9e24-a4f963b4578c"} {"msg":"3d5b54b8-cd22-4a11-b147-32f000e14c14 is now active with session: 68b88b45-908f-4b94-9e24-a4f963b4578c","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:29.532822326Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1870,"session_id":"68b88b45-908f-4b94-9e24-a4f963b4578c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:29.532834245Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1870,"session_id":"68b88b45-908f-4b94-9e24-a4f963b4578c"} {"msg":"The guest has finished waiting for activation with:172","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:29.532846264Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1870} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 416 len 28672 data: 13 11 12 13 13 13 14 02/30 Read block 51 len 4096 03/30 Read block 325 len 20480 04/30 Write block 10 len 28672 data: 16 17 15 12 12 13 12 05/30 Write block 516 len 28672 data: 10 13 18 21 21 21 19 06/30 Read block 88 len 8192 07/30 Read block 129 len 4096 08/30 Flush 09/30 Write block 8 len 8192 data: 14 17 10/30 Write block 79 len 32768 data: 13 14 12 8 9 8 9 9 11/30 Write block 451 len 32768 data: 20 18 14 17 16 16 19 22 12/30 Read block 272 len 20480 13/30 Write block 64 len 24576 data: 11 11 12 11 11 11 14/30 Flush 15/30 Write block 439 len 40960 data: 9 9 9 10 10 10 10 13 16 16 16/30 Read block 132 len 32768 17/30 Read block 552 len 24576 18/30 Read block 72 len 20480 19/30 Read block 5 len 36864 20/30 Read block 272 len 12288 21/30 Read block 405 len 24576 22/30 Write block 379 len 40960 data: 16 16 16 18 16 14 15 17 14 13 23/30 Read block 413 len 20480 24/30 Read block 301 len 32768 25/30 Read block 0 len 32768 26/30 Write block 23 len 12288 data: 15 17 15 27/30 Read block 483 len 36864 28/30 Write block 50 len 36864 data: 16 15 13 15 15 19 23 24 21 29/30 Read block 402 len 24576 30/30 Write block 305 len 28672 data: 10 12 11 13 13 11 14 ---------------------------------------------------------------- Crucible gen:172 GIO:true work queues: Upstairs:0 downstairs:16 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1014 Acked Write 10 Done Done Done false 1015 Acked Read 8 Done Done Done false 1016 Acked Read 6 Done Done Done false 1017 Acked Read 5 Done Done Done false 1018 Acked Read 9 Done Done Done false 1019 Acked Read 3 Done Done Done false 1020 Acked Read 6 Done Done Done false 1021 Acked Write 10 Done Done Done false 1022 Acked Read 5 Done Done Done false 1023 Acked Read 8 Done Done Done false 1024 Acked Read 8 Done Done Done false 1025 Acked Write 3 Done Done Done false 1026 Acked Read 9 Done Done Done false 1027 Acked Write 9 Done Done Done false 1028 Acked Read 6 Done Done Done false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 15 15 15 45 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1013 1013 1013 Downstairs last five completed: 1013 1012 1011 1010 1009 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:29.554126107Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1870} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 172 172 172 323 323 323 F F F 1 020-039 171 171 171 321 321 321 T T T 2 040-059 170 170 170 318 318 318 T T T 3 060-079 172 172 172 323 323 323 F F F 4 080-099 172 172 172 323 323 323 F F F 5 100-119 168 168 168 312 312 312 F F F 6 120-139 170 170 170 318 318 318 F F F 7 140-159 168 168 168 313 313 313 F F F 8 160-179 169 169 169 316 316 316 F F F 9 180-199 169 169 169 316 316 316 F F F 10 200-219 167 167 167 310 310 310 F F F 11 220-239 170 170 170 318 318 318 F F F 12 240-259 168 168 168 313 313 313 F F F 13 260-279 170 170 170 318 318 318 F F F 14 280-299 170 170 170 318 318 318 F F F 15 300-319 170 170 170 318 318 318 T T T 16 320-339 171 171 171 321 321 321 F F F 17 340-359 162 162 162 301 301 301 F F F 18 360-379 170 170 170 317 317 317 T T T 19 380-399 154 154 154 285 285 285 T T T 20 400-419 172 172 172 322 322 322 F F F 21 420-439 172 172 172 322 322 322 T T T 22 440-459 172 172 172 323 323 323 T T T 23 460-479 170 170 170 318 318 318 F F F 24 480-499 168 168 168 311 311 311 F F F 25 500-519 172 172 172 322 322 322 F F F 26 520-539 172 172 172 322 322 322 F F F 27 540-559 166 166 166 309 309 309 F F F 28 560-579 171 171 171 321 321 321 F F F 29 580-599 170 170 170 318 318 318 F F F Max gen: 172, Max flush: 323 A Difference in extent metadata is expected here On loop 86 Verifying data now No change after verify Loop: 86 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 172 172 172 323 323 323 F F F 1 020-039 173 173 173 324 324 324 F F F 2 040-059 173 173 173 324 324 324 F F F 3 060-079 172 172 172 323 323 323 F F F 4 080-099 172 172 172 323 323 323 F F F 5 100-119 168 168 168 312 312 312 F F F 6 120-139 170 170 170 318 318 318 F F F 7 140-159 168 168 168 313 313 313 F F F 8 160-179 169 169 169 316 316 316 F F F 9 180-199 169 169 169 316 316 316 F F F 10 200-219 167 167 167 310 310 310 F F F 11 220-239 170 170 170 318 318 318 F F F 12 240-259 168 168 168 313 313 313 F F F 13 260-279 170 170 170 318 318 318 F F F 14 280-299 170 170 170 318 318 318 F F F 15 300-319 173 173 173 324 324 324 F F F 16 320-339 171 171 171 321 321 321 F F F 17 340-359 162 162 162 301 301 301 F F F 18 360-379 173 173 173 324 324 324 F F F 19 380-399 173 173 173 324 324 324 F F F 20 400-419 172 172 172 322 322 322 F F F 21 420-439 173 173 173 324 324 324 F F F 22 440-459 173 173 173 324 324 324 F F F 23 460-479 170 170 170 318 318 318 F F F 24 480-499 168 168 168 311 311 311 F F F 25 500-519 172 172 172 322 322 322 F F F 26 520-539 172 172 172 322 322 322 F F F 27 540-559 166 166 166 309 309 309 F F F 28 560-579 171 171 171 321 321 321 F F F 29 580-599 170 170 170 318 318 318 F F F Max gen: 173, Max flush: 324 Begin loop 87 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:33.86929807Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1878,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:33.914219852Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1878,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:33.914898155Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1878} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"26e7d501b33f47379e53601720353c2983e7654b\",\n git_commit_timestamp: \"2024-12-12T20:16:40.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:33.914935823Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1878} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:33.914950432Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1878} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:33.914964721Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1878} {"msg":"Crucible 54a883d6-eac1-4615-b5f6-a2a6f4e7dcad has session id: 14be9535-a78d-43fb-bd4c-ff91aa99a6d2","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:33.915361583Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1878,"session_id":"14be9535-a78d-43fb-bd4c-ff91aa99a6d2"} {"msg":"Upstairs opts: Upstairs UUID: 54a883d6-eac1-4615-b5f6-a2a6f4e7dcad, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:33.91540666Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1878,"session_id":"14be9535-a78d-43fb-bd4c-ff91aa99a6d2"} {"msg":"Crucible stats registered with UUID: 54a883d6-eac1-4615-b5f6-a2a6f4e7dcad","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:33.915431198Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1878,"session_id":"14be9535-a78d-43fb-bd4c-ff91aa99a6d2"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:33.94630251Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1878,"task":"crutest"} {"msg":"The guest has requested activation with gen:174","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:33.946329248Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1878} {"msg":"54a883d6-eac1-4615-b5f6-a2a6f4e7dcad active request set","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:33.946396263Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1878,"session_id":"14be9535-a78d-43fb-bd4c-ff91aa99a6d2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:33.946419242Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1878,"client":"0","":"downstairs","session_id":"14be9535-a78d-43fb-bd4c-ff91aa99a6d2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:33.946482197Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1878,"client":"0","":"downstairs","session_id":"14be9535-a78d-43fb-bd4c-ff91aa99a6d2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:33.946537904Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1878,"client":"1","":"downstairs","session_id":"14be9535-a78d-43fb-bd4c-ff91aa99a6d2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:33.946571381Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1878,"client":"1","":"downstairs","session_id":"14be9535-a78d-43fb-bd4c-ff91aa99a6d2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:33.946598599Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1878,"client":"2","":"downstairs","session_id":"14be9535-a78d-43fb-bd4c-ff91aa99a6d2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:33.946625927Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1878,"client":"2","":"downstairs","session_id":"14be9535-a78d-43fb-bd4c-ff91aa99a6d2"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:33.946648916Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1878,"":"io task","client":"2","":"downstairs","session_id":"14be9535-a78d-43fb-bd4c-ff91aa99a6d2"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:33.946676614Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1878,"":"io task","client":"0","":"downstairs","session_id":"14be9535-a78d-43fb-bd4c-ff91aa99a6d2"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:33.946703972Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1878,"":"io task","client":"1","":"downstairs","session_id":"14be9535-a78d-43fb-bd4c-ff91aa99a6d2"} {"msg":"ds_connection connected from Ok(127.0.0.1:43275)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:33.946779657Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1878,"":"io task","client":"2","":"downstairs","session_id":"14be9535-a78d-43fb-bd4c-ff91aa99a6d2"} {"msg":"ds_connection connected from Ok(127.0.0.1:53598)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:33.946959584Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1878,"":"io task","client":"0","":"downstairs","session_id":"14be9535-a78d-43fb-bd4c-ff91aa99a6d2"} {"msg":"ds_connection connected from Ok(127.0.0.1:57119)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:33.946987593Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1878,"":"io task","client":"1","":"downstairs","session_id":"14be9535-a78d-43fb-bd4c-ff91aa99a6d2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:33.947400064Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1878,"client":"2","":"downstairs","session_id":"14be9535-a78d-43fb-bd4c-ff91aa99a6d2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:33.947434552Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1878,"client":"0","":"downstairs","session_id":"14be9535-a78d-43fb-bd4c-ff91aa99a6d2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:33.947683175Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1878,"client":"1","":"downstairs","session_id":"14be9535-a78d-43fb-bd4c-ff91aa99a6d2"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:33.948173321Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1878,"client":"0","":"downstairs","session_id":"14be9535-a78d-43fb-bd4c-ff91aa99a6d2"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:33.948241926Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1878,"client":"2","":"downstairs","session_id":"14be9535-a78d-43fb-bd4c-ff91aa99a6d2"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:33.948286673Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1878,"client":"1","":"downstairs","session_id":"14be9535-a78d-43fb-bd4c-ff91aa99a6d2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:33.94877003Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1878,"session_id":"14be9535-a78d-43fb-bd4c-ff91aa99a6d2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:33.948800757Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1878,"session_id":"14be9535-a78d-43fb-bd4c-ff91aa99a6d2"} {"msg":"[0]R flush_numbers[0..12]: [323, 324, 324, 323, 323, 312, 318, 313, 316, 316, 310, 318]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:33.948880842Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1878,"":"downstairs","":"downstairs","session_id":"14be9535-a78d-43fb-bd4c-ff91aa99a6d2"} {"msg":"[0]R generation[0..12]: [172, 173, 173, 172, 172, 168, 170, 168, 169, 169, 167, 170]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:33.948900211Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1878,"":"downstairs","":"downstairs","session_id":"14be9535-a78d-43fb-bd4c-ff91aa99a6d2"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:33.94891452Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1878,"":"downstairs","":"downstairs","session_id":"14be9535-a78d-43fb-bd4c-ff91aa99a6d2"} {"msg":"[1]R flush_numbers[0..12]: [323, 324, 324, 323, 323, 312, 318, 313, 316, 316, 310, 318]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:33.948928499Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1878,"":"downstairs","":"downstairs","session_id":"14be9535-a78d-43fb-bd4c-ff91aa99a6d2"} {"msg":"[1]R generation[0..12]: [172, 173, 173, 172, 172, 168, 170, 168, 169, 169, 167, 170]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:33.948942578Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1878,"":"downstairs","":"downstairs","session_id":"14be9535-a78d-43fb-bd4c-ff91aa99a6d2"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:33.948956767Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1878,"":"downstairs","":"downstairs","session_id":"14be9535-a78d-43fb-bd4c-ff91aa99a6d2"} {"msg":"[2]R flush_numbers[0..12]: [323, 324, 324, 323, 323, 312, 318, 313, 316, 316, 310, 318]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:33.948970386Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1878,"":"downstairs","":"downstairs","session_id":"14be9535-a78d-43fb-bd4c-ff91aa99a6d2"} {"msg":"[2]R generation[0..12]: [172, 173, 173, 172, 172, 168, 170, 168, 169, 169, 167, 170]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:33.948984045Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1878,"":"downstairs","":"downstairs","session_id":"14be9535-a78d-43fb-bd4c-ff91aa99a6d2"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:33.949016103Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1878,"":"downstairs","":"downstairs","session_id":"14be9535-a78d-43fb-bd4c-ff91aa99a6d2"} {"msg":"Max found gen is 174","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:33.949031362Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1878,"":"downstairs","":"downstairs","session_id":"14be9535-a78d-43fb-bd4c-ff91aa99a6d2"} {"msg":"Generation requested: 174 >= found:174","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:33.949045711Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1878,"":"downstairs","":"downstairs","session_id":"14be9535-a78d-43fb-bd4c-ff91aa99a6d2"} {"msg":"Next flush: 325","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:33.94905862Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1878,"":"downstairs","":"downstairs","session_id":"14be9535-a78d-43fb-bd4c-ff91aa99a6d2"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:33.949071469Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1878,"":"downstairs","":"downstairs","session_id":"14be9535-a78d-43fb-bd4c-ff91aa99a6d2"} Wait for a query_work_queue command to finish before sending IO {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:33.949088968Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1878,"session_id":"14be9535-a78d-43fb-bd4c-ff91aa99a6d2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:33.949108766Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1878,"client":"0","":"downstairs","session_id":"14be9535-a78d-43fb-bd4c-ff91aa99a6d2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:33.949133135Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1878,"client":"1","":"downstairs","session_id":"14be9535-a78d-43fb-bd4c-ff91aa99a6d2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:33.949147884Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1878,"client":"2","":"downstairs","session_id":"14be9535-a78d-43fb-bd4c-ff91aa99a6d2"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:33.949161513Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1878,"session_id":"14be9535-a78d-43fb-bd4c-ff91aa99a6d2"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:33.949177112Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1878,"session_id":"14be9535-a78d-43fb-bd4c-ff91aa99a6d2"} {"msg":"54a883d6-eac1-4615-b5f6-a2a6f4e7dcad is now active with session: 14be9535-a78d-43fb-bd4c-ff91aa99a6d2","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:33.94920079Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1878,"session_id":"14be9535-a78d-43fb-bd4c-ff91aa99a6d2"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:33.949215559Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1878,"session_id":"14be9535-a78d-43fb-bd4c-ff91aa99a6d2"} {"msg":"The guest has finished waiting for activation with:174","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:33.949227858Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1878} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 418 len 28672 data: 13 14 14 14 15 12 14 02/30 Read block 354 len 28672 03/30 Write block 581 len 36864 data: 12 14 15 17 18 20 20 18 18 04/30 Read block 471 len 12288 05/30 Read block 29 len 36864 06/30 Read block 19 len 16384 07/30 Read block 49 len 8192 08/30 Read block 11 len 8192 09/30 Read block 125 len 40960 10/30 Write block 254 len 12288 data: 15 15 12 11/30 Write block 44 len 4096 data: 12 12/30 Write block 557 len 24576 data: 12 10 13 9 9 10 13/30 Read block 351 len 8192 14/30 Write block 238 len 24576 data: 15 16 15 15 12 11 15/30 Write block 343 len 4096 data: 15 16/30 Write block 20 len 12288 data: 12 10 12 17/30 Read block 373 len 24576 18/30 Flush 19/30 Write block 560 len 12288 data: 10 10 11 20/30 Read block 509 len 24576 21/30 Write block 501 len 24576 data: 10 11 12 15 14 15 22/30 Flush 23/30 Write block 51 len 12288 data: 16 14 16 24/30 Read block 177 len 24576 25/30 Write block 397 len 28672 data: 15 16 18 16 18 16 17 26/30 Read block 388 len 32768 27/30 Write block 141 len 20480 data: 10 14 15 14 14 28/30 Write block 557 len 12288 data: 13 11 14 29/30 Write block 201 len 16384 data: 10 12 14 15 30/30 Write block 255 len 4096 data: 16 ---------------------------------------------------------------- Crucible gen:174 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 3 Done Done Done false 1023 Acked Read 6 Done Done Done false 1024 Acked Write 7 Done Done Done false 1025 Acked Read 8 Done Done Done false 1026 Acked Write 5 Done Done Done false 1027 Acked Write 3 Done Done Done false 1028 Acked Write 4 Sent Sent Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 6 6 6 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1021 1021 1021 Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:33.967880002Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1878} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 172 172 172 323 323 323 F F F 1 020-039 174 174 174 325 325 325 F F F 2 040-059 174 174 174 325 325 325 T T T 3 060-079 172 172 172 323 323 323 F F F 4 080-099 172 172 172 323 323 323 F F F 5 100-119 168 168 168 312 312 312 F F F 6 120-139 170 170 170 318 318 318 F F F 7 140-159 168 168 168 313 313 313 T T T 8 160-179 169 169 169 316 316 316 F F F 9 180-199 169 169 169 316 316 316 F F F 10 200-219 167 167 167 310 310 310 T T T 11 220-239 174 174 174 325 325 325 F F F 12 240-259 174 174 174 325 325 325 T T T 13 260-279 170 170 170 318 318 318 F F F 14 280-299 170 170 170 318 318 318 F F F 15 300-319 173 173 173 324 324 324 F F F 16 320-339 171 171 171 321 321 321 F F F 17 340-359 174 174 174 325 325 325 F F F 18 360-379 173 173 173 324 324 324 F F F 19 380-399 173 173 173 324 324 324 T T T 20 400-419 174 174 174 325 325 325 T T T 21 420-439 174 174 174 325 325 325 F F F 22 440-459 173 173 173 324 324 324 F F F 23 460-479 170 170 170 318 318 318 F F F 24 480-499 168 168 168 311 311 311 F F F 25 500-519 174 174 174 326 326 326 F F F 26 520-539 172 172 172 322 322 322 F F F 27 540-559 174 174 174 325 325 325 T T T 28 560-579 174 174 174 326 326 326 F F F 29 580-599 174 174 174 325 325 325 F F F Max gen: 174, Max flush: 326 A Difference in extent metadata is expected here On loop 87 Verifying data now No change after verify Loop: 87 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 172 172 172 323 323 323 F F F 1 020-039 174 174 174 325 325 325 F F F 2 040-059 175 175 175 327 327 327 F F F 3 060-079 172 172 172 323 323 323 F F F 4 080-099 172 172 172 323 323 323 F F F 5 100-119 168 168 168 312 312 312 F F F 6 120-139 170 170 170 318 318 318 F F F 7 140-159 175 175 175 327 327 327 F F F 8 160-179 169 169 169 316 316 316 F F F 9 180-199 169 169 169 316 316 316 F F F 10 200-219 175 175 175 327 327 327 F F F 11 220-239 174 174 174 325 325 325 F F F 12 240-259 175 175 175 327 327 327 F F F 13 260-279 170 170 170 318 318 318 F F F 14 280-299 170 170 170 318 318 318 F F F 15 300-319 173 173 173 324 324 324 F F F 16 320-339 171 171 171 321 321 321 F F F 17 340-359 174 174 174 325 325 325 F F F 18 360-379 173 173 173 324 324 324 F F F 19 380-399 175 175 175 327 327 327 F F F 20 400-419 175 175 175 327 327 327 F F F 21 420-439 174 174 174 325 325 325 F F F 22 440-459 173 173 173 324 324 324 F F F 23 460-479 170 170 170 318 318 318 F F F 24 480-499 168 168 168 311 311 311 F F F 25 500-519 174 174 174 326 326 326 F F F 26 520-539 172 172 172 322 322 322 F F F 27 540-559 175 175 175 327 327 327 F F F 28 560-579 174 174 174 326 326 326 F F F 29 580-599 174 174 174 325 325 325 F F F Max gen: 175, Max flush: 327 Begin loop 88 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:38.25382747Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1886,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:38.296851993Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1886,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:38.297263485Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1886,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:38.328839478Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1886,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:38.329418387Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1886} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"26e7d501b33f47379e53601720353c2983e7654b\",\n git_commit_timestamp: \"2024-12-12T20:16:40.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:38.329453125Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1886} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:38.329466954Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1886} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:38.329481323Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1886} {"msg":"Crucible 067e1dd7-3641-457e-8b02-76145ba2bff1 has session id: a464e6aa-4609-445f-bdd5-d3e9e092a96d","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:38.329944251Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1886,"session_id":"a464e6aa-4609-445f-bdd5-d3e9e092a96d"} {"msg":"Upstairs opts: Upstairs UUID: 067e1dd7-3641-457e-8b02-76145ba2bff1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:38.329986708Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1886,"session_id":"a464e6aa-4609-445f-bdd5-d3e9e092a96d"} {"msg":"Crucible stats registered with UUID: 067e1dd7-3641-457e-8b02-76145ba2bff1","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:38.330012506Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1886,"session_id":"a464e6aa-4609-445f-bdd5-d3e9e092a96d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:38.361234884Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1886,"task":"crutest"} {"msg":"The guest has requested activation with gen:176","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:38.361269121Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1886} {"msg":"067e1dd7-3641-457e-8b02-76145ba2bff1 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:38.361320938Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1886,"session_id":"a464e6aa-4609-445f-bdd5-d3e9e092a96d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:38.361341366Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1886,"client":"0","":"downstairs","session_id":"a464e6aa-4609-445f-bdd5-d3e9e092a96d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:38.361369584Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1886,"client":"0","":"downstairs","session_id":"a464e6aa-4609-445f-bdd5-d3e9e092a96d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:38.361387903Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1886,"client":"1","":"downstairs","session_id":"a464e6aa-4609-445f-bdd5-d3e9e092a96d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:38.361405612Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1886,"client":"1","":"downstairs","session_id":"a464e6aa-4609-445f-bdd5-d3e9e092a96d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:38.361422201Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1886,"client":"2","":"downstairs","session_id":"a464e6aa-4609-445f-bdd5-d3e9e092a96d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:38.361439779Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1886,"client":"2","":"downstairs","session_id":"a464e6aa-4609-445f-bdd5-d3e9e092a96d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:38.361463588Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1886,"":"io task","client":"2","":"downstairs","session_id":"a464e6aa-4609-445f-bdd5-d3e9e092a96d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:38.361565831Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1886,"":"io task","client":"0","":"downstairs","session_id":"a464e6aa-4609-445f-bdd5-d3e9e092a96d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:38.361606338Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1886,"":"io task","client":"1","":"downstairs","session_id":"a464e6aa-4609-445f-bdd5-d3e9e092a96d"} {"msg":"ds_connection connected from Ok(127.0.0.1:39542)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:38.361634196Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1886,"":"io task","client":"2","":"downstairs","session_id":"a464e6aa-4609-445f-bdd5-d3e9e092a96d"} {"msg":"ds_connection connected from Ok(127.0.0.1:45921)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:38.361758068Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1886,"":"io task","client":"1","":"downstairs","session_id":"a464e6aa-4609-445f-bdd5-d3e9e092a96d"} {"msg":"ds_connection connected from Ok(127.0.0.1:62498)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:38.361791675Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1886,"":"io task","client":"0","":"downstairs","session_id":"a464e6aa-4609-445f-bdd5-d3e9e092a96d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:38.362316249Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1886,"client":"2","":"downstairs","session_id":"a464e6aa-4609-445f-bdd5-d3e9e092a96d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:38.362355476Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1886,"client":"0","":"downstairs","session_id":"a464e6aa-4609-445f-bdd5-d3e9e092a96d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:38.362384044Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1886,"client":"1","":"downstairs","session_id":"a464e6aa-4609-445f-bdd5-d3e9e092a96d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:38.362988203Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1886,"client":"2","":"downstairs","session_id":"a464e6aa-4609-445f-bdd5-d3e9e092a96d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:38.363117224Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1886,"client":"1","":"downstairs","session_id":"a464e6aa-4609-445f-bdd5-d3e9e092a96d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:38.363138202Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1886,"client":"0","":"downstairs","session_id":"a464e6aa-4609-445f-bdd5-d3e9e092a96d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:38.363491148Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1886,"session_id":"a464e6aa-4609-445f-bdd5-d3e9e092a96d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:38.363649967Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1886,"session_id":"a464e6aa-4609-445f-bdd5-d3e9e092a96d"} {"msg":"[0]R flush_numbers[0..12]: [323, 325, 327, 323, 323, 312, 318, 327, 316, 316, 327, 325]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:38.363737131Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1886,"":"downstairs","":"downstairs","session_id":"a464e6aa-4609-445f-bdd5-d3e9e092a96d"} {"msg":"[0]R generation[0..12]: [172, 174, 175, 172, 172, 168, 170, 175, 169, 169, 175, 174]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:38.36375715Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1886,"":"downstairs","":"downstairs","session_id":"a464e6aa-4609-445f-bdd5-d3e9e092a96d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:38.363775928Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1886,"":"downstairs","":"downstairs","session_id":"a464e6aa-4609-445f-bdd5-d3e9e092a96d"} {"msg":"[1]R flush_numbers[0..12]: [323, 325, 327, 323, 323, 312, 318, 327, 316, 316, 327, 325]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:38.363790457Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1886,"":"downstairs","":"downstairs","session_id":"a464e6aa-4609-445f-bdd5-d3e9e092a96d"} {"msg":"[1]R generation[0..12]: [172, 174, 175, 172, 172, 168, 170, 175, 169, 169, 175, 174]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:38.363804487Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1886,"":"downstairs","":"downstairs","session_id":"a464e6aa-4609-445f-bdd5-d3e9e092a96d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:38.363818675Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1886,"":"downstairs","":"downstairs","session_id":"a464e6aa-4609-445f-bdd5-d3e9e092a96d"} {"msg":"[2]R flush_numbers[0..12]: [323, 325, 327, 323, 323, 312, 318, 327, 316, 316, 327, 325]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:38.363832634Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1886,"":"downstairs","":"downstairs","session_id":"a464e6aa-4609-445f-bdd5-d3e9e092a96d"} {"msg":"[2]R generation[0..12]: [172, 174, 175, 172, 172, 168, 170, 175, 169, 169, 175, 174]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:38.363846224Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1886,"":"downstairs","":"downstairs","session_id":"a464e6aa-4609-445f-bdd5-d3e9e092a96d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:38.363860113Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1886,"":"downstairs","":"downstairs","session_id":"a464e6aa-4609-445f-bdd5-d3e9e092a96d"} {"msg":"Max found gen is 176","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:38.363873172Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1886,"":"downstairs","":"downstairs","session_id":"a464e6aa-4609-445f-bdd5-d3e9e092a96d"} {"msg":"Generation requested: 176 >= found:176","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:38.363886281Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1886,"":"downstairs","":"downstairs","session_id":"a464e6aa-4609-445f-bdd5-d3e9e092a96d"} {"msg":"Next flush: 328","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:38.36389931Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1886,"":"downstairs","":"downstairs","session_id":"a464e6aa-4609-445f-bdd5-d3e9e092a96d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:38.363912299Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1886,"":"downstairs","":"downstairs","session_id":"a464e6aa-4609-445f-bdd5-d3e9e092a96d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:38.363925328Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1886,"session_id":"a464e6aa-4609-445f-bdd5-d3e9e092a96d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:38.363937407Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1886,"client":"0","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO a464e6aa-4609-445f-bdd5-d3e9e092a96d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:38.363958716Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1886,"client":"1","":"downstairs","session_id":"a464e6aa-4609-445f-bdd5-d3e9e092a96d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:38.363971765Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1886,"client":"2","":"downstairs","session_id":"a464e6aa-4609-445f-bdd5-d3e9e092a96d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:38.363986614Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1886,"session_id":"a464e6aa-4609-445f-bdd5-d3e9e092a96d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:38.363998933Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1886,"session_id":"a464e6aa-4609-445f-bdd5-d3e9e092a96d"} {"msg":"067e1dd7-3641-457e-8b02-76145ba2bff1 is now active with session: a464e6aa-4609-445f-bdd5-d3e9e092a96d","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:38.364011772Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1886,"session_id":"a464e6aa-4609-445f-bdd5-d3e9e092a96d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:38.364023741Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1886,"session_id":"a464e6aa-4609-445f-bdd5-d3e9e092a96d"} {"msg":"The guest has finished waiting for activation with:176","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:38.36403586Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1886} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 37 len 40960 data: 14 13 13 11 11 11 12 13 15 13 02/30 Write block 220 len 32768 data: 16 13 11 14 13 14 16 13 03/30 Read block 457 len 16384 04/30 Write block 119 len 4096 data: 11 05/30 Read block 11 len 20480 06/30 Write block 334 len 12288 data: 17 16 14 07/30 Write block 545 len 32768 data: 11 10 11 11 12 12 10 10 08/30 Flush 09/30 Flush 10/30 Write block 311 len 24576 data: 15 12 16 16 18 19 11/30 Read block 374 len 40960 12/30 Read block 67 len 4096 13/30 Flush 14/30 Write block 203 len 12288 data: 15 16 17 15/30 Write block 319 len 8192 data: 14 15 16/30 Write block 269 len 40960 data: 13 14 13 17 16 17 15 13 13 16 17/30 Flush 18/30 Write block 462 len 24576 data: 22 20 20 18 15 14 19/30 Write block 383 len 24576 data: 17 15 16 18 15 14 20/30 Read block 297 len 8192 21/30 Flush 22/30 Write block 49 len 28672 data: 14 17 17 15 17 16 20 23/30 Read block 30 len 4096 24/30 Read block 387 len 20480 25/30 Write block 401 len 20480 data: 19 17 18 15 14 26/30 Write block 557 len 8192 data: 14 12 27/30 Write block 244 len 24576 data: 10 9 10 8 8 9 28/30 Read block 354 len 24576 29/30 Read block 317 len 28672 30/30 Read block 585 len 8192 ---------------------------------------------------------------- Crucible gen:176 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 7 Done Done Done false 1022 Acked Read 1 Done Done Done false 1023 Acked Read 5 Done Done Done false 1024 Acked Write 5 Done Done Done false 1025 Acked Write 2 Done Done Done false 1026 Acked Write 6 Done Done Done false 1027 Acked Read 6 Done Done Done false 1028 Acked Read 7 Done Done Done false 1029 Acked Read 2 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 9 9 9 27 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1020 1020 1020 Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:38.382941797Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1886} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 172 172 172 323 323 323 F F F 1 020-039 176 176 176 328 328 328 F F F 2 040-059 176 176 176 328 328 328 T T T 3 060-079 172 172 172 323 323 323 F F F 4 080-099 172 172 172 323 323 323 F F F 5 100-119 176 176 176 328 328 328 F F F 6 120-139 170 170 170 318 318 318 F F F 7 140-159 175 175 175 327 327 327 F F F 8 160-179 169 169 169 316 316 316 F F F 9 180-199 169 169 169 316 316 316 F F F 10 200-219 176 176 176 331 331 331 F F F 11 220-239 176 176 176 328 328 328 F F F 12 240-259 175 175 175 327 327 327 T T T 13 260-279 176 176 176 331 331 331 F F F 14 280-299 170 170 170 318 318 318 F F F 15 300-319 176 176 176 331 331 331 F F F 16 320-339 176 176 176 331 331 331 F F F 17 340-359 174 174 174 325 325 325 F F F 18 360-379 173 173 173 324 324 324 F F F 19 380-399 176 176 176 332 332 332 F F F 20 400-419 175 175 175 327 327 327 T T T 21 420-439 174 174 174 325 325 325 F F F 22 440-459 173 173 173 324 324 324 F F F 23 460-479 176 176 176 332 332 332 F F F 24 480-499 168 168 168 311 311 311 F F F 25 500-519 174 174 174 326 326 326 F F F 26 520-539 172 172 172 322 322 322 F F F 27 540-559 176 176 176 328 328 328 T T T 28 560-579 174 174 174 326 326 326 F F F 29 580-599 174 174 174 325 325 325 F F F Max gen: 176, Max flush: 332 A Difference in extent metadata is expected here On loop 88 Verifying data now No change after verify Loop: 88 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 172 172 172 323 323 323 F F F 1 020-039 176 176 176 328 328 328 F F F 2 040-059 177 177 177 333 333 333 F F F 3 060-079 172 172 172 323 323 323 F F F 4 080-099 172 172 172 323 323 323 F F F 5 100-119 176 176 176 328 328 328 F F F 6 120-139 170 170 170 318 318 318 F F F 7 140-159 175 175 175 327 327 327 F F F 8 160-179 169 169 169 316 316 316 F F F 9 180-199 169 169 169 316 316 316 F F F 10 200-219 176 176 176 331 331 331 F F F 11 220-239 176 176 176 328 328 328 F F F 12 240-259 177 177 177 333 333 333 F F F 13 260-279 176 176 176 331 331 331 F F F 14 280-299 170 170 170 318 318 318 F F F 15 300-319 176 176 176 331 331 331 F F F 16 320-339 176 176 176 331 331 331 F F F 17 340-359 174 174 174 325 325 325 F F F 18 360-379 173 173 173 324 324 324 F F F 19 380-399 176 176 176 332 332 332 F F F 20 400-419 177 177 177 333 333 333 F F F 21 420-439 174 174 174 325 325 325 F F F 22 440-459 173 173 173 324 324 324 F F F 23 460-479 176 176 176 332 332 332 F F F 24 480-499 168 168 168 311 311 311 F F F 25 500-519 174 174 174 326 326 326 F F F 26 520-539 172 172 172 322 322 322 F F F 27 540-559 177 177 177 333 333 333 F F F 28 560-579 174 174 174 326 326 326 F F F 29 580-599 174 174 174 325 325 325 F F F Max gen: 177, Max flush: 333 Begin loop 89 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:42.694082388Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1894,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:42.738491935Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1894,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:42.739187118Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1894} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"26e7d501b33f47379e53601720353c2983e7654b\",\n git_commit_timestamp: \"2024-12-12T20:16:40.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:42.739224495Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1894} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:42.739238554Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1894} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:42.739253133Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1894} {"msg":"Crucible 6dc5da0b-749f-421a-a132-1605eea687d1 has session id: fa7aaf70-abf0-4826-96d8-25a05825226f","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:42.739640746Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1894,"session_id":"fa7aaf70-abf0-4826-96d8-25a05825226f"} {"msg":"Upstairs opts: Upstairs UUID: 6dc5da0b-749f-421a-a132-1605eea687d1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:42.739685083Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1894,"session_id":"fa7aaf70-abf0-4826-96d8-25a05825226f"} {"msg":"Crucible stats registered with UUID: 6dc5da0b-749f-421a-a132-1605eea687d1","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:42.739708522Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1894,"session_id":"fa7aaf70-abf0-4826-96d8-25a05825226f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:42.770745311Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1894,"task":"crutest"} {"msg":"The guest has requested activation with gen:178","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:42.770779649Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1894} {"msg":"6dc5da0b-749f-421a-a132-1605eea687d1 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:42.770848244Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1894,"session_id":"fa7aaf70-abf0-4826-96d8-25a05825226f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:42.770871123Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1894,"client":"0","":"downstairs","session_id":"fa7aaf70-abf0-4826-96d8-25a05825226f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:42.770898351Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1894,"client":"0","":"downstairs","session_id":"fa7aaf70-abf0-4826-96d8-25a05825226f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:42.77091592Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1894,"client":"1","":"downstairs","session_id":"fa7aaf70-abf0-4826-96d8-25a05825226f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:42.770933638Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1894,"client":"1","":"downstairs","session_id":"fa7aaf70-abf0-4826-96d8-25a05825226f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:42.770964956Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1894,"client":"2","":"downstairs","session_id":"fa7aaf70-abf0-4826-96d8-25a05825226f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:42.770994354Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1894,"client":"2","":"downstairs","session_id":"fa7aaf70-abf0-4826-96d8-25a05825226f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:42.771019243Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1894,"":"io task","client":"0","":"downstairs","session_id":"fa7aaf70-abf0-4826-96d8-25a05825226f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:42.771037671Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1894,"":"io task","client":"2","":"downstairs","session_id":"fa7aaf70-abf0-4826-96d8-25a05825226f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:42.771068459Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1894,"":"io task","client":"1","":"downstairs","session_id":"fa7aaf70-abf0-4826-96d8-25a05825226f"} {"msg":"ds_connection connected from Ok(127.0.0.1:33835)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:42.771103257Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1894,"":"io task","client":"0","":"downstairs","session_id":"fa7aaf70-abf0-4826-96d8-25a05825226f"} {"msg":"ds_connection connected from Ok(127.0.0.1:49621)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:42.771158613Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1894,"":"io task","client":"1","":"downstairs","session_id":"fa7aaf70-abf0-4826-96d8-25a05825226f"} {"msg":"ds_connection connected from Ok(127.0.0.1:60183)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:42.77134902Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1894,"":"io task","client":"2","":"downstairs","session_id":"fa7aaf70-abf0-4826-96d8-25a05825226f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:42.77192681Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1894,"client":"0","":"downstairs","session_id":"fa7aaf70-abf0-4826-96d8-25a05825226f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:42.771956888Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1894,"client":"1","":"downstairs","session_id":"fa7aaf70-abf0-4826-96d8-25a05825226f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:42.771973567Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1894,"client":"2","":"downstairs","session_id":"fa7aaf70-abf0-4826-96d8-25a05825226f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:42.77251045Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1894,"client":"1","":"downstairs","session_id":"fa7aaf70-abf0-4826-96d8-25a05825226f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:42.772646491Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1894,"client":"2","":"downstairs","session_id":"fa7aaf70-abf0-4826-96d8-25a05825226f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:42.772669699Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1894,"client":"0","":"downstairs","session_id":"fa7aaf70-abf0-4826-96d8-25a05825226f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:42.773020855Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1894,"session_id":"fa7aaf70-abf0-4826-96d8-25a05825226f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:42.773303995Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1894,"session_id":"fa7aaf70-abf0-4826-96d8-25a05825226f"} {"msg":"[0]R flush_numbers[0..12]: [323, 328, 333, 323, 323, 328, 318, 327, 316, 316, 331, 328]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:42.77337222Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1894,"":"downstairs","":"downstairs","session_id":"fa7aaf70-abf0-4826-96d8-25a05825226f"} {"msg":"[0]R generation[0..12]: [172, 176, 177, 172, 172, 176, 170, 175, 169, 169, 176, 176]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:42.773400448Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1894,"":"downstairs","":"downstairs","session_id":"fa7aaf70-abf0-4826-96d8-25a05825226f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:42.773416177Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1894,"":"downstairs","":"downstairs","session_id":"fa7aaf70-abf0-4826-96d8-25a05825226f"} {"msg":"[1]R flush_numbers[0..12]: [323, 328, 333, 323, 323, 328, 318, 327, 316, 316, 331, 328]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:42.773441546Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1894,"":"downstairs","":"downstairs","session_id":"fa7aaf70-abf0-4826-96d8-25a05825226f"} {"msg":"[1]R generation[0..12]: [172, 176, 177, 172, 172, 176, 170, 175, 169, 169, 176, 176]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:42.773456524Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1894,"":"downstairs","":"downstairs","session_id":"fa7aaf70-abf0-4826-96d8-25a05825226f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:42.773470554Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1894,"":"downstairs","":"downstairs","session_id":"fa7aaf70-abf0-4826-96d8-25a05825226f"} {"msg":"[2]R flush_numbers[0..12]: [323, 328, 333, 323, 323, 328, 318, 327, 316, 316, 331, 328]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:42.773484443Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1894,"":"downstairs","":"downstairs","session_id":"fa7aaf70-abf0-4826-96d8-25a05825226f"} {"msg":"[2]R generation[0..12]: [172, 176, 177, 172, 172, 176, 170, 175, 169, 169, 176, 176]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:42.773498162Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1894,"":"downstairs","":"downstairs","session_id":"fa7aaf70-abf0-4826-96d8-25a05825226f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:42.773512171Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1894,"":"downstairs","":"downstairs","session_id":"fa7aaf70-abf0-4826-96d8-25a05825226f"} {"msg":"Max found gen is 178","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:42.77352513Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1894,"":"downstairs","":"downstairs","session_id":"fa7aaf70-abf0-4826-96d8-25a05825226f"} {"msg":"Generation requested: 178 >= found:178","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:42.773538209Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1894,"":"downstairs","":"downstairs","session_id":"fa7aaf70-abf0-4826-96d8-25a05825226f"} {"msg":"Next flush: 334","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:42.773551088Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1894,"":"downstairs","":"downstairs","session_id":"fa7aaf70-abf0-4826-96d8-25a05825226f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:42.773566957Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1894,"":"downstairs","":"downstairs","session_id":"fa7aaf70-abf0-4826-96d8-25a05825226f"Wait for a query_work_queue command to finish before sending IO } {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:42.773585196Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1894,"session_id":"fa7aaf70-abf0-4826-96d8-25a05825226f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:42.773597205Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1894,"client":"0","":"downstairs","session_id":"fa7aaf70-abf0-4826-96d8-25a05825226f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:42.773610804Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1894,"client":"1","":"downstairs","session_id":"fa7aaf70-abf0-4826-96d8-25a05825226f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:42.773624263Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1894,"client":"2","":"downstairs","session_id":"fa7aaf70-abf0-4826-96d8-25a05825226f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:42.773637632Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1894,"session_id":"fa7aaf70-abf0-4826-96d8-25a05825226f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:42.773649761Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1894,"session_id":"fa7aaf70-abf0-4826-96d8-25a05825226f"} {"msg":"6dc5da0b-749f-421a-a132-1605eea687d1 is now active with session: fa7aaf70-abf0-4826-96d8-25a05825226f","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:42.77366239Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1894,"session_id":"fa7aaf70-abf0-4826-96d8-25a05825226f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:42.773674349Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1894,"session_id":"fa7aaf70-abf0-4826-96d8-25a05825226f"} {"msg":"The guest has finished waiting for activation with:178","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:42.773686159Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1894} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 177 len 8192 data: 13 12 02/30 Read block 24 len 36864 03/30 Write block 571 len 12288 data: 17 18 16 04/30 Write block 39 len 12288 data: 14 12 12 05/30 Read block 8 len 36864 06/30 Read block 137 len 20480 07/30 Flush 08/30 Write block 468 len 40960 data: 13 13 14 12 9 11 12 14 15 13 09/30 Read block 358 len 24576 10/30 Write block 25 len 40960 data: 16 13 14 14 14 12 13 13 14 15 11/30 Read block 182 len 32768 12/30 Write block 242 len 32768 data: 13 12 11 10 11 9 9 10 13/30 Read block 121 len 8192 14/30 Write block 181 len 28672 data: 10 10 11 14 14 14 13 15/30 Write block 449 len 24576 data: 20 19 21 19 15 18 16/30 Write block 2 len 8192 data: 6 6 17/30 Read block 193 len 40960 18/30 Read block 536 len 32768 19/30 Read block 100 len 32768 20/30 Write block 587 len 40960 data: 21 19 19 22 21 18 15 10 9 7 21/30 Read block 248 len 20480 22/30 Write block 382 len 24576 data: 19 18 16 17 19 16 23/30 Flush 24/30 Write block 189 len 36864 data: 16 15 16 17 18 14 15 13 10 25/30 Read block 346 len 12288 26/30 Write block 426 len 16384 data: 10 13 14 16 27/30 Write block 309 len 12288 data: 14 12 16 28/30 Read block 58 len 36864 29/30 Write block 214 len 28672 data: 15 15 12 14 14 16 17 30/30 Read block 1 len 12288 ---------------------------------------------------------------- Crucible gen:178 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 9 Done Done Done false 1024 Acked Read 3 Done Done Done false 1025 Acked Write 4 Done Done Done false 1026 Acked Write 3 Done Done Done false 1027 Acked Read 9 Done Done Done false 1028 Acked Write 7 Done Done Done false 1029 Acked Read 3 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 7 7 7 21 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:42.79482683Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1894} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 178 178 178 335 335 335 F F F 1 020-039 178 178 178 335 335 335 F F F 2 040-059 178 178 178 334 334 334 F F F 3 060-079 172 172 172 323 323 323 F F F 4 080-099 172 172 172 323 323 323 F F F 5 100-119 176 176 176 328 328 328 F F F 6 120-139 170 170 170 318 318 318 F F F 7 140-159 175 175 175 327 327 327 F F F 8 160-179 178 178 178 334 334 334 F F F 9 180-199 178 178 178 335 335 335 T T T 10 200-219 176 176 176 331 331 331 T T T 11 220-239 176 176 176 328 328 328 T T T 12 240-259 178 178 178 335 335 335 F F F 13 260-279 176 176 176 331 331 331 F F F 14 280-299 170 170 170 318 318 318 F F F 15 300-319 176 176 176 331 331 331 T T T 16 320-339 176 176 176 331 331 331 F F F 17 340-359 174 174 174 325 325 325 F F F 18 360-379 173 173 173 324 324 324 F F F 19 380-399 178 178 178 335 335 335 F F F 20 400-419 177 177 177 333 333 333 F F F 21 420-439 174 174 174 325 325 325 T T T 22 440-459 178 178 178 335 335 335 F F F 23 460-479 178 178 178 335 335 335 F F F 24 480-499 168 168 168 311 311 311 F F F 25 500-519 174 174 174 326 326 326 F F F 26 520-539 172 172 172 322 322 322 F F F 27 540-559 177 177 177 333 333 333 F F F 28 560-579 178 178 178 334 334 334 F F F 29 580-599 178 178 178 335 335 335 F F F Max gen: 178, Max flush: 335 A Difference in extent metadata is expected here On loop 89 Verifying data now No change after verify Loop: 89 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 178 178 178 335 335 335 F F F 1 020-039 178 178 178 335 335 335 F F F 2 040-059 178 178 178 334 334 334 F F F 3 060-079 172 172 172 323 323 323 F F F 4 080-099 172 172 172 323 323 323 F F F 5 100-119 176 176 176 328 328 328 F F F 6 120-139 170 170 170 318 318 318 F F F 7 140-159 175 175 175 327 327 327 F F F 8 160-179 178 178 178 334 334 334 F F F 9 180-199 179 179 179 336 336 336 F F F 10 200-219 179 179 179 336 336 336 F F F 11 220-239 179 179 179 336 336 336 F F F 12 240-259 178 178 178 335 335 335 F F F 13 260-279 176 176 176 331 331 331 F F F 14 280-299 170 170 170 318 318 318 F F F 15 300-319 179 179 179 336 336 336 F F F 16 320-339 176 176 176 331 331 331 F F F 17 340-359 174 174 174 325 325 325 F F F 18 360-379 173 173 173 324 324 324 F F F 19 380-399 178 178 178 335 335 335 F F F 20 400-419 177 177 177 333 333 333 F F F 21 420-439 179 179 179 336 336 336 F F F 22 440-459 178 178 178 335 335 335 F F F 23 460-479 178 178 178 335 335 335 F F F 24 480-499 168 168 168 311 311 311 F F F 25 500-519 174 174 174 326 326 326 F F F 26 520-539 172 172 172 322 322 322 F F F 27 540-559 177 177 177 333 333 333 F F F 28 560-579 178 178 178 334 334 334 F F F 29 580-599 178 178 178 335 335 335 F F F Max gen: 179, Max flush: 336 Begin loop 90 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:47.067193375Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1902,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:47.110512248Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1902,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:47.11091141Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1902,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:47.14252131Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1902,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:47.14310699Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1902} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"26e7d501b33f47379e53601720353c2983e7654b\",\n git_commit_timestamp: \"2024-12-12T20:16:40.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:47.143143158Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1902} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:47.143157337Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1902} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:47.143171785Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1902} {"msg":"Crucible ec42cd9e-edfd-46b6-a08b-c362f64e3222 has session id: 3a403e59-3abe-418a-87cc-923d50128cc1","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:47.143639163Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1902,"session_id":"3a403e59-3abe-418a-87cc-923d50128cc1"} {"msg":"Upstairs opts: Upstairs UUID: ec42cd9e-edfd-46b6-a08b-c362f64e3222, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:47.1436842Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1902,"session_id":"3a403e59-3abe-418a-87cc-923d50128cc1"} {"msg":"Crucible stats registered with UUID: ec42cd9e-edfd-46b6-a08b-c362f64e3222","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:47.143709388Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1902,"session_id":"3a403e59-3abe-418a-87cc-923d50128cc1"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:47.174388553Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1902,"task":"crutest"} {"msg":"The guest has requested activation with gen:180","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:47.174423301Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1902} {"msg":"ec42cd9e-edfd-46b6-a08b-c362f64e3222 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:47.174474147Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1902,"session_id":"3a403e59-3abe-418a-87cc-923d50128cc1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:47.174495116Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1902,"client":"0","":"downstairs","session_id":"3a403e59-3abe-418a-87cc-923d50128cc1"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:47.174524194Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1902,"client":"0","":"downstairs","session_id":"3a403e59-3abe-418a-87cc-923d50128cc1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:47.174542192Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1902,"client":"1","":"downstairs","session_id":"3a403e59-3abe-418a-87cc-923d50128cc1"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:47.174559831Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1902,"client":"1","":"downstairs","session_id":"3a403e59-3abe-418a-87cc-923d50128cc1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:47.17457645Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1902,"client":"2","":"downstairs","session_id":"3a403e59-3abe-418a-87cc-923d50128cc1"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:47.174593909Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1902,"client":"2","":"downstairs","session_id":"3a403e59-3abe-418a-87cc-923d50128cc1"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:47.174610078Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1902,"":"io task","client":"2","":"downstairs","session_id":"3a403e59-3abe-418a-87cc-923d50128cc1"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:47.174632606Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1902,"":"io task","client":"0","":"downstairs","session_id":"3a403e59-3abe-418a-87cc-923d50128cc1"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:47.174665604Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1902,"":"io task","client":"1","":"downstairs","session_id":"3a403e59-3abe-418a-87cc-923d50128cc1"} {"msg":"ds_connection connected from Ok(127.0.0.1:43560)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:47.174699371Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1902,"":"io task","client":"2","":"downstairs","session_id":"3a403e59-3abe-418a-87cc-923d50128cc1"} {"msg":"ds_connection connected from Ok(127.0.0.1:50284)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:47.174745218Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1902,"":"io task","client":"1","":"downstairs","session_id":"3a403e59-3abe-418a-87cc-923d50128cc1"} {"msg":"ds_connection connected from Ok(127.0.0.1:42640)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:47.174926366Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1902,"":"io task","client":"0","":"downstairs","session_id":"3a403e59-3abe-418a-87cc-923d50128cc1"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:47.175499686Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1902,"client":"1","":"downstairs","session_id":"3a403e59-3abe-418a-87cc-923d50128cc1"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:47.175528874Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1902,"client":"2","":"downstairs","session_id":"3a403e59-3abe-418a-87cc-923d50128cc1"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:47.175543573Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1902,"client":"0","":"downstairs","session_id":"3a403e59-3abe-418a-87cc-923d50128cc1"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:47.176183729Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1902,"client":"1","":"downstairs","session_id":"3a403e59-3abe-418a-87cc-923d50128cc1"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:47.176213117Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1902,"client":"2","":"downstairs","session_id":"3a403e59-3abe-418a-87cc-923d50128cc1"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:47.176237936Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1902,"client":"0","":"downstairs","session_id":"3a403e59-3abe-418a-87cc-923d50128cc1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:47.176739551Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1902,"session_id":"3a403e59-3abe-418a-87cc-923d50128cc1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:47.176834394Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1902,"session_id":"3a403e59-3abe-418a-87cc-923d50128cc1"} {"msg":"[0]R flush_numbers[0..12]: [335, 335, 334, 323, 323, 328, 318, 327, 334, 336, 336, 336]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:47.176922918Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1902,"":"downstairs","":"downstairs","session_id":"3a403e59-3abe-418a-87cc-923d50128cc1"} {"msg":"[0]R generation[0..12]: [178, 178, 178, 172, 172, 176, 170, 175, 178, 179, 179, 179]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:47.176943607Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1902,"":"downstairs","":"downstairs","session_id":"3a403e59-3abe-418a-87cc-923d50128cc1"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:47.176958216Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1902,"":"downstairs","":"downstairs","session_id":"3a403e59-3abe-418a-87cc-923d50128cc1"} {"msg":"[1]R flush_numbers[0..12]: [335, 335, 334, 323, 323, 328, 318, 327, 334, 336, 336, 336]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:47.176972185Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1902,"":"downstairs","":"downstairs","session_id":"3a403e59-3abe-418a-87cc-923d50128cc1"} {"msg":"[1]R generation[0..12]: [178, 178, 178, 172, 172, 176, 170, 175, 178, 179, 179, 179]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:47.176985744Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1902,"":"downstairs","":"downstairs","session_id":"3a403e59-3abe-418a-87cc-923d50128cc1"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:47.177023681Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1902,"":"downstairs","":"downstairs","session_id":"3a403e59-3abe-418a-87cc-923d50128cc1"} {"msg":"[2]R flush_numbers[0..12]: [335, 335, 334, 323, 323, 328, 318, 327, 334, 336, 336, 336]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:47.1770418Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1902,"":"downstairs","":"downstairs","session_id":"3a403e59-3abe-418a-87cc-923d50128cc1"} {"msg":"[2]R generation[0..12]: [178, 178, 178, 172, 172, 176, 170, 175, 178, 179, 179, 179]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:47.177055939Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1902,"":"downstairs","":"downstairs","session_id":"3a403e59-3abe-418a-87cc-923d50128cc1"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:47.177069698Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1902,"":"downstairs","":"downstairs","session_id":"3a403e59-3abe-418a-87cc-923d50128cc1"} {"msg":"Max found gen is 180","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:47.177082447Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1902,"":"downstairs","":"downstairs","session_id":"3a403e59-3abe-418a-87cc-923d50128cc1"} {"msg":"Generation requested: 180 >= found:180","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:47.177095526Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1902,"":"downstairs","":"downstairs","session_id":"3a403e59-3abe-418a-87cc-923d50128cc1"} {"msg":"Next flush: 337","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:47.177108325Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1902,"":"downstairs","":"downstairs","session_id":"3a403e59-3abe-418a-87cc-923d50128cc1"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-12-12T20:39:47.177121155Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1902,"":"downstairs","":"downstairs","session_id":"3a403e59-3abe-418a-87cc-923d50128cc1"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:47.177138943Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1902,"session_id":"3a403e59-3abe-418a-87cc-923d50128cc1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:47.177151053Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1902,"client":"0","":"downstairs","session_id":"3a403e59-3abe-418a-87cc-923d50128cc1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:47.177164501Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1902,"client":"1","":"downstairs","session_id":"3a403e59-3abe-418a-87cc-923d50128cc1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:47.177177781Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1902,"client":"2","":"downstairs","session_id":"3a403e59-3abe-418a-87cc-923d50128cc1"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:47.17719094Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1902,"session_id":"3a403e59-3abe-418a-87cc-923d50128cc1"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:47.177203169Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1902,"session_id":"3a403e59-3abe-418a-87cc-923d50128cc1"} {"msg":"ec42cd9e-edfd-46b6-a08b-c362f64e3222 is now active with session: 3a403e59-3abe-418a-87cc-923d50128cc1","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:47.177215748Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1902,"session_id":"3a403e59-3abe-418a-87cc-923d50128cc1"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:47.177227697Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1902,"session_id":"3a403e59-3abe-418a-87cc-923d50128cc1"} {"msg":"The guest has finished waiting for activation with:180","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:47.177241556Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1902} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 380 len 36864 data: 17 17 20 19 17 18 20 17 15 02/30 Read block 79 len 4096 03/30 Write block 537 len 12288 data: 12 9 8 04/30 Write block 111 len 16384 data: 14 16 13 12 05/30 Write block 64 len 24576 data: 12 12 13 12 12 12 06/30 Read block 43 len 20480 07/30 Write block 298 len 4096 data: 9 08/30 Write block 311 len 4096 data: 17 09/30 Write block 82 len 16384 data: 9 10 9 10 10/30 Read block 308 len 12288 11/30 Write block 264 len 20480 data: 15 20 23 24 23 12/30 Flush 13/30 Write block 38 len 40960 data: 14 15 13 13 12 13 14 16 14 12 14/30 Read block 264 len 32768 15/30 Flush 16/30 Write block 264 len 12288 data: 16 21 24 17/30 Write block 428 len 12288 data: 15 17 14 18/30 Write block 597 len 4096 data: 5 19/30 Read block 185 len 20480 20/30 Flush 21/30 Write block 541 len 32768 data: 8 10 11 12 12 11 12 12 22/30 Read block 449 len 40960 23/30 Read block 321 len 24576 24/30 Write block 538 len 8192 data: 10 9 25/30 Write block 85 len 8192 data: 11 10 26/30 Read block 489 len 24576 27/30 Read block 270 len 16384 28/30 Write block 303 len 12288 data: 9 8 11 29/30 Write block 132 len 4096 data: 7 30/30 Write block 174 len 16384 data: 9 12 12 14 ---------------------------------------------------------------- Crucible gen:180 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1020 Acked Write 8 Done Done Done false 1021 Acked Read 10 Done Done Done false 1022 Acked Read 6 Done Done Done false 1023 Acked Write 2 Done Done Done false 1024 Acked Write 2 Done Done Done false 1025 Acked Read 6 Done Done Done false 1026 Acked Read 4 Done Done Done false 1027 Acked Write 3 Sent Sent Sent false 1028 Acked Write 1 Sent Sent Sent false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 3 3 9 Done 7 7 7 21 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1019 1019 1019 Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:47.194339567Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1902} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 178 178 178 335 335 335 F F F 1 020-039 180 180 180 338 338 338 F F F 2 040-059 180 180 180 338 338 338 F F F 3 060-079 180 180 180 337 337 337 F F F 4 080-099 180 180 180 337 337 337 T T T 5 100-119 180 180 180 337 337 337 F F F 6 120-139 170 170 170 318 318 318 T T T 7 140-159 175 175 175 327 327 327 F F F 8 160-179 178 178 178 334 334 334 T T T 9 180-199 179 179 179 336 336 336 F F F 10 200-219 179 179 179 336 336 336 F F F 11 220-239 179 179 179 336 336 336 F F F 12 240-259 178 178 178 335 335 335 F F F 13 260-279 180 180 180 339 339 339 F F F 14 280-299 180 180 180 337 337 337 F F F 15 300-319 180 180 180 337 337 337 T T T 16 320-339 176 176 176 331 331 331 F F F 17 340-359 174 174 174 325 325 325 F F F 18 360-379 173 173 173 324 324 324 F F F 19 380-399 180 180 180 337 337 337 F F F 20 400-419 177 177 177 333 333 333 F F F 21 420-439 180 180 180 339 339 339 F F F 22 440-459 178 178 178 335 335 335 F F F 23 460-479 178 178 178 335 335 335 F F F 24 480-499 168 168 168 311 311 311 F F F 25 500-519 174 174 174 326 326 326 F F F 26 520-539 180 180 180 337 337 337 T T T 27 540-559 177 177 177 333 333 333 T T T 28 560-579 178 178 178 334 334 334 F F F 29 580-599 180 180 180 339 339 339 F F F Max gen: 180, Max flush: 339 A Difference in extent metadata is expected here On loop 90 Verifying data now No change after verify Loop: 90 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 178 178 178 335 335 335 F F F 1 020-039 180 180 180 338 338 338 F F F 2 040-059 180 180 180 338 338 338 F F F 3 060-079 180 180 180 337 337 337 F F F 4 080-099 181 181 181 340 340 340 F F F 5 100-119 180 180 180 337 337 337 F F F 6 120-139 181 181 181 340 340 340 F F F 7 140-159 175 175 175 327 327 327 F F F 8 160-179 181 181 181 340 340 340 F F F 9 180-199 179 179 179 336 336 336 F F F 10 200-219 179 179 179 336 336 336 F F F 11 220-239 179 179 179 336 336 336 F F F 12 240-259 178 178 178 335 335 335 F F F 13 260-279 180 180 180 339 339 339 F F F 14 280-299 180 180 180 337 337 337 F F F 15 300-319 181 181 181 340 340 340 F F F 16 320-339 176 176 176 331 331 331 F F F 17 340-359 174 174 174 325 325 325 F F F 18 360-379 173 173 173 324 324 324 F F F 19 380-399 180 180 180 337 337 337 F F F 20 400-419 177 177 177 333 333 333 F F F 21 420-439 180 180 180 339 339 339 F F F 22 440-459 178 178 178 335 335 335 F F F 23 460-479 178 178 178 335 335 335 F F F 24 480-499 168 168 168 311 311 311 F F F 25 500-519 174 174 174 326 326 326 F F F 26 520-539 181 181 181 340 340 340 F F F 27 540-559 181 181 181 340 340 340 F F F 28 560-579 178 178 178 334 334 334 F F F 29 580-599 180 180 180 339 339 339 F F F Max gen: 181, Max flush: 340 Begin loop 91 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:51.501752205Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1910,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:51.545303372Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1910,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:51.545694445Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1910,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:51.577709108Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1910,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:51.578326035Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1910} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"26e7d501b33f47379e53601720353c2983e7654b\",\n git_commit_timestamp: \"2024-12-12T20:16:40.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:51.578361383Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1910} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:51.578378941Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1910} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:51.5783936Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1910} {"msg":"Crucible 2d088e49-b218-4a63-81c4-64cfc8a682e2 has session id: f8ac7215-5137-44ac-be69-ea2ebc100f33","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:51.578845049Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1910,"session_id":"f8ac7215-5137-44ac-be69-ea2ebc100f33"} {"msg":"Upstairs opts: Upstairs UUID: 2d088e49-b218-4a63-81c4-64cfc8a682e2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:51.578893656Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1910,"session_id":"f8ac7215-5137-44ac-be69-ea2ebc100f33"} {"msg":"Crucible stats registered with UUID: 2d088e49-b218-4a63-81c4-64cfc8a682e2","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:51.578918324Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1910,"session_id":"f8ac7215-5137-44ac-be69-ea2ebc100f33"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:51.609653955Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1910,"task":"crutest"} {"msg":"The guest has requested activation with gen:182","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:51.609683593Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1910} {"msg":"2d088e49-b218-4a63-81c4-64cfc8a682e2 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:51.609733299Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1910,"session_id":"f8ac7215-5137-44ac-be69-ea2ebc100f33"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:51.609753308Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1910,"client":"0","":"downstairs","session_id":"f8ac7215-5137-44ac-be69-ea2ebc100f33"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:51.609780466Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1910,"client":"0","":"downstairs","session_id":"f8ac7215-5137-44ac-be69-ea2ebc100f33"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:51.609798055Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1910,"client":"1","":"downstairs","session_id":"f8ac7215-5137-44ac-be69-ea2ebc100f33"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:51.609816263Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1910,"client":"1","":"downstairs","session_id":"f8ac7215-5137-44ac-be69-ea2ebc100f33"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:51.609832982Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1910,"client":"2","":"downstairs","session_id":"f8ac7215-5137-44ac-be69-ea2ebc100f33"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:51.60985924Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1910,"client":"2","":"downstairs","session_id":"f8ac7215-5137-44ac-be69-ea2ebc100f33"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:51.609895918Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1910,"":"io task","client":"2","":"downstairs","session_id":"f8ac7215-5137-44ac-be69-ea2ebc100f33"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:51.609929616Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1910,"":"io task","client":"0","":"downstairs","session_id":"f8ac7215-5137-44ac-be69-ea2ebc100f33"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:51.609961944Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1910,"":"io task","client":"1","":"downstairs","session_id":"f8ac7215-5137-44ac-be69-ea2ebc100f33"} {"msg":"ds_connection connected from Ok(127.0.0.1:41188)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:51.609995391Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1910,"":"io task","client":"2","":"downstairs","session_id":"f8ac7215-5137-44ac-be69-ea2ebc100f33"} {"msg":"ds_connection connected from Ok(127.0.0.1:62235)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:51.610028569Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1910,"":"io task","client":"1","":"downstairs","session_id":"f8ac7215-5137-44ac-be69-ea2ebc100f33"} {"msg":"ds_connection connected from Ok(127.0.0.1:54046)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:51.610063997Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1910,"":"io task","client":"0","":"downstairs","session_id":"f8ac7215-5137-44ac-be69-ea2ebc100f33"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:51.610792626Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1910,"client":"1","":"downstairs","session_id":"f8ac7215-5137-44ac-be69-ea2ebc100f33"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:51.610834253Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1910,"client":"2","":"downstairs","session_id":"f8ac7215-5137-44ac-be69-ea2ebc100f33"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:51.610866971Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1910,"client":"0","":"downstairs","session_id":"f8ac7215-5137-44ac-be69-ea2ebc100f33"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:51.611492368Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1910,"client":"2","":"downstairs","session_id":"f8ac7215-5137-44ac-be69-ea2ebc100f33"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:51.611623919Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1910,"client":"1","":"downstairs","session_id":"f8ac7215-5137-44ac-be69-ea2ebc100f33"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:51.611644968Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1910,"client":"0","":"downstairs","session_id":"f8ac7215-5137-44ac-be69-ea2ebc100f33"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:51.612002703Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1910,"session_id":"f8ac7215-5137-44ac-be69-ea2ebc100f33"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:51.612148443Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1910,"session_id":"f8ac7215-5137-44ac-be69-ea2ebc100f33"} {"msg":"[0]R flush_numbers[0..12]: [335, 338, 338, 337, 340, 337, 340, 327, 340, 336, 336, 336]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:51.612278934Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1910,"":"downstairs","":"downstairs","session_id":"f8ac7215-5137-44ac-be69-ea2ebc100f33"} {"msg":"[0]R generation[0..12]: [178, 180, 180, 180, 181, 180, 181, 175, 181, 179, 179, 179]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:51.612300772Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1910,"":"downstairs","":"downstairs","session_id":"f8ac7215-5137-44ac-be69-ea2ebc100f33"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:51.612316671Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1910,"":"downstairs","":"downstairs","session_id":"f8ac7215-5137-44ac-be69-ea2ebc100f33"} {"msg":"[1]R flush_numbers[0..12]: [335, 338, 338, 337, 340, 337, 340, 327, 340, 336, 336, 336]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:51.6123308Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1910,"":"downstairs","":"downstairs","session_id":"f8ac7215-5137-44ac-be69-ea2ebc100f33"} {"msg":"[1]R generation[0..12]: [178, 180, 180, 180, 181, 180, 181, 175, 181, 179, 179, 179]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:51.612344699Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1910,"":"downstairs","":"downstairs","session_id":"f8ac7215-5137-44ac-be69-ea2ebc100f33"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:51.612358748Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1910,"":"downstairs","":"downstairs","session_id":"f8ac7215-5137-44ac-be69-ea2ebc100f33"} {"msg":"[2]R flush_numbers[0..12]: [335, 338, 338, 337, 340, 337, 340, 327, 340, 336, 336, 336]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:51.612372367Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1910,"":"downstairs","":"downstairs","session_id":"f8ac7215-5137-44ac-be69-ea2ebc100f33"} {"msg":"[2]R generation[0..12]: [178, 180, 180, 180, 181, 180, 181, 175, 181, 179, 179, 179]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:51.612386016Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1910,"":"downstairs","":"downstairs","session_id":"f8ac7215-5137-44ac-be69-ea2ebc100f33"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:51.612399945Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1910,"":"downstairs","":"downstairs","session_id":"f8ac7215-5137-44ac-be69-ea2ebc100f33"} {"msg":"Max found gen is 182","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:51.612413015Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1910,"":"downstairs","":"downstairs","session_id":"f8ac7215-5137-44ac-be69-ea2ebc100f33"} {"msg":"Generation requested: 182 >= found:182","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:51.612426094Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1910,"":"downstairs","":"downstairs","session_id":"f8ac7215-5137-44ac-be69-ea2ebc100f33"} {"msg":"Next flush: 341","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:51.612438953Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1910,"":"downstairs","":"downstairs","session_id":"f8ac7215-5137-44ac-be69-ea2ebc100f33"} Wait for a query_work_queue command to finish before sending IO {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:51.612455781Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1910,"":"downstairs","":"downstairs","session_id":"f8ac7215-5137-44ac-be69-ea2ebc100f33"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:51.612469391Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1910,"session_id":"f8ac7215-5137-44ac-be69-ea2ebc100f33"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:51.61248387Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1910,"client":"0","":"downstairs","session_id":"f8ac7215-5137-44ac-be69-ea2ebc100f33"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:51.612497519Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1910,"client":"1","":"downstairs","session_id":"f8ac7215-5137-44ac-be69-ea2ebc100f33"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:51.612510768Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1910,"client":"2","":"downstairs","session_id":"f8ac7215-5137-44ac-be69-ea2ebc100f33"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:51.612523967Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1910,"session_id":"f8ac7215-5137-44ac-be69-ea2ebc100f33"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:51.612536206Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1910,"session_id":"f8ac7215-5137-44ac-be69-ea2ebc100f33"} {"msg":"2d088e49-b218-4a63-81c4-64cfc8a682e2 is now active with session: f8ac7215-5137-44ac-be69-ea2ebc100f33","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:51.612548975Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1910,"session_id":"f8ac7215-5137-44ac-be69-ea2ebc100f33"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2024-12-12T20:39:51.612560994Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1910,"session_id":"f8ac7215-5137-44ac-be69-ea2ebc100f33"} {"msg":"The guest has finished waiting for activation with:182","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:51.612573373Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1910} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 254 len 40960 data: 16 17 13 10 11 12 15 11 12 16 02/30 Read block 384 len 36864 03/30 Write block 278 len 4096 data: 17 04/30 Read block 477 len 40960 05/30 Write block 551 len 36864 data: 11 11 12 13 12 12 15 13 15 06/30 Read block 202 len 36864 07/30 Read block 554 len 12288 08/30 Flush 09/30 Write block 475 len 36864 data: 15 16 14 13 12 9 10 10 8 10/30 Read block 590 len 12288 11/30 Write block 497 len 20480 data: 11 8 7 8 11 12/30 Write block 290 len 8192 data: 9 12 13/30 Write block 461 len 20480 data: 20 23 21 21 19 14/30 Flush 15/30 Flush 16/30 Write block 368 len 24576 data: 13 12 10 8 8 8 17/30 Write block 358 len 32768 data: 13 13 13 13 12 12 16 17 18/30 Write block 178 len 36864 data: 13 14 11 11 11 12 15 15 15 19/30 Read block 437 len 20480 20/30 Read block 136 len 12288 21/30 Read block 293 len 28672 22/30 Read block 245 len 4096 23/30 Write block 400 len 36864 data: 17 20 18 19 16 15 15 16 17 24/30 Flush 25/30 Write block 167 len 28672 data: 11 10 11 11 8 7 9 26/30 Write block 120 len 32768 data: 16 18 18 16 14 15 15 13 27/30 Write block 301 len 8192 data: 10 9 28/30 Write block 521 len 20480 data: 22 20 22 21 18 29/30 Read block 419 len 32768 30/30 Read block 545 len 16384 ---------------------------------------------------------------- Crucible gen:182 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 7 Done Done Done false 1025 Acked Write 8 Done Done Done false 1026 Acked Write 2 Done Done Done false 1027 Acked Write 5 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Read 4 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 6 5 5 16 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:51.632536337Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1910} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 178 178 178 335 335 335 F F F 1 020-039 180 180 180 338 338 338 F F F 2 040-059 180 180 180 338 338 338 F F F 3 060-079 180 180 180 337 337 337 F F F 4 080-099 181 181 181 340 340 340 F F F 5 100-119 180 180 180 337 337 337 F F F 6 120-139 181 181 181 340 340 340 T T T 7 140-159 175 175 175 327 327 327 F F F 8 160-179 182 182 182 344 344 344 T T T 9 180-199 182 182 182 344 344 344 F F F 10 200-219 179 179 179 336 336 336 F F F 11 220-239 179 179 179 336 336 336 F F F 12 240-259 182 182 182 341 341 341 F F F 13 260-279 182 182 182 341 341 341 F F F 14 280-299 182 182 182 342 342 342 F F F 15 300-319 181 181 181 340 340 340 T T T 16 320-339 176 176 176 331 331 331 F F F 17 340-359 182 182 182 344 344 344 F F F 18 360-379 182 182 182 344 344 344 F F F 19 380-399 180 180 180 337 337 337 F F F 20 400-419 182 182 182 344 344 344 F F F 21 420-439 180 180 180 339 339 339 F F F 22 440-459 178 178 178 335 335 335 F F F 23 460-479 182 182 182 342 342 342 F F F 24 480-499 182 182 182 342 342 342 F F F 25 500-519 182 182 182 342 342 342 F F F 26 520-539 181 181 181 340 340 340 T T T 27 540-559 182 182 182 341 341 341 F F F 28 560-579 178 178 178 334 334 334 F F F 29 580-599 180 180 180 339 339 339 F F F Max gen: 182, Max flush: 344 A Difference in extent metadata is expected here On loop 91 Verifying data now No change after verify Loop: 91 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 178 178 178 335 335 335 F F F 1 020-039 180 180 180 338 338 338 F F F 2 040-059 180 180 180 338 338 338 F F F 3 060-079 180 180 180 337 337 337 F F F 4 080-099 181 181 181 340 340 340 F F F 5 100-119 180 180 180 337 337 337 F F F 6 120-139 183 183 183 345 345 345 F F F 7 140-159 175 175 175 327 327 327 F F F 8 160-179 183 183 183 345 345 345 F F F 9 180-199 182 182 182 344 344 344 F F F 10 200-219 179 179 179 336 336 336 F F F 11 220-239 179 179 179 336 336 336 F F F 12 240-259 182 182 182 341 341 341 F F F 13 260-279 182 182 182 341 341 341 F F F 14 280-299 182 182 182 342 342 342 F F F 15 300-319 183 183 183 345 345 345 F F F 16 320-339 176 176 176 331 331 331 F F F 17 340-359 182 182 182 344 344 344 F F F 18 360-379 182 182 182 344 344 344 F F F 19 380-399 180 180 180 337 337 337 F F F 20 400-419 182 182 182 344 344 344 F F F 21 420-439 180 180 180 339 339 339 F F F 22 440-459 178 178 178 335 335 335 F F F 23 460-479 182 182 182 342 342 342 F F F 24 480-499 182 182 182 342 342 342 F F F 25 500-519 182 182 182 342 342 342 F F F 26 520-539 183 183 183 345 345 345 F F F 27 540-559 182 182 182 341 341 341 F F F 28 560-579 178 178 178 334 334 334 F F F 29 580-599 180 180 180 339 339 339 F F F Max gen: 183, Max flush: 345 Begin loop 92 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:55.925998177Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1918,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:55.971229498Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1918,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:55.971851855Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1918} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"26e7d501b33f47379e53601720353c2983e7654b\",\n git_commit_timestamp: \"2024-12-12T20:16:40.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:55.971883553Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1918} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:55.971896872Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1918} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:55.971911031Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1918} {"msg":"Crucible b2d1b844-8e96-42ab-b30c-2207be2eff84 has session id: b4417c24-4d02-4c4d-9d4f-a585bc78bb34","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:55.972427805Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1918,"session_id":"b4417c24-4d02-4c4d-9d4f-a585bc78bb34"} {"msg":"Upstairs opts: Upstairs UUID: b2d1b844-8e96-42ab-b30c-2207be2eff84, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:55.972473772Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1918,"session_id":"b4417c24-4d02-4c4d-9d4f-a585bc78bb34"} {"msg":"Crucible stats registered with UUID: b2d1b844-8e96-42ab-b30c-2207be2eff84","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:55.972496781Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1918,"session_id":"b4417c24-4d02-4c4d-9d4f-a585bc78bb34"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:56.004230632Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1918,"task":"crutest"} {"msg":"The guest has requested activation with gen:184","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:56.00426257Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1918} {"msg":"b2d1b844-8e96-42ab-b30c-2207be2eff84 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:56.004313156Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1918,"session_id":"b4417c24-4d02-4c4d-9d4f-a585bc78bb34"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:56.004333295Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1918,"client":"0","":"downstairs","session_id":"b4417c24-4d02-4c4d-9d4f-a585bc78bb34"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:56.004361853Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1918,"client":"0","":"downstairs","session_id":"b4417c24-4d02-4c4d-9d4f-a585bc78bb34"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:56.004393011Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1918,"client":"1","":"downstairs","session_id":"b4417c24-4d02-4c4d-9d4f-a585bc78bb34"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:56.004500354Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1918,"client":"1","":"downstairs","session_id":"b4417c24-4d02-4c4d-9d4f-a585bc78bb34"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:56.00454975Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1918,"client":"2","":"downstairs","session_id":"b4417c24-4d02-4c4d-9d4f-a585bc78bb34"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:56.004585148Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1918,"client":"2","":"downstairs","session_id":"b4417c24-4d02-4c4d-9d4f-a585bc78bb34"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:56.004611536Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1918,"":"io task","client":"2","":"downstairs","session_id":"b4417c24-4d02-4c4d-9d4f-a585bc78bb34"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:56.004638064Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1918,"":"io task","client":"0","":"downstairs","session_id":"b4417c24-4d02-4c4d-9d4f-a585bc78bb34"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:56.004663422Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1918,"":"io task","client":"1","":"downstairs","session_id":"b4417c24-4d02-4c4d-9d4f-a585bc78bb34"} {"msg":"ds_connection connected from Ok(127.0.0.1:34289)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:56.004689071Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1918,"":"io task","client":"0","":"downstairs","session_id":"b4417c24-4d02-4c4d-9d4f-a585bc78bb34"} {"msg":"ds_connection connected from Ok(127.0.0.1:61344)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:56.004713829Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1918,"":"io task","client":"2","":"downstairs","session_id":"b4417c24-4d02-4c4d-9d4f-a585bc78bb34"} {"msg":"ds_connection connected from Ok(127.0.0.1:44434)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:56.004738357Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1918,"":"io task","client":"1","":"downstairs","session_id":"b4417c24-4d02-4c4d-9d4f-a585bc78bb34"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:56.005252642Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1918,"client":"1","":"downstairs","session_id":"b4417c24-4d02-4c4d-9d4f-a585bc78bb34"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:56.005412901Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1918,"client":"0","":"downstairs","session_id":"b4417c24-4d02-4c4d-9d4f-a585bc78bb34"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:56.005457818Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1918,"client":"2","":"downstairs","session_id":"b4417c24-4d02-4c4d-9d4f-a585bc78bb34"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:56.005894287Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1918,"client":"1","":"downstairs","session_id":"b4417c24-4d02-4c4d-9d4f-a585bc78bb34"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:56.005975682Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1918,"client":"0","":"downstairs","session_id":"b4417c24-4d02-4c4d-9d4f-a585bc78bb34"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:56.006197097Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1918,"client":"2","":"downstairs","session_id":"b4417c24-4d02-4c4d-9d4f-a585bc78bb34"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:56.006479867Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1918,"session_id":"b4417c24-4d02-4c4d-9d4f-a585bc78bb34"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:56.006572041Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1918,"session_id":"b4417c24-4d02-4c4d-9d4f-a585bc78bb34"} {"msg":"[0]R flush_numbers[0..12]: [335, 338, 338, 337, 340, 337, 345, 327, 345, 344, 336, 336]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:56.006696512Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1918,"":"downstairs","":"downstairs","session_id":"b4417c24-4d02-4c4d-9d4f-a585bc78bb34"} {"msg":"[0]R generation[0..12]: [178, 180, 180, 180, 181, 180, 183, 175, 183, 182, 179, 179]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:56.006716301Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1918,"":"downstairs","":"downstairs","session_id":"b4417c24-4d02-4c4d-9d4f-a585bc78bb34"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:56.00673178Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1918,"":"downstairs","":"downstairs","session_id":"b4417c24-4d02-4c4d-9d4f-a585bc78bb34"} {"msg":"[1]R flush_numbers[0..12]: [335, 338, 338, 337, 340, 337, 345, 327, 345, 344, 336, 336]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:56.006746069Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1918,"":"downstairs","":"downstairs","session_id":"b4417c24-4d02-4c4d-9d4f-a585bc78bb34"} {"msg":"[1]R generation[0..12]: [178, 180, 180, 180, 181, 180, 183, 175, 183, 182, 179, 179]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:56.006759878Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1918,"":"downstairs","":"downstairs","session_id":"b4417c24-4d02-4c4d-9d4f-a585bc78bb34"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:56.006773807Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1918,"":"downstairs","":"downstairs","session_id":"b4417c24-4d02-4c4d-9d4f-a585bc78bb34"} {"msg":"[2]R flush_numbers[0..12]: [335, 338, 338, 337, 340, 337, 345, 327, 345, 344, 336, 336]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:56.006787456Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1918,"":"downstairs","":"downstairs","session_id":"b4417c24-4d02-4c4d-9d4f-a585bc78bb34"} {"msg":"[2]R generation[0..12]: [178, 180, 180, 180, 181, 180, 183, 175, 183, 182, 179, 179]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:56.006801255Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1918,"":"downstairs","":"downstairs","session_id":"b4417c24-4d02-4c4d-9d4f-a585bc78bb34"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:56.006815344Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1918,"":"downstairs","":"downstairs","session_id":"b4417c24-4d02-4c4d-9d4f-a585bc78bb34"} {"msg":"Max found gen is 184","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:56.006831303Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1918,"":"downstairs","":"downstairs","session_id":"b4417c24-4d02-4c4d-9d4f-a585bc78bb34"} {"msg":"Generation requested: 184 >= found:184","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:56.006844782Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1918,"":"downstairs","":"downstairs","session_id":"b4417c24-4d02-4c4d-9d4f-a585bc78bb34"} {"msg":"Next flush: 346","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:56.006857921Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1918,"":"downstairs","":"downstairs","session_id":"b4417c24-4d02-4c4d-9d4f-a585bc78bb34"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:56.00687084Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1918Wait for a query_work_queue command to finish before sending IO ,"":"downstairs","":"downstairs","session_id":"b4417c24-4d02-4c4d-9d4f-a585bc78bb34"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:56.006894538Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1918,"session_id":"b4417c24-4d02-4c4d-9d4f-a585bc78bb34"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:56.006918447Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1918,"client":"0","":"downstairs","session_id":"b4417c24-4d02-4c4d-9d4f-a585bc78bb34"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:56.006944475Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1918,"client":"1","":"downstairs","session_id":"b4417c24-4d02-4c4d-9d4f-a585bc78bb34"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:56.006969433Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1918,"client":"2","":"downstairs","session_id":"b4417c24-4d02-4c4d-9d4f-a585bc78bb34"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:56.006995062Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1918,"session_id":"b4417c24-4d02-4c4d-9d4f-a585bc78bb34"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"name":"crucible","level":30,"time":"2024-12-12T20:39:56.00702064Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1918,"session_id":"b4417c24-4d02-4c4d-9d4f-a585bc78bb34"} {"msg":"b2d1b844-8e96-42ab-b30c-2207be2eff84 is now active with session: b4417c24-4d02-4c4d-9d4f-a585bc78bb34","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:56.007045228Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1918,"session_id":"b4417c24-4d02-4c4d-9d4f-a585bc78bb34"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:56.007059257Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1918,"session_id":"b4417c24-4d02-4c4d-9d4f-a585bc78bb34"} {"msg":"The guest has finished waiting for activation with:184","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:39:56.007071726Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1918} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 235 len 24576 data: 16 16 16 16 17 16 02/30 Read block 178 len 12288 03/30 Read block 129 len 4096 04/30 Read block 310 len 16384 05/30 Read block 451 len 20480 06/30 Write block 413 len 20480 data: 10 12 11 14 12 07/30 Read block 298 len 40960 08/30 Read block 530 len 24576 09/30 Flush 10/30 Write block 187 len 12288 data: 14 16 17 11/30 Write block 241 len 40960 data: 16 14 13 12 11 12 10 10 11 9 12/30 Read block 239 len 20480 13/30 Write block 114 len 24576 data: 13 13 12 11 13 12 14/30 Read block 260 len 16384 15/30 Read block 589 len 16384 16/30 Read block 147 len 20480 17/30 Write block 232 len 16384 data: 16 16 16 17 18/30 Write block 317 len 20480 data: 17 17 15 16 15 19/30 Flush 20/30 Write block 84 len 24576 data: 10 12 11 14 17 15 21/30 Read block 34 len 20480 22/30 Write block 580 len 28672 data: 11 13 15 16 18 19 21 23/30 Read block 83 len 32768 24/30 Read block 410 len 28672 25/30 Write block 594 len 4096 data: 11 26/30 Write block 280 len 16384 data: 12 13 11 10 27/30 Write block 418 len 24576 data: 14 15 15 15 16 13 28/30 Write block 338 len 36864 data: 12 10 13 10 13 16 19 18 18 29/30 Write block 22 len 40960 data: 13 16 18 17 14 15 15 15 13 14 30/30 Read block 470 len 12288 ---------------------------------------------------------------- Crucible gen:184 GIO:true work queues: Upstairs:0 downstairs:11 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1019 Acked Write 6 Done Done Done false 1020 Acked Read 5 Done Done Done false 1021 Acked Write 7 Done Done Done false 1022 Acked Read 8 Done Done Done false 1023 Acked Read 7 Done Done Done false 1024 Acked Write 1 Done Done Done false 1025 Acked Write 4 Done Done Done false 1026 Acked Write 6 Done Done Done false 1027 Acked Write 9 Done Done Done false 1028 Acked Write 10 Done Done Done false 1029 Acked Read 3 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 11 10 10 31 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1018 1018 1018 Downstairs last five completed: 1018 1017 1016 1015 1014 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:39:56.027074517Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1918} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 178 178 178 335 335 335 F F F 1 020-039 180 180 180 338 338 338 T T T 2 040-059 180 180 180 338 338 338 F F F 3 060-079 180 180 180 337 337 337 F F F 4 080-099 181 181 181 340 340 340 T T T 5 100-119 184 184 184 347 347 347 F F F 6 120-139 183 183 183 345 345 345 F F F 7 140-159 175 175 175 327 327 327 F F F 8 160-179 183 183 183 345 345 345 F F F 9 180-199 184 184 184 347 347 347 F F F 10 200-219 179 179 179 336 336 336 F F F 11 220-239 184 184 184 347 347 347 F F F 12 240-259 184 184 184 347 347 347 F F F 13 260-279 182 182 182 341 341 341 F F F 14 280-299 182 182 182 342 342 342 T T T 15 300-319 184 184 184 347 347 347 F F F 16 320-339 184 184 184 347 347 347 T T T 17 340-359 182 182 182 344 344 344 T T T 18 360-379 182 182 182 344 344 344 F F F 19 380-399 180 180 180 337 337 337 F F F 20 400-419 184 184 184 346 346 346 T T T 21 420-439 180 180 180 339 339 339 T T T 22 440-459 178 178 178 335 335 335 F F F 23 460-479 182 182 182 342 342 342 F F F 24 480-499 182 182 182 342 342 342 F F F 25 500-519 182 182 182 342 342 342 F F F 26 520-539 183 183 183 345 345 345 F F F 27 540-559 182 182 182 341 341 341 F F F 28 560-579 178 178 178 334 334 334 F F F 29 580-599 180 180 180 339 339 339 T T T Max gen: 184, Max flush: 347 A Difference in extent metadata is expected here On loop 92 Verifying data now No change after verify Loop: 92 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 178 178 178 335 335 335 F F F 1 020-039 185 185 185 348 348 348 F F F 2 040-059 180 180 180 338 338 338 F F F 3 060-079 180 180 180 337 337 337 F F F 4 080-099 185 185 185 348 348 348 F F F 5 100-119 184 184 184 347 347 347 F F F 6 120-139 183 183 183 345 345 345 F F F 7 140-159 175 175 175 327 327 327 F F F 8 160-179 183 183 183 345 345 345 F F F 9 180-199 184 184 184 347 347 347 F F F 10 200-219 179 179 179 336 336 336 F F F 11 220-239 184 184 184 347 347 347 F F F 12 240-259 184 184 184 347 347 347 F F F 13 260-279 182 182 182 341 341 341 F F F 14 280-299 185 185 185 348 348 348 F F F 15 300-319 184 184 184 347 347 347 F F F 16 320-339 185 185 185 348 348 348 F F F 17 340-359 185 185 185 348 348 348 F F F 18 360-379 182 182 182 344 344 344 F F F 19 380-399 180 180 180 337 337 337 F F F 20 400-419 185 185 185 348 348 348 F F F 21 420-439 185 185 185 348 348 348 F F F 22 440-459 178 178 178 335 335 335 F F F 23 460-479 182 182 182 342 342 342 F F F 24 480-499 182 182 182 342 342 342 F F F 25 500-519 182 182 182 342 342 342 F F F 26 520-539 183 183 183 345 345 345 F F F 27 540-559 182 182 182 341 341 341 F F F 28 560-579 178 178 178 334 334 334 F F F 29 580-599 185 185 185 348 348 348 F F F Max gen: 185, Max flush: 348 Begin loop 93 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:00.315088482Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1926,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:00.35964192Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1926,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:00.360350431Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1926} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"26e7d501b33f47379e53601720353c2983e7654b\",\n git_commit_timestamp: \"2024-12-12T20:16:40.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:00.360386768Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1926} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:00.360401187Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1926} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:00.360415876Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1926} {"msg":"Crucible 2463420c-bac1-42d4-9ca6-f36a27e52a08 has session id: 80281d4b-d207-4884-96e1-8f8bb521db09","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:00.36080102Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1926,"session_id":"80281d4b-d207-4884-96e1-8f8bb521db09"} {"msg":"Upstairs opts: Upstairs UUID: 2463420c-bac1-42d4-9ca6-f36a27e52a08, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:00.360844967Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1926,"session_id":"80281d4b-d207-4884-96e1-8f8bb521db09"} {"msg":"Crucible stats registered with UUID: 2463420c-bac1-42d4-9ca6-f36a27e52a08","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:00.360875215Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1926,"session_id":"80281d4b-d207-4884-96e1-8f8bb521db09"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:00.392056654Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1926,"task":"crutest"} {"msg":"The guest has requested activation with gen:186","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:00.392089202Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1926} {"msg":"2463420c-bac1-42d4-9ca6-f36a27e52a08 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:00.392171897Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1926,"session_id":"80281d4b-d207-4884-96e1-8f8bb521db09"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:00.392194415Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1926,"client":"0","":"downstairs","session_id":"80281d4b-d207-4884-96e1-8f8bb521db09"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:00.392223473Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1926,"client":"0","":"downstairs","session_id":"80281d4b-d207-4884-96e1-8f8bb521db09"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:00.392241172Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1926,"client":"1","":"downstairs","session_id":"80281d4b-d207-4884-96e1-8f8bb521db09"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:00.39225908Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1926,"client":"1","":"downstairs","session_id":"80281d4b-d207-4884-96e1-8f8bb521db09"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:00.392283719Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1926,"client":"2","":"downstairs","session_id":"80281d4b-d207-4884-96e1-8f8bb521db09"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:00.392317047Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1926,"client":"2","":"downstairs","session_id":"80281d4b-d207-4884-96e1-8f8bb521db09"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:00.392349924Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1926,"":"io task","client":"0","":"downstairs","session_id":"80281d4b-d207-4884-96e1-8f8bb521db09"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:00.392382982Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1926,"":"io task","client":"2","":"downstairs","session_id":"80281d4b-d207-4884-96e1-8f8bb521db09"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:00.39241756Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1926,"":"io task","client":"1","":"downstairs","session_id":"80281d4b-d207-4884-96e1-8f8bb521db09"} {"msg":"ds_connection connected from Ok(127.0.0.1:52580)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:00.392461417Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1926,"":"io task","client":"0","":"downstairs","session_id":"80281d4b-d207-4884-96e1-8f8bb521db09"} {"msg":"ds_connection connected from Ok(127.0.0.1:44663)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:00.392497094Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1926,"":"io task","client":"1","":"downstairs","session_id":"80281d4b-d207-4884-96e1-8f8bb521db09"} {"msg":"ds_connection connected from Ok(127.0.0.1:45438)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:00.39255074Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1926,"":"io task","client":"2","":"downstairs","session_id":"80281d4b-d207-4884-96e1-8f8bb521db09"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:00.393366294Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1926,"client":"0","":"downstairs","session_id":"80281d4b-d207-4884-96e1-8f8bb521db09"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:00.393401422Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1926,"client":"2","":"downstairs","session_id":"80281d4b-d207-4884-96e1-8f8bb521db09"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:00.39343154Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1926,"client":"1","":"downstairs","session_id":"80281d4b-d207-4884-96e1-8f8bb521db09"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:00.393976022Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1926,"client":"0","":"downstairs","session_id":"80281d4b-d207-4884-96e1-8f8bb521db09"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:00.394027569Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1926,"client":"1","":"downstairs","session_id":"80281d4b-d207-4884-96e1-8f8bb521db09"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:00.394132221Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1926,"client":"2","":"downstairs","session_id":"80281d4b-d207-4884-96e1-8f8bb521db09"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:00.394544893Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1926,"session_id":"80281d4b-d207-4884-96e1-8f8bb521db09"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:00.394628247Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1926,"session_id":"80281d4b-d207-4884-96e1-8f8bb521db09"} {"msg":"[0]R flush_numbers[0..12]: [335, 348, 338, 337, 348, 347, 345, 327, 345, 347, 336, 347]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:00.394751249Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1926,"":"downstairs","":"downstairs","session_id":"80281d4b-d207-4884-96e1-8f8bb521db09"} {"msg":"[0]R generation[0..12]: [178, 185, 180, 180, 185, 184, 183, 175, 183, 184, 179, 184]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:00.394771567Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1926,"":"downstairs","":"downstairs","session_id":"80281d4b-d207-4884-96e1-8f8bb521db09"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:00.394786776Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1926,"":"downstairs","":"downstairs","session_id":"80281d4b-d207-4884-96e1-8f8bb521db09"} {"msg":"[1]R flush_numbers[0..12]: [335, 348, 338, 337, 348, 347, 345, 327, 345, 347, 336, 347]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:00.394800965Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1926,"":"downstairs","":"downstairs","session_id":"80281d4b-d207-4884-96e1-8f8bb521db09"} {"msg":"[1]R generation[0..12]: [178, 185, 180, 180, 185, 184, 183, 175, 183, 184, 179, 184]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:00.394814854Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1926,"":"downstairs","":"downstairs","session_id":"80281d4b-d207-4884-96e1-8f8bb521db09"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:00.394831873Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1926,"":"downstairs","":"downstairs","session_id":"80281d4b-d207-4884-96e1-8f8bb521db09"} {"msg":"[2]R flush_numbers[0..12]: [335, 348, 338, 337, 348, 347, 345, 327, 345, 347, 336, 347]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:00.394846912Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1926,"":"downstairs","":"downstairs","session_id":"80281d4b-d207-4884-96e1-8f8bb521db09"} {"msg":"[2]R generation[0..12]: [178, 185, 180, 180, 185, 184, 183, 175, 183, 184, 179, 184]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:00.394860721Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1926,"":"downstairs","":"downstairs","session_id":"80281d4b-d207-4884-96e1-8f8bb521db09"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:00.39487463Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1926,"":"downstairs","":"downstairs","session_id":"80281d4b-d207-4884-96e1-8f8bb521db09"} {"msg":"Max found gen is 186","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:00.394888559Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1926,"":"downstairs","":"downstairs","session_id":"80281d4b-d207-4884-96e1-8f8bb521db09"} {"msg":"Generation requested: 186 >= found:186","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:00.394902138Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1926Wait for a query_work_queue command to finish before sending IO ,"":"downstairs","":"downstairs","session_id":"80281d4b-d207-4884-96e1-8f8bb521db09"} {"msg":"Next flush: 349","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:00.394919047Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1926,"":"downstairs","":"downstairs","session_id":"80281d4b-d207-4884-96e1-8f8bb521db09"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:00.394932166Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1926,"":"downstairs","":"downstairs","session_id":"80281d4b-d207-4884-96e1-8f8bb521db09"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:00.394945415Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1926,"session_id":"80281d4b-d207-4884-96e1-8f8bb521db09"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:00.394957194Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1926,"client":"0","":"downstairs","session_id":"80281d4b-d207-4884-96e1-8f8bb521db09"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:00.394970353Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1926,"client":"1","":"downstairs","session_id":"80281d4b-d207-4884-96e1-8f8bb521db09"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:00.394984433Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1926,"client":"2","":"downstairs","session_id":"80281d4b-d207-4884-96e1-8f8bb521db09"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:00.394997932Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1926,"session_id":"80281d4b-d207-4884-96e1-8f8bb521db09"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:00.395010231Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1926,"session_id":"80281d4b-d207-4884-96e1-8f8bb521db09"} {"msg":"2463420c-bac1-42d4-9ca6-f36a27e52a08 is now active with session: 80281d4b-d207-4884-96e1-8f8bb521db09","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:00.39502305Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1926,"session_id":"80281d4b-d207-4884-96e1-8f8bb521db09"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:00.395036619Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1926,"session_id":"80281d4b-d207-4884-96e1-8f8bb521db09"} {"msg":"The guest has finished waiting for activation with:186","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:00.395049138Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1926} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 544 len 36864 data: 13 13 12 13 13 13 13 12 12 02/30 Write block 431 len 12288 data: 16 16 16 03/30 Read block 192 len 16384 04/30 Write block 509 len 40960 data: 12 12 12 13 13 9 8 11 14 19 05/30 Write block 349 len 8192 data: 13 16 06/30 Read block 381 len 36864 07/30 Read block 122 len 28672 08/30 Read block 202 len 40960 09/30 Write block 284 len 32768 data: 11 12 12 11 10 10 10 13 10/30 Read block 455 len 4096 11/30 Write block 189 len 28672 data: 18 16 17 18 19 15 16 12/30 Write block 482 len 20480 data: 11 9 13 16 15 13/30 Read block 411 len 36864 14/30 Read block 457 len 40960 15/30 Read block 304 len 24576 16/30 Read block 159 len 8192 17/30 Read block 227 len 40960 18/30 Write block 457 len 8192 data: 20 23 19/30 Write block 34 len 32768 data: 16 17 15 15 15 16 14 14 20/30 Write block 249 len 32768 data: 12 10 11 11 14 17 18 14 21/30 Read block 173 len 12288 22/30 Write block 489 len 4096 data: 20 23/30 Flush 24/30 Write block 122 len 20480 data: 19 17 15 16 16 25/30 Write block 14 len 36864 data: 13 14 13 12 13 9 13 11 14 26/30 Write block 509 len 24576 data: 13 13 13 14 14 10 27/30 Write block 433 len 36864 data: 17 12 16 13 12 12 10 10 10 28/30 Write block 318 len 8192 data: 18 16 29/30 Write block 345 len 32768 data: 19 19 17 15 14 17 12 16 30/30 Write block 583 len 12288 data: 17 19 20 ---------------------------------------------------------------- Crucible gen:186 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 5 Done Done Done false 1024 Acked Write 9 Done Done Done false 1025 Acked Write 6 Done Done Done false 1026 Acked Write 9 Done Done Done false 1027 Acked Write 2 Done Done Sent false 1028 Acked Write 8 Sent Sent Sent false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 3 7 Done 5 5 4 14 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:40:00.414254044Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1926} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 178 178 178 335 335 335 T T T 1 020-039 186 186 186 349 349 349 T T T 2 040-059 186 186 186 349 349 349 F F F 3 060-079 180 180 180 337 337 337 F F F 4 080-099 185 185 185 348 348 348 F F F 5 100-119 184 184 184 347 347 347 F F F 6 120-139 183 183 183 345 345 345 T T T 7 140-159 175 175 175 327 327 327 F F F 8 160-179 183 183 183 345 345 345 F F F 9 180-199 186 186 186 349 349 349 F F F 10 200-219 179 179 179 336 336 336 F F F 11 220-239 184 184 184 347 347 347 F F F 12 240-259 186 186 186 349 349 349 F F F 13 260-279 182 182 182 341 341 341 F F F 14 280-299 186 186 186 349 349 349 F F F 15 300-319 184 184 184 347 347 347 T T T 16 320-339 185 185 185 348 348 348 F F F 17 340-359 186 186 186 349 349 349 T T T 18 360-379 182 182 182 344 344 344 F F F 19 380-399 180 180 180 337 337 337 F F F 20 400-419 185 185 185 348 348 348 F F F 21 420-439 186 186 186 349 349 349 T T T 22 440-459 186 186 186 349 349 349 T T T 23 460-479 182 182 182 342 342 342 F F F 24 480-499 186 186 186 349 349 349 F F F 25 500-519 186 186 186 349 349 349 T T T 26 520-539 183 183 183 345 345 345 F F F 27 540-559 186 186 186 349 349 349 F F F 28 560-579 178 178 178 334 334 334 F F F 29 580-599 185 185 185 348 348 348 T T T Max gen: 186, Max flush: 349 A Difference in extent metadata is expected here On loop 93 Verifying data now No change after verify Loop: 93 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 187 187 187 350 350 350 F F F 1 020-039 187 187 187 350 350 350 F F F 2 040-059 186 186 186 349 349 349 F F F 3 060-079 180 180 180 337 337 337 F F F 4 080-099 185 185 185 348 348 348 F F F 5 100-119 184 184 184 347 347 347 F F F 6 120-139 187 187 187 350 350 350 F F F 7 140-159 175 175 175 327 327 327 F F F 8 160-179 183 183 183 345 345 345 F F F 9 180-199 186 186 186 349 349 349 F F F 10 200-219 179 179 179 336 336 336 F F F 11 220-239 184 184 184 347 347 347 F F F 12 240-259 186 186 186 349 349 349 F F F 13 260-279 182 182 182 341 341 341 F F F 14 280-299 186 186 186 349 349 349 F F F 15 300-319 187 187 187 350 350 350 F F F 16 320-339 185 185 185 348 348 348 F F F 17 340-359 187 187 187 350 350 350 F F F 18 360-379 182 182 182 344 344 344 F F F 19 380-399 180 180 180 337 337 337 F F F 20 400-419 185 185 185 348 348 348 F F F 21 420-439 187 187 187 350 350 350 F F F 22 440-459 187 187 187 350 350 350 F F F 23 460-479 182 182 182 342 342 342 F F F 24 480-499 186 186 186 349 349 349 F F F 25 500-519 187 187 187 350 350 350 F F F 26 520-539 183 183 183 345 345 345 F F F 27 540-559 186 186 186 349 349 349 F F F 28 560-579 178 178 178 334 334 334 F F F 29 580-599 187 187 187 350 350 350 F F F Max gen: 187, Max flush: 350 Begin loop 94 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:04.712241412Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1934,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:04.757428756Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1934,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:04.758043074Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1934} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"26e7d501b33f47379e53601720353c2983e7654b\",\n git_commit_timestamp: \"2024-12-12T20:16:40.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:04.758079291Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1934} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:04.75809843Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1934} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:04.758113599Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1934} {"msg":"Crucible 930e7465-21d3-4f00-99bd-c56d9189ba87 has session id: f3387521-3f15-4873-8ce3-aeb285807540","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:04.758541049Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1934,"session_id":"f3387521-3f15-4873-8ce3-aeb285807540"} {"msg":"Upstairs opts: Upstairs UUID: 930e7465-21d3-4f00-99bd-c56d9189ba87, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:04.758590406Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1934,"session_id":"f3387521-3f15-4873-8ce3-aeb285807540"} {"msg":"Crucible stats registered with UUID: 930e7465-21d3-4f00-99bd-c56d9189ba87","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:04.758617394Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1934,"session_id":"f3387521-3f15-4873-8ce3-aeb285807540"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:04.789522333Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1934,"task":"crutest"} {"msg":"The guest has requested activation with gen:188","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:04.789554271Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1934} {"msg":"930e7465-21d3-4f00-99bd-c56d9189ba87 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:04.789585278Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1934,"session_id":"f3387521-3f15-4873-8ce3-aeb285807540"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:04.789614066Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1934,"client":"0","":"downstairs","session_id":"f3387521-3f15-4873-8ce3-aeb285807540"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:04.789646174Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1934,"client":"0","":"downstairs","session_id":"f3387521-3f15-4873-8ce3-aeb285807540"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:04.789663723Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1934,"client":"1","":"downstairs","session_id":"f3387521-3f15-4873-8ce3-aeb285807540"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:04.789681542Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1934,"client":"1","":"downstairs","session_id":"f3387521-3f15-4873-8ce3-aeb285807540"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:04.789698021Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1934,"client":"2","":"downstairs","session_id":"f3387521-3f15-4873-8ce3-aeb285807540"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:04.789718459Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1934,"client":"2","":"downstairs","session_id":"f3387521-3f15-4873-8ce3-aeb285807540"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:04.789753257Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1934,"":"io task","client":"2","":"downstairs","session_id":"f3387521-3f15-4873-8ce3-aeb285807540"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:04.789786295Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1934,"":"io task","client":"0","":"downstairs","session_id":"f3387521-3f15-4873-8ce3-aeb285807540"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:04.789818242Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1934,"":"io task","client":"1","":"downstairs","session_id":"f3387521-3f15-4873-8ce3-aeb285807540"} {"msg":"ds_connection connected from Ok(127.0.0.1:48892)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:04.78985248Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1934,"":"io task","client":"2","":"downstairs","session_id":"f3387521-3f15-4873-8ce3-aeb285807540"} {"msg":"ds_connection connected from Ok(127.0.0.1:57487)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:04.789885618Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1934,"":"io task","client":"1","":"downstairs","session_id":"f3387521-3f15-4873-8ce3-aeb285807540"} {"msg":"ds_connection connected from Ok(127.0.0.1:54507)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:04.789937194Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1934,"":"io task","client":"0","":"downstairs","session_id":"f3387521-3f15-4873-8ce3-aeb285807540"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:04.790570121Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1934,"client":"2","":"downstairs","session_id":"f3387521-3f15-4873-8ce3-aeb285807540"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:04.790610388Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1934,"client":"0","":"downstairs","session_id":"f3387521-3f15-4873-8ce3-aeb285807540"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:04.790746758Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1934,"client":"1","":"downstairs","session_id":"f3387521-3f15-4873-8ce3-aeb285807540"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:04.79129468Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1934,"client":"0","":"downstairs","session_id":"f3387521-3f15-4873-8ce3-aeb285807540"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:04.791372745Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1934,"client":"2","":"downstairs","session_id":"f3387521-3f15-4873-8ce3-aeb285807540"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:04.791403293Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1934,"client":"1","":"downstairs","session_id":"f3387521-3f15-4873-8ce3-aeb285807540"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:04.791868771Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1934,"session_id":"f3387521-3f15-4873-8ce3-aeb285807540"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:04.791895309Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1934,"session_id":"f3387521-3f15-4873-8ce3-aeb285807540"} {"msg":"[0]R flush_numbers[0..12]: [350, 350, 349, 337, 348, 347, 350, 327, 345, 349, 336, 347]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:04.791953415Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1934,"":"downstairs","":"downstairs","session_id":"f3387521-3f15-4873-8ce3-aeb285807540"} {"msg":"[0]R generation[0..12]: [187, 187, 186, 180, 185, 184, 187, 175, 183, 186, 179, 184]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:04.791978783Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1934,"":"downstairs","":"downstairs","session_id":"f3387521-3f15-4873-8ce3-aeb285807540"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:04.791994462Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1934,"":"downstairs","":"downstairs","session_id":"f3387521-3f15-4873-8ce3-aeb285807540"} {"msg":"[1]R flush_numbers[0..12]: [350, 350, 349, 337, 348, 347, 350, 327, 345, 349, 336, 347]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:04.792008651Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1934,"":"downstairs","":"downstairs","session_id":"f3387521-3f15-4873-8ce3-aeb285807540"} {"msg":"[1]R generation[0..12]: [187, 187, 186, 180, 185, 184, 187, 175, 183, 186, 179, 184]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:04.79202303Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1934,"":"downstairs","":"downstairs","session_id":"f3387521-3f15-4873-8ce3-aeb285807540"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:04.792038239Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1934,"":"downstairs","":"downstairs","session_id":"f3387521-3f15-4873-8ce3-aeb285807540"} {"msg":"[2]R flush_numbers[0..12]: [350, 350, 349, 337, 348, 347, 350, 327, 345, 349, 336, 347]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:04.792052338Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1934,"":"downstairs","":"downstairs","session_id":"f3387521-3f15-4873-8ce3-aeb285807540"} {"msg":"[2]R generation[0..12]: [187, 187, 186, 180, 185, 184, 187, 175, 183, 186, 179, 184]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:04.792066157Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1934,"":"downstairs","":"downstairs","session_id":"f3387521-3f15-4873-8ce3-aeb285807540"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:04.792080076Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1934,"":"downstairs","":"downstairs","session_id":"f3387521-3f15-4873-8ce3-aeb285807540"} {"msg":"Max found gen is 188","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:04.792093216Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1934,"":"downstairs","":"downstairs","session_id":"f3387521-3f15-4873-8ce3-aeb285807540"} {"msg":"Generation requested: 188 >= found:188","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:04.792106535Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1934,"":"downstairs","":"downstairs","session_id":"f3387521-3f15-4873-8ce3-aeb285807540"} {"msg":"Next flush: 351","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:04.792119404Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1934,"":"downstairs","":"downstairs","session_id":"f3387521-3f15-4873-8ce3-aeb285807540"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:04.792132223Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1934,"":"downstairs","":"downstairs","session_id":"f3387521-3f15-4873-8ce3-aeb285807540"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:04.792145332Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1934,"session_id":"f3387521-3f15-4873-8ce3-aeb285807540"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:04.792157351Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1934,"client":"0","":"downstairs","session_id":"f3387521-3f15-4873-8ce3-aeb285807540"} Wait for a query_work_queue command to finish before sending IO {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:04.79217728Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1934,"client":"1","":"downstairs","session_id":"f3387521-3f15-4873-8ce3-aeb285807540"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:04.792190859Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1934,"client":"2","":"downstairs","session_id":"f3387521-3f15-4873-8ce3-aeb285807540"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:04.792204148Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1934,"session_id":"f3387521-3f15-4873-8ce3-aeb285807540"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:04.792216497Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1934,"session_id":"f3387521-3f15-4873-8ce3-aeb285807540"} {"msg":"930e7465-21d3-4f00-99bd-c56d9189ba87 is now active with session: f3387521-3f15-4873-8ce3-aeb285807540","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:04.792229036Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1934,"session_id":"f3387521-3f15-4873-8ce3-aeb285807540"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:04.792240875Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1934,"session_id":"f3387521-3f15-4873-8ce3-aeb285807540"} {"msg":"The guest has finished waiting for activation with:188","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:04.792253074Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1934} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 468 len 4096 data: 14 02/30 Flush 03/30 Write block 363 len 28672 data: 13 17 18 20 17 14 13 04/30 Write block 38 len 20480 data: 16 17 15 15 13 05/30 Read block 377 len 28672 06/30 Write block 165 len 4096 data: 13 07/30 Write block 260 len 32768 data: 16 12 13 17 17 22 25 25 08/30 Read block 22 len 24576 09/30 Write block 259 len 4096 data: 13 10/30 Write block 377 len 20480 data: 10 13 17 18 18 11/30 Write block 35 len 24576 data: 18 16 16 17 18 16 12/30 Read block 108 len 12288 13/30 Read block 227 len 36864 14/30 Read block 36 len 20480 15/30 Write block 535 len 28672 data: 7 11 13 11 10 12 9 16/30 Flush 17/30 Write block 179 len 20480 data: 15 12 12 12 13 18/30 Read block 298 len 36864 19/30 Read block 202 len 16384 20/30 Write block 539 len 12288 data: 11 13 10 21/30 Read block 375 len 24576 22/30 Read block 336 len 8192 23/30 Read block 348 len 20480 24/30 Flush 25/30 Write block 592 len 28672 data: 19 16 12 10 8 6 5 26/30 Write block 130 len 20480 data: 14 10 8 7 8 27/30 Write block 525 len 8192 data: 19 11 28/30 Write block 396 len 12288 data: 14 16 17 29/30 Write block 298 len 20480 data: 10 10 12 11 10 30/30 Write block 220 len 24576 data: 18 14 12 15 14 15 ---------------------------------------------------------------- Crucible gen:188 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 7 Done Done Done false 1025 Acked Write 5 Done Done Done false 1026 Acked Write 2 Done Done Done false 1027 Acked Write 3 Done Done Done false 1028 Acked Write 5 Sent Sent Sent false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:40:04.810222325Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1934} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 187 187 187 350 350 350 F F F 1 020-039 188 188 188 352 352 352 F F F 2 040-059 188 188 188 352 352 352 F F F 3 060-079 180 180 180 337 337 337 F F F 4 080-099 185 185 185 348 348 348 F F F 5 100-119 184 184 184 347 347 347 F F F 6 120-139 187 187 187 350 350 350 T T T 7 140-159 175 175 175 327 327 327 F F F 8 160-179 188 188 188 353 353 353 F F F 9 180-199 188 188 188 353 353 353 F F F 10 200-219 179 179 179 336 336 336 F F F 11 220-239 184 184 184 347 347 347 T T T 12 240-259 188 188 188 352 352 352 F F F 13 260-279 188 188 188 352 352 352 F F F 14 280-299 186 186 186 349 349 349 T T T 15 300-319 187 187 187 350 350 350 T T T 16 320-339 185 185 185 348 348 348 F F F 17 340-359 187 187 187 350 350 350 F F F 18 360-379 188 188 188 352 352 352 F F F 19 380-399 188 188 188 352 352 352 T T T 20 400-419 185 185 185 348 348 348 F F F 21 420-439 187 187 187 350 350 350 F F F 22 440-459 187 187 187 350 350 350 F F F 23 460-479 188 188 188 351 351 351 F F F 24 480-499 186 186 186 349 349 349 F F F 25 500-519 187 187 187 350 350 350 F F F 26 520-539 188 188 188 353 353 353 T T T 27 540-559 188 188 188 353 353 353 F F F 28 560-579 178 178 178 334 334 334 F F F 29 580-599 187 187 187 350 350 350 T T T Max gen: 188, Max flush: 353 A Difference in extent metadata is expected here On loop 94 Verifying data now No change after verify Loop: 94 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 187 187 187 350 350 350 F F F 1 020-039 188 188 188 352 352 352 F F F 2 040-059 188 188 188 352 352 352 F F F 3 060-079 180 180 180 337 337 337 F F F 4 080-099 185 185 185 348 348 348 F F F 5 100-119 184 184 184 347 347 347 F F F 6 120-139 189 189 189 354 354 354 F F F 7 140-159 175 175 175 327 327 327 F F F 8 160-179 188 188 188 353 353 353 F F F 9 180-199 188 188 188 353 353 353 F F F 10 200-219 179 179 179 336 336 336 F F F 11 220-239 189 189 189 354 354 354 F F F 12 240-259 188 188 188 352 352 352 F F F 13 260-279 188 188 188 352 352 352 F F F 14 280-299 189 189 189 354 354 354 F F F 15 300-319 189 189 189 354 354 354 F F F 16 320-339 185 185 185 348 348 348 F F F 17 340-359 187 187 187 350 350 350 F F F 18 360-379 188 188 188 352 352 352 F F F 19 380-399 189 189 189 354 354 354 F F F 20 400-419 185 185 185 348 348 348 F F F 21 420-439 187 187 187 350 350 350 F F F 22 440-459 187 187 187 350 350 350 F F F 23 460-479 188 188 188 351 351 351 F F F 24 480-499 186 186 186 349 349 349 F F F 25 500-519 187 187 187 350 350 350 F F F 26 520-539 189 189 189 354 354 354 F F F 27 540-559 188 188 188 353 353 353 F F F 28 560-579 178 178 178 334 334 334 F F F 29 580-599 189 189 189 354 354 354 F F F Max gen: 189, Max flush: 354 Begin loop 95 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:09.091109981Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1942,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:09.135574465Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1942,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:09.13623924Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1942} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"26e7d501b33f47379e53601720353c2983e7654b\",\n git_commit_timestamp: \"2024-12-12T20:16:40.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:09.136275567Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1942} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:09.136289696Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1942} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:09.136304085Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1942} {"msg":"Crucible 3c11e6c7-d348-46a5-b7c1-bc988023f6c2 has session id: 4a42bfba-b4f3-44c6-93cf-2b377cabe2ec","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:09.136752294Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1942,"session_id":"4a42bfba-b4f3-44c6-93cf-2b377cabe2ec"} {"msg":"Upstairs opts: Upstairs UUID: 3c11e6c7-d348-46a5-b7c1-bc988023f6c2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:09.136798601Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1942,"session_id":"4a42bfba-b4f3-44c6-93cf-2b377cabe2ec"} {"msg":"Crucible stats registered with UUID: 3c11e6c7-d348-46a5-b7c1-bc988023f6c2","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:09.136824599Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1942,"session_id":"4a42bfba-b4f3-44c6-93cf-2b377cabe2ec"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:09.168256132Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1942,"task":"crutest"} {"msg":"The guest has requested activation with gen:190","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:09.168288279Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1942} {"msg":"3c11e6c7-d348-46a5-b7c1-bc988023f6c2 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:09.168336886Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1942,"session_id":"4a42bfba-b4f3-44c6-93cf-2b377cabe2ec"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:09.168356105Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1942,"client":"0","":"downstairs","session_id":"4a42bfba-b4f3-44c6-93cf-2b377cabe2ec"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:09.168388573Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1942,"client":"0","":"downstairs","session_id":"4a42bfba-b4f3-44c6-93cf-2b377cabe2ec"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:09.1684265Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1942,"client":"1","":"downstairs","session_id":"4a42bfba-b4f3-44c6-93cf-2b377cabe2ec"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:09.168463827Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1942,"client":"1","":"downstairs","session_id":"4a42bfba-b4f3-44c6-93cf-2b377cabe2ec"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:09.168498265Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1942,"client":"2","":"downstairs","session_id":"4a42bfba-b4f3-44c6-93cf-2b377cabe2ec"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:09.168537013Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1942,"client":"2","":"downstairs","session_id":"4a42bfba-b4f3-44c6-93cf-2b377cabe2ec"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:09.16857255Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1942,"":"io task","client":"2","":"downstairs","session_id":"4a42bfba-b4f3-44c6-93cf-2b377cabe2ec"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:09.168607568Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1942,"":"io task","client":"0","":"downstairs","session_id":"4a42bfba-b4f3-44c6-93cf-2b377cabe2ec"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:09.168641275Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1942,"":"io task","client":"1","":"downstairs","session_id":"4a42bfba-b4f3-44c6-93cf-2b377cabe2ec"} {"msg":"ds_connection connected from Ok(127.0.0.1:56773)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:09.168708341Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1942,"":"io task","client":"2","":"downstairs","session_id":"4a42bfba-b4f3-44c6-93cf-2b377cabe2ec"} {"msg":"ds_connection connected from Ok(127.0.0.1:57362)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:09.168744808Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1942,"":"io task","client":"0","":"downstairs","session_id":"4a42bfba-b4f3-44c6-93cf-2b377cabe2ec"} {"msg":"ds_connection connected from Ok(127.0.0.1:37592)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:09.168787415Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1942,"":"io task","client":"1","":"downstairs","session_id":"4a42bfba-b4f3-44c6-93cf-2b377cabe2ec"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:09.169331098Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1942,"client":"0","":"downstairs","session_id":"4a42bfba-b4f3-44c6-93cf-2b377cabe2ec"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:09.169361256Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1942,"client":"1","":"downstairs","session_id":"4a42bfba-b4f3-44c6-93cf-2b377cabe2ec"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:09.169377315Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1942,"client":"2","":"downstairs","session_id":"4a42bfba-b4f3-44c6-93cf-2b377cabe2ec"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:09.17002006Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1942,"client":"1","":"downstairs","session_id":"4a42bfba-b4f3-44c6-93cf-2b377cabe2ec"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:09.170053818Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1942,"client":"0","":"downstairs","session_id":"4a42bfba-b4f3-44c6-93cf-2b377cabe2ec"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:09.170075316Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1942,"client":"2","":"downstairs","session_id":"4a42bfba-b4f3-44c6-93cf-2b377cabe2ec"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:09.17060283Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1942,"session_id":"4a42bfba-b4f3-44c6-93cf-2b377cabe2ec"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:09.170639277Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1942,"session_id":"4a42bfba-b4f3-44c6-93cf-2b377cabe2ec"} {"msg":"[0]R flush_numbers[0..12]: [350, 352, 352, 337, 348, 347, 354, 327, 353, 353, 336, 354]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:09.170765259Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1942,"":"downstairs","":"downstairs","session_id":"4a42bfba-b4f3-44c6-93cf-2b377cabe2ec"} {"msg":"[0]R generation[0..12]: [187, 188, 188, 180, 185, 184, 189, 175, 188, 188, 179, 189]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:09.170786237Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1942,"":"downstairs","":"downstairs","session_id":"4a42bfba-b4f3-44c6-93cf-2b377cabe2ec"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:09.170800816Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1942,"":"downstairs","":"downstairs","session_id":"4a42bfba-b4f3-44c6-93cf-2b377cabe2ec"} {"msg":"[1]R flush_numbers[0..12]: [350, 352, 352, 337, 348, 347, 354, 327, 353, 353, 336, 354]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:09.170814945Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1942,"":"downstairs","":"downstairs","session_id":"4a42bfba-b4f3-44c6-93cf-2b377cabe2ec"} {"msg":"[1]R generation[0..12]: [187, 188, 188, 180, 185, 184, 189, 175, 188, 188, 179, 189]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:09.170828834Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1942,"":"downstairs","":"downstairs","session_id":"4a42bfba-b4f3-44c6-93cf-2b377cabe2ec"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:09.170842894Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1942,"":"downstairs","":"downstairs","session_id":"4a42bfba-b4f3-44c6-93cf-2b377cabe2ec"} {"msg":"[2]R flush_numbers[0..12]: [350, 352, 352, 337, 348, 347, 354, 327, 353, 353, 336, 354]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:09.170856862Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1942,"":"downstairs","":"downstairs","session_id":"4a42bfba-b4f3-44c6-93cf-2b377cabe2ec"} {"msg":"[2]R generation[0..12]: [187, 188, 188, 180, 185, 184, 189, 175, 188, 188, 179, 189]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:09.170870632Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1942,"":"downstairs","":"downstairs","session_id":"4a42bfba-b4f3-44c6-93cf-2b377cabe2ec"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:09.17088456Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1942,"":"downstairs","":"downstairs","session_id":"4a42bfba-b4f3-44c6-93cf-2b377cabe2ec"} {"msg":"Max found gen is 190","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:09.17089781Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1942,"":"downstairs","":"downstairs","session_id":"4a42bfba-b4f3-44c6-93cf-2b377cabe2ec"} {"msg":"Generation requested: 190 >= found:190","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:09.170914318Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1942,"":"downstairs","":"downstairs","session_id":"4a42bfba-b4f3-44c6-93cf-2b377cabe2ec"} {"msg":"Next flush: 355","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:09.170927857Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1942,"":"downstairs","":"downstairs","session_id":"4a42bfba-b4f3-44c6-93cf-2b377cabe2ec"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-12-12T20:40:09.170940997Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1942,"":"downstairs","":"downstairs","session_id":"4a42bfba-b4f3-44c6-93cf-2b377cabe2ec"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:09.170957906Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1942,"session_id":"4a42bfba-b4f3-44c6-93cf-2b377cabe2ec"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:09.170970155Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1942,"client":"0","":"downstairs","session_id":"4a42bfba-b4f3-44c6-93cf-2b377cabe2ec"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:09.170983464Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1942,"client":"1","":"downstairs","session_id":"4a42bfba-b4f3-44c6-93cf-2b377cabe2ec"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:09.170996563Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1942,"client":"2","":"downstairs","session_id":"4a42bfba-b4f3-44c6-93cf-2b377cabe2ec"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:09.171009852Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1942,"session_id":"4a42bfba-b4f3-44c6-93cf-2b377cabe2ec"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:09.171022051Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1942,"session_id":"4a42bfba-b4f3-44c6-93cf-2b377cabe2ec"} {"msg":"3c11e6c7-d348-46a5-b7c1-bc988023f6c2 is now active with session: 4a42bfba-b4f3-44c6-93cf-2b377cabe2ec","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:09.1710349Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1942,"session_id":"4a42bfba-b4f3-44c6-93cf-2b377cabe2ec"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:09.171046919Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1942,"session_id":"4a42bfba-b4f3-44c6-93cf-2b377cabe2ec"} {"msg":"The guest has finished waiting for activation with:190","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2024-12-12T20:40:09.171059259Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1942} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 529 len 24576 data: 11 6 7 8 8 8 02/30 Flush 03/30 Write block 380 len 8192 data: 19 19 04/30 Read block 411 len 24576 05/30 Read block 502 len 8192 06/30 Read block 62 len 16384 07/30 Read block 277 len 36864 08/30 Write block 453 len 28672 data: 16 19 17 17 21 24 20 09/30 Write block 389 len 12288 data: 11 14 13 10/30 Flush 11/30 Write block 321 len 28672 data: 16 15 13 11 8 10 9 12/30 Read block 387 len 28672 13/30 Flush 14/30 Write block 168 len 36864 data: 11 12 12 9 8 10 10 13 13 15/30 Read block 39 len 36864 16/30 Read block 499 len 28672 17/30 Read block 437 len 20480 18/30 Read block 12 len 16384 19/30 Read block 346 len 40960 20/30 Read block 306 len 40960 21/30 Read block 142 len 32768 22/30 Read block 390 len 24576 23/30 Read block 571 len 12288 24/30 Read block 14 len 20480 25/30 Read block 589 len 36864 26/30 Read block 50 len 12288 27/30 Read block 100 len 32768 28/30 Write block 19 len 24576 data: 10 14 12 15 17 19 29/30 Read block 173 len 20480 30/30 Write block 311 len 12288 data: 18 13 17 ---------------------------------------------------------------- Crucible gen:190 GIO:true work queues: Upstairs:0 downstairs:17 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1013 Acked Write 9 Done Done Done false 1014 Acked Read 9 Done Done Done false 1015 Acked Read 7 Done Done Done false 1016 Acked Read 5 Done Done Done false 1017 Acked Read 4 Done Done Done false 1018 Acked Read 10 Done Done Done false 1019 Acked Read 10 Done Done Done false 1020 Acked Read 8 Done Done Done false 1021 Acked Read 6 Done Done Done false 1022 Acked Read 3 Done Done Done false 1023 Acked Read 5 Done Done Done false 1024 Acked Read 9 Done Done Done false 1025 Acked Read 3 Done Done Done false 1026 Acked Read 8 Done Done Done false 1027 Acked Write 6 Done Done Done false 1028 Acked Read 5 Done Done Sent false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 2 4 Done 16 16 15 47 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1012 1012 1012 Downstairs last five completed: 1012 1011 1010 1009 1008 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:40:09.191426264Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1942} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 187 187 187 350 350 350 T T T 1 020-039 188 188 188 352 352 352 T T T 2 040-059 188 188 188 352 352 352 F F F 3 060-079 180 180 180 337 337 337 F F F 4 080-099 185 185 185 348 348 348 F F F 5 100-119 184 184 184 347 347 347 F F F 6 120-139 189 189 189 354 354 354 F F F 7 140-159 175 175 175 327 327 327 F F F 8 160-179 188 188 188 353 353 353 T T T 9 180-199 188 188 188 353 353 353 F F F 10 200-219 179 179 179 336 336 336 F F F 11 220-239 189 189 189 354 354 354 F F F 12 240-259 188 188 188 352 352 352 F F F 13 260-279 188 188 188 352 352 352 F F F 14 280-299 189 189 189 354 354 354 F F F 15 300-319 189 189 189 354 354 354 T T T 16 320-339 190 190 190 357 357 357 F F F 17 340-359 187 187 187 350 350 350 F F F 18 360-379 188 188 188 352 352 352 F F F 19 380-399 190 190 190 356 356 356 F F F 20 400-419 185 185 185 348 348 348 F F F 21 420-439 187 187 187 350 350 350 F F F 22 440-459 190 190 190 356 356 356 F F F 23 460-479 188 188 188 351 351 351 F F F 24 480-499 186 186 186 349 349 349 F F F 25 500-519 187 187 187 350 350 350 F F F 26 520-539 190 190 190 355 355 355 F F F 27 540-559 188 188 188 353 353 353 F F F 28 560-579 178 178 178 334 334 334 F F F 29 580-599 189 189 189 354 354 354 F F F Max gen: 190, Max flush: 357 A Difference in extent metadata is expected here On loop 95 Verifying data now No change after verify Loop: 95 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 191 191 191 358 358 358 F F F 1 020-039 191 191 191 358 358 358 F F F 2 040-059 188 188 188 352 352 352 F F F 3 060-079 180 180 180 337 337 337 F F F 4 080-099 185 185 185 348 348 348 F F F 5 100-119 184 184 184 347 347 347 F F F 6 120-139 189 189 189 354 354 354 F F F 7 140-159 175 175 175 327 327 327 F F F 8 160-179 191 191 191 358 358 358 F F F 9 180-199 188 188 188 353 353 353 F F F 10 200-219 179 179 179 336 336 336 F F F 11 220-239 189 189 189 354 354 354 F F F 12 240-259 188 188 188 352 352 352 F F F 13 260-279 188 188 188 352 352 352 F F F 14 280-299 189 189 189 354 354 354 F F F 15 300-319 191 191 191 358 358 358 F F F 16 320-339 190 190 190 357 357 357 F F F 17 340-359 187 187 187 350 350 350 F F F 18 360-379 188 188 188 352 352 352 F F F 19 380-399 190 190 190 356 356 356 F F F 20 400-419 185 185 185 348 348 348 F F F 21 420-439 187 187 187 350 350 350 F F F 22 440-459 190 190 190 356 356 356 F F F 23 460-479 188 188 188 351 351 351 F F F 24 480-499 186 186 186 349 349 349 F F F 25 500-519 187 187 187 350 350 350 F F F 26 520-539 190 190 190 355 355 355 F F F 27 540-559 188 188 188 353 353 353 F F F 28 560-579 178 178 178 334 334 334 F F F 29 580-599 189 189 189 354 354 354 F F F Max gen: 191, Max flush: 358 Begin loop 96 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:13.454135665Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1950,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:13.49872969Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1950,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:13.499321389Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1950} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"26e7d501b33f47379e53601720353c2983e7654b\",\n git_commit_timestamp: \"2024-12-12T20:16:40.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:13.499356976Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1950} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:13.499371245Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1950} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:13.499385454Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1950} {"msg":"Crucible bc1fc8ce-838a-4e5f-a9de-c313ed529fc7 has session id: b2751a59-1060-4ee5-801a-c7ed5d11fbe6","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:13.499775737Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1950,"session_id":"b2751a59-1060-4ee5-801a-c7ed5d11fbe6"} {"msg":"Upstairs opts: Upstairs UUID: bc1fc8ce-838a-4e5f-a9de-c313ed529fc7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:13.499819225Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1950,"session_id":"b2751a59-1060-4ee5-801a-c7ed5d11fbe6"} {"msg":"Crucible stats registered with UUID: bc1fc8ce-838a-4e5f-a9de-c313ed529fc7","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:13.499847283Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1950,"session_id":"b2751a59-1060-4ee5-801a-c7ed5d11fbe6"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:13.530474521Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1950,"task":"crutest"} {"msg":"The guest has requested activation with gen:192","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:13.530500939Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1950} {"msg":"bc1fc8ce-838a-4e5f-a9de-c313ed529fc7 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:13.530572644Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1950,"session_id":"b2751a59-1060-4ee5-801a-c7ed5d11fbe6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:13.530595252Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1950,"client":"0","":"downstairs","session_id":"b2751a59-1060-4ee5-801a-c7ed5d11fbe6"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:13.53062889Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1950,"client":"0","":"downstairs","session_id":"b2751a59-1060-4ee5-801a-c7ed5d11fbe6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:13.530661508Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1950,"client":"1","":"downstairs","session_id":"b2751a59-1060-4ee5-801a-c7ed5d11fbe6"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:13.530833846Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1950,"client":"1","":"downstairs","session_id":"b2751a59-1060-4ee5-801a-c7ed5d11fbe6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:13.53106724Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1950,"client":"2","":"downstairs","session_id":"b2751a59-1060-4ee5-801a-c7ed5d11fbe6"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:13.531105707Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1950,"client":"2","":"downstairs","session_id":"b2751a59-1060-4ee5-801a-c7ed5d11fbe6"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:13.531136035Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1950,"":"io task","client":"2","":"downstairs","session_id":"b2751a59-1060-4ee5-801a-c7ed5d11fbe6"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:13.531169893Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1950,"":"io task","client":"0","":"downstairs","session_id":"b2751a59-1060-4ee5-801a-c7ed5d11fbe6"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:13.531202281Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1950,"":"io task","client":"1","":"downstairs","session_id":"b2751a59-1060-4ee5-801a-c7ed5d11fbe6"} {"msg":"ds_connection connected from Ok(127.0.0.1:45734)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:13.531233998Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1950,"":"io task","client":"2","":"downstairs","session_id":"b2751a59-1060-4ee5-801a-c7ed5d11fbe6"} {"msg":"ds_connection connected from Ok(127.0.0.1:54788)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:13.531380838Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1950,"":"io task","client":"0","":"downstairs","session_id":"b2751a59-1060-4ee5-801a-c7ed5d11fbe6"} {"msg":"ds_connection connected from Ok(127.0.0.1:43394)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:13.531414506Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1950,"":"io task","client":"1","":"downstairs","session_id":"b2751a59-1060-4ee5-801a-c7ed5d11fbe6"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:13.531528298Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1950,"client":"2","":"downstairs","session_id":"b2751a59-1060-4ee5-801a-c7ed5d11fbe6"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:13.531688147Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1950,"client":"0","":"downstairs","session_id":"b2751a59-1060-4ee5-801a-c7ed5d11fbe6"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:13.531868384Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1950,"client":"1","":"downstairs","session_id":"b2751a59-1060-4ee5-801a-c7ed5d11fbe6"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:13.532263897Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1950,"client":"2","":"downstairs","session_id":"b2751a59-1060-4ee5-801a-c7ed5d11fbe6"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:13.532292405Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1950,"client":"0","":"downstairs","session_id":"b2751a59-1060-4ee5-801a-c7ed5d11fbe6"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:13.532427176Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1950,"client":"1","":"downstairs","session_id":"b2751a59-1060-4ee5-801a-c7ed5d11fbe6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:13.532830278Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1950,"session_id":"b2751a59-1060-4ee5-801a-c7ed5d11fbe6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:13.532929971Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1950,"session_id":"b2751a59-1060-4ee5-801a-c7ed5d11fbe6"} {"msg":"[0]R flush_numbers[0..12]: [358, 358, 352, 337, 348, 347, 354, 327, 358, 353, 336, 354]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:13.533003816Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1950,"":"downstairs","":"downstairs","session_id":"b2751a59-1060-4ee5-801a-c7ed5d11fbe6"} {"msg":"[0]R generation[0..12]: [191, 191, 188, 180, 185, 184, 189, 175, 191, 188, 179, 189]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:13.533024255Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1950,"":"downstairs","":"downstairs","session_id":"b2751a59-1060-4ee5-801a-c7ed5d11fbe6"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:13.533039454Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1950,"":"downstairs","":"downstairs","session_id":"b2751a59-1060-4ee5-801a-c7ed5d11fbe6"} {"msg":"[1]R flush_numbers[0..12]: [358, 358, 352, 337, 348, 347, 354, 327, 358, 353, 336, 354]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:13.533053953Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1950,"":"downstairs","":"downstairs","session_id":"b2751a59-1060-4ee5-801a-c7ed5d11fbe6"} {"msg":"[1]R generation[0..12]: [191, 191, 188, 180, 185, 184, 189, 175, 191, 188, 179, 189]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:13.533067782Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1950,"":"downstairs","":"downstairs","session_id":"b2751a59-1060-4ee5-801a-c7ed5d11fbe6"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:13.533084881Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1950,"":"downstairs","":"downstairs","session_id":"b2751a59-1060-4ee5-801a-c7ed5d11fbe6"} {"msg":"[2]R flush_numbers[0..12]: [358, 358, 352, 337, 348, 347, 354, 327, 358, 353, 336, 354]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:13.53309892Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1950,"":"downstairs","":"downstairs","session_id":"b2751a59-1060-4ee5-801a-c7ed5d11fbe6"} {"msg":"[2]R generation[0..12]: [191, 191, 188, 180, 185, 184, 189, 175, 191, 188, 179, 189]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:13.533112809Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1950,"":"downstairs","":"downstairs","session_id":"b2751a59-1060-4ee5-801a-c7ed5d11fbe6"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:13.533126748Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1950,"":"downstairs","":"downstairs","session_id":"b2751a59-1060-4ee5-801a-c7ed5d11fbe6"} {"msg":"Max found gen is 192","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:13.533139757Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1950,"":"downstairs","":"downstairs","session_id":"b2751a59-1060-4ee5-801a-c7ed5d11fbe6"} {"msg":"Generation requested: 192 >= found:192","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:13.533152966Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1950,"":"downstairs","":"downstairs","session_id":"b2751a59-1060-4ee5-801a-c7ed5d11fbe6"} {"msg":"Next flush: 359","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:13.533165835Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1950,"":"downstairs","":"downstairs","session_id":"b2751a59-1060-4ee5-801a-c7ed5d11fbe6"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-12-12T20:40:13.533178724Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1950,"":"downstairs","":"downstairs","session_id":"b2751a59-1060-4ee5-801a-c7ed5d11fbe6"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:13.533201113Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1950,"session_id":"b2751a59-1060-4ee5-801a-c7ed5d11fbe6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:13.533225791Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1950,"client":"0","":"downstairs","session_id":"b2751a59-1060-4ee5-801a-c7ed5d11fbe6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:13.533251089Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1950,"client":"1","":"downstairs","session_id":"b2751a59-1060-4ee5-801a-c7ed5d11fbe6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:13.533275898Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1950,"client":"2","":"downstairs","session_id":"b2751a59-1060-4ee5-801a-c7ed5d11fbe6"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:13.533299726Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1950,"session_id":"b2751a59-1060-4ee5-801a-c7ed5d11fbe6"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:13.533316775Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1950,"session_id":"b2751a59-1060-4ee5-801a-c7ed5d11fbe6"} {"msg":"bc1fc8ce-838a-4e5f-a9de-c313ed529fc7 is now active with session: b2751a59-1060-4ee5-801a-c7ed5d11fbe6","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:13.533340973Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1950,"session_id":"b2751a59-1060-4ee5-801a-c7ed5d11fbe6"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:13.533369451Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1950,"session_id":"b2751a59-1060-4ee5-801a-c7ed5d11fbe6"} {"msg":"The guest has finished waiting for activation with:192","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:13.53338433Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1950} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 23 len 4096 data: 18 02/30 Read block 553 len 20480 03/30 Read block 430 len 8192 04/30 Read block 268 len 8192 05/30 Read block 188 len 16384 06/30 Read block 509 len 8192 07/30 Read block 231 len 28672 08/30 Write block 252 len 24576 data: 12 15 18 19 15 11 09/30 Read block 471 len 12288 10/30 Read block 278 len 36864 11/30 Write block 376 len 12288 data: 10 11 14 12/30 Flush 13/30 Write block 360 len 8192 data: 14 14 14/30 Flush 15/30 Write block 286 len 4096 data: 13 16/30 Write block 10 len 24576 data: 17 18 16 13 14 15 17/30 Read block 355 len 8192 18/30 Flush 19/30 Write block 401 len 32768 data: 21 19 20 17 16 16 17 18 20/30 Read block 140 len 20480 21/30 Write block 244 len 40960 data: 13 12 13 11 11 13 11 12 13 16 22/30 Read block 283 len 20480 23/30 Read block 259 len 32768 24/30 Write block 285 len 8192 data: 13 14 25/30 Write block 174 len 16384 data: 11 14 14 15 26/30 Write block 201 len 20480 data: 11 13 16 17 18 27/30 Write block 448 len 20480 data: 17 21 20 22 20 28/30 Read block 495 len 36864 29/30 Write block 76 len 16384 data: 10 15 14 14 30/30 Write block 407 len 28672 data: 18 19 17 15 12 10 11 ---------------------------------------------------------------- Crucible gen:192 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1018 Acked Write 8 Done Done Done false 1019 Acked Read 5 Done Done Done false 1020 Acked Write 10 Done Done Done false 1021 Acked Read 5 Done Done Done false 1022 Acked Read 8 Done Done Done false 1023 Acked Write 2 Done Done Done false 1024 Acked Write 4 Done Done Done false 1025 Acked Write 5 Done Done Done false 1026 Acked Write 5 Done Done Done false 1027 Acked Read 9 Done Done Done false 1028 Acked Write 4 Done Done Done false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 11 11 11 33 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1017 1017 1017 Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:40:13.551880424Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1950} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 192 192 192 361 361 361 F F F 1 020-039 192 192 192 359 359 359 F F F 2 040-059 188 188 188 352 352 352 F F F 3 060-079 180 180 180 337 337 337 T T T 4 080-099 185 185 185 348 348 348 F F F 5 100-119 184 184 184 347 347 347 F F F 6 120-139 189 189 189 354 354 354 F F F 7 140-159 175 175 175 327 327 327 F F F 8 160-179 191 191 191 358 358 358 T T T 9 180-199 188 188 188 353 353 353 F F F 10 200-219 179 179 179 336 336 336 T T T 11 220-239 189 189 189 354 354 354 F F F 12 240-259 192 192 192 359 359 359 T T T 13 260-279 188 188 188 352 352 352 F F F 14 280-299 192 192 192 361 361 361 T T T 15 300-319 191 191 191 358 358 358 F F F 16 320-339 190 190 190 357 357 357 F F F 17 340-359 187 187 187 350 350 350 F F F 18 360-379 192 192 192 360 360 360 F F F 19 380-399 190 190 190 356 356 356 F F F 20 400-419 185 185 185 348 348 348 T T T 21 420-439 187 187 187 350 350 350 F F F 22 440-459 190 190 190 356 356 356 T T T 23 460-479 188 188 188 351 351 351 F F F 24 480-499 186 186 186 349 349 349 F F F 25 500-519 187 187 187 350 350 350 F F F 26 520-539 190 190 190 355 355 355 F F F 27 540-559 188 188 188 353 353 353 F F F 28 560-579 178 178 178 334 334 334 F F F 29 580-599 189 189 189 354 354 354 F F F Max gen: 192, Max flush: 361 A Difference in extent metadata is expected here On loop 96 Verifying data now No change after verify Loop: 96 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 192 192 192 361 361 361 F F F 1 020-039 192 192 192 359 359 359 F F F 2 040-059 188 188 188 352 352 352 F F F 3 060-079 193 193 193 362 362 362 F F F 4 080-099 185 185 185 348 348 348 F F F 5 100-119 184 184 184 347 347 347 F F F 6 120-139 189 189 189 354 354 354 F F F 7 140-159 175 175 175 327 327 327 F F F 8 160-179 193 193 193 362 362 362 F F F 9 180-199 188 188 188 353 353 353 F F F 10 200-219 193 193 193 362 362 362 F F F 11 220-239 189 189 189 354 354 354 F F F 12 240-259 193 193 193 362 362 362 F F F 13 260-279 188 188 188 352 352 352 F F F 14 280-299 193 193 193 362 362 362 F F F 15 300-319 191 191 191 358 358 358 F F F 16 320-339 190 190 190 357 357 357 F F F 17 340-359 187 187 187 350 350 350 F F F 18 360-379 192 192 192 360 360 360 F F F 19 380-399 190 190 190 356 356 356 F F F 20 400-419 193 193 193 362 362 362 F F F 21 420-439 187 187 187 350 350 350 F F F 22 440-459 193 193 193 362 362 362 F F F 23 460-479 188 188 188 351 351 351 F F F 24 480-499 186 186 186 349 349 349 F F F 25 500-519 187 187 187 350 350 350 F F F 26 520-539 190 190 190 355 355 355 F F F 27 540-559 188 188 188 353 353 353 F F F 28 560-579 178 178 178 334 334 334 F F F 29 580-599 189 189 189 354 354 354 F F F Max gen: 193, Max flush: 362 Begin loop 97 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:17.83206007Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1958,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:17.87614203Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1958,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:17.876803534Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1958} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"26e7d501b33f47379e53601720353c2983e7654b\",\n git_commit_timestamp: \"2024-12-12T20:16:40.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:17.876838452Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1958} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:17.876856521Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1958} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:17.876871549Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1958} {"msg":"Crucible 73f6527a-b827-4393-ad53-d5b973a6612e has session id: 45d05d00-723c-4202-aa3e-6fc7dfde93a7","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:17.877317039Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1958,"session_id":"45d05d00-723c-4202-aa3e-6fc7dfde93a7"} {"msg":"Upstairs opts: Upstairs UUID: 73f6527a-b827-4393-ad53-d5b973a6612e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:17.877363376Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1958,"session_id":"45d05d00-723c-4202-aa3e-6fc7dfde93a7"} {"msg":"Crucible stats registered with UUID: 73f6527a-b827-4393-ad53-d5b973a6612e","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:17.877391484Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1958,"session_id":"45d05d00-723c-4202-aa3e-6fc7dfde93a7"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:17.908468291Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1958,"task":"crutest"} {"msg":"The guest has requested activation with gen:194","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:17.908500879Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1958} {"msg":"73f6527a-b827-4393-ad53-d5b973a6612e active request set","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:17.908560875Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1958,"session_id":"45d05d00-723c-4202-aa3e-6fc7dfde93a7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:17.908589373Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1958,"client":"0","":"downstairs","session_id":"45d05d00-723c-4202-aa3e-6fc7dfde93a7"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:17.908614941Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1958,"client":"0","":"downstairs","session_id":"45d05d00-723c-4202-aa3e-6fc7dfde93a7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:17.90862944Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1958,"client":"1","":"downstairs","session_id":"45d05d00-723c-4202-aa3e-6fc7dfde93a7"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:17.908644059Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1958,"client":"1","":"downstairs","session_id":"45d05d00-723c-4202-aa3e-6fc7dfde93a7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:17.908657328Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1958,"client":"2","":"downstairs","session_id":"45d05d00-723c-4202-aa3e-6fc7dfde93a7"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:17.908679836Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1958,"client":"2","":"downstairs","session_id":"45d05d00-723c-4202-aa3e-6fc7dfde93a7"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:17.908710284Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1958,"":"io task","client":"2","":"downstairs","session_id":"45d05d00-723c-4202-aa3e-6fc7dfde93a7"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:17.908738632Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1958,"":"io task","client":"0","":"downstairs","session_id":"45d05d00-723c-4202-aa3e-6fc7dfde93a7"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:17.90876803Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1958,"":"io task","client":"1","":"downstairs","session_id":"45d05d00-723c-4202-aa3e-6fc7dfde93a7"} {"msg":"ds_connection connected from Ok(127.0.0.1:57006)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:17.908794608Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1958,"":"io task","client":"2","":"downstairs","session_id":"45d05d00-723c-4202-aa3e-6fc7dfde93a7"} {"msg":"ds_connection connected from Ok(127.0.0.1:34764)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:17.908821246Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1958,"":"io task","client":"1","":"downstairs","session_id":"45d05d00-723c-4202-aa3e-6fc7dfde93a7"} {"msg":"ds_connection connected from Ok(127.0.0.1:64571)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:17.909092458Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1958,"":"io task","client":"0","":"downstairs","session_id":"45d05d00-723c-4202-aa3e-6fc7dfde93a7"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:17.909595553Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1958,"client":"2","":"downstairs","session_id":"45d05d00-723c-4202-aa3e-6fc7dfde93a7"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:17.909623441Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1958,"client":"0","":"downstairs","session_id":"45d05d00-723c-4202-aa3e-6fc7dfde93a7"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:17.909744003Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1958,"client":"1","":"downstairs","session_id":"45d05d00-723c-4202-aa3e-6fc7dfde93a7"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:17.910275916Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1958,"client":"0","":"downstairs","session_id":"45d05d00-723c-4202-aa3e-6fc7dfde93a7"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:17.910441475Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1958,"client":"1","":"downstairs","session_id":"45d05d00-723c-4202-aa3e-6fc7dfde93a7"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:17.910464343Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1958,"client":"2","":"downstairs","session_id":"45d05d00-723c-4202-aa3e-6fc7dfde93a7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:17.910825508Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1958,"session_id":"45d05d00-723c-4202-aa3e-6fc7dfde93a7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:17.911028274Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1958,"session_id":"45d05d00-723c-4202-aa3e-6fc7dfde93a7"} {"msg":"[0]R flush_numbers[0..12]: [361, 359, 352, 362, 348, 347, 354, 327, 362, 353, 362, 354]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:17.911105519Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1958,"":"downstairs","":"downstairs","session_id":"45d05d00-723c-4202-aa3e-6fc7dfde93a7"} {"msg":"[0]R generation[0..12]: [192, 192, 188, 193, 185, 184, 189, 175, 193, 188, 193, 189]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:17.911138407Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1958,"":"downstairs","":"downstairs","session_id":"45d05d00-723c-4202-aa3e-6fc7dfde93a7"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:17.911154425Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1958,"":"downstairs","":"downstairs","session_id":"45d05d00-723c-4202-aa3e-6fc7dfde93a7"} {"msg":"[1]R flush_numbers[0..12]: [361, 359, 352, 362, 348, 347, 354, 327, 362, 353, 362, 354]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:17.911168665Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1958,"":"downstairs","":"downstairs","session_id":"45d05d00-723c-4202-aa3e-6fc7dfde93a7"} {"msg":"[1]R generation[0..12]: [192, 192, 188, 193, 185, 184, 189, 175, 193, 188, 193, 189]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:17.911182814Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1958,"":"downstairs","":"downstairs","session_id":"45d05d00-723c-4202-aa3e-6fc7dfde93a7"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:17.911197043Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1958,"":"downstairs","":"downstairs","session_id":"45d05d00-723c-4202-aa3e-6fc7dfde93a7"} {"msg":"[2]R flush_numbers[0..12]: [361, 359, 352, 362, 348, 347, 354, 327, 362, 353, 362, 354]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:17.911210772Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1958,"":"downstairs","":"downstairs","session_id":"45d05d00-723c-4202-aa3e-6fc7dfde93a7"} {"msg":"[2]R generation[0..12]: [192, 192, 188, 193, 185, 184, 189, 175, 193, 188, 193, 189]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:17.911224501Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1958,"":"downstairs","":"downstairs","session_id":"45d05d00-723c-4202-aa3e-6fc7dfde93a7"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:17.91123846Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1958,"":"downstairs","":"downstairs","session_id":"45d05d00-723c-4202-aa3e-6fc7dfde93a7"} {"msg":"Max found gen is 194","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:17.911251559Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1958,"":"downstairs","":"downstairs","session_id":"45d05d00-723c-4202-aa3e-6fc7dfde93a7"} {"msg":"Generation requested: 194 >= found:194","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:17.911264898Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1958,"":"downstairs","":"downstairs","session_id":"45d05d00-723c-4202-aa3e-6fc7dfde93a7"} {"msg":"Next flush: 363","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:17.911277757Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1958,"":"downstairs","":"downstairs","session_id":"45d05d00-723c-4202-aa3e-6fc7dfde93a7"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:17.911290786Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1958,"":"downstairs","":"downstairs","session_id":"45d05d00-723c-4202-aa3e-6fc7dfde93a7"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:17.911303745Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1958,"session_id":"45d05d00-723c-4202-aa3e-6fc7dfde93a7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:17.911315554Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1958,"Wait for a query_work_queue command to finish before sending IO client":"0","":"downstairs","session_id":"45d05d00-723c-4202-aa3e-6fc7dfde93a7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:17.911341363Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1958,"client":"1","":"downstairs","session_id":"45d05d00-723c-4202-aa3e-6fc7dfde93a7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:17.911354412Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1958,"client":"2","":"downstairs","session_id":"45d05d00-723c-4202-aa3e-6fc7dfde93a7"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:17.911367381Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1958,"session_id":"45d05d00-723c-4202-aa3e-6fc7dfde93a7"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:17.91137947Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1958,"session_id":"45d05d00-723c-4202-aa3e-6fc7dfde93a7"} {"msg":"73f6527a-b827-4393-ad53-d5b973a6612e is now active with session: 45d05d00-723c-4202-aa3e-6fc7dfde93a7","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:17.911392359Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1958,"session_id":"45d05d00-723c-4202-aa3e-6fc7dfde93a7"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:17.911404448Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1958,"session_id":"45d05d00-723c-4202-aa3e-6fc7dfde93a7"} {"msg":"The guest has finished waiting for activation with:194","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:17.911416447Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1958} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 132 len 36864 data: 9 8 9 9 11 13 16 9 10 02/30 Read block 284 len 28672 03/30 Write block 48 len 28672 data: 12 15 18 18 16 18 17 04/30 Read block 290 len 40960 05/30 Flush 06/30 Write block 558 len 8192 data: 14 16 07/30 Write block 461 len 4096 data: 21 08/30 Write block 145 len 12288 data: 15 14 13 09/30 Write block 103 len 28672 data: 11 11 10 10 13 11 11 10/30 Read block 349 len 32768 11/30 Read block 269 len 32768 12/30 Flush 13/30 Write block 87 len 20480 data: 15 18 16 16 13 14/30 Flush 15/30 Write block 487 len 20480 data: 16 17 21 18 14 16/30 Read block 101 len 8192 17/30 Flush 18/30 Write block 550 len 28672 data: 14 13 13 13 14 13 13 19/30 Read block 59 len 28672 20/30 Write block 72 len 28672 data: 11 11 10 12 11 16 15 21/30 Write block 81 len 12288 data: 13 10 11 22/30 Write block 263 len 4096 data: 18 23/30 Write block 104 len 8192 data: 12 11 24/30 Write block 289 len 12288 data: 11 11 14 25/30 Write block 179 len 8192 data: 16 13 26/30 Write block 347 len 12288 data: 18 16 15 27/30 Write block 98 len 36864 data: 9 12 12 10 11 12 13 12 11 28/30 Read block 262 len 32768 29/30 Read block 113 len 16384 30/30 Write block 101 len 8192 data: 11 12 ---------------------------------------------------------------- Crucible gen:194 GIO:true work queues: Upstairs:0 downstairs:13 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1017 Acked Write 7 Done Done Done false 1018 Acked Read 7 Done Done Done false 1019 Acked Write 7 Done Done Done false 1020 Acked Write 3 Done Done Done false 1021 Acked Write 1 Done Done Done false 1022 Acked Write 2 Done Done Done false 1023 Acked Write 3 Done Done Done false 1024 Acked Write 2 Done Done Done false 1025 Acked Write 3 Done Done Done false 1026 Acked Write 9 Done Done Done false 1027 Acked Read 8 Done Done Done false 1028 Acked Read 4 Done Done Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 2 4 Done 12 12 11 35 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1016 1016 1016 Downstairs last five completed: 1016 1015 1014 1013 1012 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:40:17.929022653Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1958} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 192 192 192 361 361 361 F F F 1 020-039 192 192 192 359 359 359 F F F 2 040-059 194 194 194 363 363 363 F F F 3 060-079 193 193 193 362 362 362 T T T 4 080-099 194 194 194 365 365 365 T T T 5 100-119 194 194 194 364 364 364 T T T 6 120-139 194 194 194 363 363 363 F F F 7 140-159 194 194 194 364 364 364 F F F 8 160-179 193 193 193 362 362 362 T T T 9 180-199 188 188 188 353 353 353 T T T 10 200-219 193 193 193 362 362 362 F F F 11 220-239 189 189 189 354 354 354 F F F 12 240-259 193 193 193 362 362 362 F F F 13 260-279 188 188 188 352 352 352 T T T 14 280-299 193 193 193 362 362 362 T T T 15 300-319 191 191 191 358 358 358 F F F 16 320-339 190 190 190 357 357 357 F F F 17 340-359 187 187 187 350 350 350 T T T 18 360-379 192 192 192 360 360 360 F F F 19 380-399 190 190 190 356 356 356 F F F 20 400-419 193 193 193 362 362 362 F F F 21 420-439 187 187 187 350 350 350 F F F 22 440-459 193 193 193 362 362 362 F F F 23 460-479 194 194 194 364 364 364 F F F 24 480-499 194 194 194 366 366 366 F F F 25 500-519 187 187 187 350 350 350 F F F 26 520-539 190 190 190 355 355 355 F F F 27 540-559 194 194 194 364 364 364 T T T 28 560-579 178 178 178 334 334 334 F F F 29 580-599 189 189 189 354 354 354 F F F Max gen: 194, Max flush: 366 A Difference in extent metadata is expected here On loop 97 Verifying data now No change after verify Loop: 97 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 192 192 192 361 361 361 F F F 1 020-039 192 192 192 359 359 359 F F F 2 040-059 194 194 194 363 363 363 F F F 3 060-079 195 195 195 367 367 367 F F F 4 080-099 195 195 195 367 367 367 F F F 5 100-119 195 195 195 367 367 367 F F F 6 120-139 194 194 194 363 363 363 F F F 7 140-159 194 194 194 364 364 364 F F F 8 160-179 195 195 195 367 367 367 F F F 9 180-199 195 195 195 367 367 367 F F F 10 200-219 193 193 193 362 362 362 F F F 11 220-239 189 189 189 354 354 354 F F F 12 240-259 193 193 193 362 362 362 F F F 13 260-279 195 195 195 367 367 367 F F F 14 280-299 195 195 195 367 367 367 F F F 15 300-319 191 191 191 358 358 358 F F F 16 320-339 190 190 190 357 357 357 F F F 17 340-359 195 195 195 367 367 367 F F F 18 360-379 192 192 192 360 360 360 F F F 19 380-399 190 190 190 356 356 356 F F F 20 400-419 193 193 193 362 362 362 F F F 21 420-439 187 187 187 350 350 350 F F F 22 440-459 193 193 193 362 362 362 F F F 23 460-479 194 194 194 364 364 364 F F F 24 480-499 194 194 194 366 366 366 F F F 25 500-519 187 187 187 350 350 350 F F F 26 520-539 190 190 190 355 355 355 F F F 27 540-559 195 195 195 367 367 367 F F F 28 560-579 178 178 178 334 334 334 F F F 29 580-599 189 189 189 354 354 354 F F F Max gen: 195, Max flush: 367 Begin loop 98 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:22.218631449Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1966,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:40:22.261736317Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1966,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:22.262139489Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1966,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:22.293931887Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1966,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:22.294497128Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1966} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"26e7d501b33f47379e53601720353c2983e7654b\",\n git_commit_timestamp: \"2024-12-12T20:16:40.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:22.294529645Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1966} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:22.294542915Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1966} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:22.294557774Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1966} {"msg":"Crucible 9babae8d-2e03-46d8-9250-965a237d73c2 has session id: d79bb895-1095-4599-8366-46c8b0085d2b","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:22.294995084Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1966,"session_id":"d79bb895-1095-4599-8366-46c8b0085d2b"} {"msg":"Upstairs opts: Upstairs UUID: 9babae8d-2e03-46d8-9250-965a237d73c2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:22.2950431Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1966,"session_id":"d79bb895-1095-4599-8366-46c8b0085d2b"} {"msg":"Crucible stats registered with UUID: 9babae8d-2e03-46d8-9250-965a237d73c2","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:22.295068159Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1966,"session_id":"d79bb895-1095-4599-8366-46c8b0085d2b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:22.326092169Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1966,"task":"crutest"} {"msg":"The guest has requested activation with gen:196","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:22.326122937Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1966} {"msg":"9babae8d-2e03-46d8-9250-965a237d73c2 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:22.326157005Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1966,"session_id":"d79bb895-1095-4599-8366-46c8b0085d2b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:22.326174763Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1966,"client":"0","":"downstairs","session_id":"d79bb895-1095-4599-8366-46c8b0085d2b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:22.326203452Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1966,"client":"0","":"downstairs","session_id":"d79bb895-1095-4599-8366-46c8b0085d2b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:22.32622091Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1966,"client":"1","":"downstairs","session_id":"d79bb895-1095-4599-8366-46c8b0085d2b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:22.326238659Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1966,"client":"1","":"downstairs","session_id":"d79bb895-1095-4599-8366-46c8b0085d2b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:22.326255008Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1966,"client":"2","":"downstairs","session_id":"d79bb895-1095-4599-8366-46c8b0085d2b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:22.326285596Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1966,"client":"2","":"downstairs","session_id":"d79bb895-1095-4599-8366-46c8b0085d2b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:22.326320213Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1966,"":"io task","client":"2","":"downstairs","session_id":"d79bb895-1095-4599-8366-46c8b0085d2b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:22.326350081Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1966,"":"io task","client":"0","":"downstairs","session_id":"d79bb895-1095-4599-8366-46c8b0085d2b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:22.326381019Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1966,"":"io task","client":"1","":"downstairs","session_id":"d79bb895-1095-4599-8366-46c8b0085d2b"} {"msg":"ds_connection connected from Ok(127.0.0.1:50435)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:22.326439275Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1966,"":"io task","client":"2","":"downstairs","session_id":"d79bb895-1095-4599-8366-46c8b0085d2b"} {"msg":"ds_connection connected from Ok(127.0.0.1:37017)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:22.326476573Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1966,"":"io task","client":"0","":"downstairs","session_id":"d79bb895-1095-4599-8366-46c8b0085d2b"} {"msg":"ds_connection connected from Ok(127.0.0.1:54734)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:22.326627452Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1966,"":"io task","client":"1","":"downstairs","session_id":"d79bb895-1095-4599-8366-46c8b0085d2b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:22.327121298Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1966,"client":"2","":"downstairs","session_id":"d79bb895-1095-4599-8366-46c8b0085d2b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:22.327150296Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1966,"client":"0","":"downstairs","session_id":"d79bb895-1095-4599-8366-46c8b0085d2b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:22.327179274Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1966,"client":"1","":"downstairs","session_id":"d79bb895-1095-4599-8366-46c8b0085d2b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:22.327788472Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1966,"client":"2","":"downstairs","session_id":"d79bb895-1095-4599-8366-46c8b0085d2b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:22.32781966Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1966,"client":"1","":"downstairs","session_id":"d79bb895-1095-4599-8366-46c8b0085d2b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:22.327846958Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1966,"client":"0","":"downstairs","session_id":"d79bb895-1095-4599-8366-46c8b0085d2b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:22.328386731Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1966,"session_id":"d79bb895-1095-4599-8366-46c8b0085d2b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:22.328418029Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1966,"session_id":"d79bb895-1095-4599-8366-46c8b0085d2b"} {"msg":"[0]R flush_numbers[0..12]: [361, 359, 363, 367, 367, 367, 363, 364, 367, 367, 362, 354]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:22.328496853Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1966,"":"downstairs","":"downstairs","session_id":"d79bb895-1095-4599-8366-46c8b0085d2b"} {"msg":"[0]R generation[0..12]: [192, 192, 194, 195, 195, 195, 194, 194, 195, 195, 193, 189]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:22.328516152Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1966,"":"downstairs","":"downstairs","session_id":"d79bb895-1095-4599-8366-46c8b0085d2b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:22.328530591Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1966,"":"downstairs","":"downstairs","session_id":"d79bb895-1095-4599-8366-46c8b0085d2b"} {"msg":"[1]R flush_numbers[0..12]: [361, 359, 363, 367, 367, 367, 363, 364, 367, 367, 362, 354]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:22.3285445Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1966,"":"downstairs","":"downstairs","session_id":"d79bb895-1095-4599-8366-46c8b0085d2b"} {"msg":"[1]R generation[0..12]: [192, 192, 194, 195, 195, 195, 194, 194, 195, 195, 193, 189]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:22.328558029Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1966,"":"downstairs","":"downstairs","session_id":"d79bb895-1095-4599-8366-46c8b0085d2b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:22.328571968Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1966,"":"downstairs","":"downstairs","session_id":"d79bb895-1095-4599-8366-46c8b0085d2b"} {"msg":"[2]R flush_numbers[0..12]: [361, 359, 363, 367, 367, 367, 363, 364, 367, 367, 362, 354]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:22.328585937Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1966,"":"downstairs","":"downstairs","session_id":"d79bb895-1095-4599-8366-46c8b0085d2b"} {"msg":"[2]R generation[0..12]: [192, 192, 194, 195, 195, 195, 194, 194, 195, 195, 193, 189]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:22.328603196Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1966,"":"downstairs","":"downstairs","session_id":"d79bb895-1095-4599-8366-46c8b0085d2b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:22.328617565Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1966,"":"downstairs","":"downstairs","session_id":"d79bb895-1095-4599-8366-46c8b0085d2b"} {"msg":"Max found gen is 196","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:22.328630434Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1966,"":"downstairs","":"downstairs","session_id":"d79bb895-1095-4599-8366-46c8b0085d2b"} {"msg":"Generation requested: 196 >= found:196","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:22.328643603Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1966,"":"downstairs","":"downstairs","session_id":"d79bb895-1095-4599-8366-46c8b0085d2b"} {"msg":"Next flush: 368","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:22.328656392Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1966,"":"downstairs","":"downstairs","session_id":"d79bb895-1095-4599-8366-46c8b0085d2b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:22.328669362Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1966,"":"downstairs","":"downstairs","session_id":"d79bb895-1095-4599-8366-46c8b0085d2b"} {"msg":"No downstairs reconciliation requiredWait for a query_work_queue command to finish before sending IO ","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:22.32868742Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1966,"session_id":"d79bb895-1095-4599-8366-46c8b0085d2b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:22.328699639Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1966,"client":"0","":"downstairs","session_id":"d79bb895-1095-4599-8366-46c8b0085d2b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:22.328712779Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1966,"client":"1","":"downstairs","session_id":"d79bb895-1095-4599-8366-46c8b0085d2b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:22.328725798Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1966,"client":"2","":"downstairs","session_id":"d79bb895-1095-4599-8366-46c8b0085d2b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:22.328738847Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1966,"session_id":"d79bb895-1095-4599-8366-46c8b0085d2b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:22.328750916Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1966,"session_id":"d79bb895-1095-4599-8366-46c8b0085d2b"} {"msg":"9babae8d-2e03-46d8-9250-965a237d73c2 is now active with session: d79bb895-1095-4599-8366-46c8b0085d2b","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:22.328763425Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1966,"session_id":"d79bb895-1095-4599-8366-46c8b0085d2b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:22.328775164Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1966,"session_id":"d79bb895-1095-4599-8366-46c8b0085d2b"} {"msg":"The guest has finished waiting for activation with:196","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:22.328787373Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1966} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 578 len 40960 data: 12 14 12 14 16 18 20 21 22 22 02/30 Flush 03/30 Flush 04/30 Write block 445 len 40960 data: 11 14 17 18 22 21 23 21 17 20 05/30 Read block 261 len 36864 06/30 Read block 81 len 12288 07/30 Write block 150 len 40960 data: 13 13 15 15 14 12 12 12 10 6 08/30 Read block 496 len 4096 09/30 Write block 78 len 24576 data: 16 15 15 14 11 12 10/30 Write block 589 len 12288 data: 20 23 22 11/30 Write block 579 len 16384 data: 15 13 15 17 12/30 Write block 423 len 36864 data: 14 15 12 11 14 16 18 15 17 13/30 Read block 531 len 20480 14/30 Write block 441 len 40960 data: 11 11 11 11 12 15 18 19 23 22 15/30 Write block 45 len 32768 data: 17 15 13 13 16 19 19 17 16/30 Read block 498 len 12288 17/30 Flush 18/30 Write block 536 len 4096 data: 12 19/30 Read block 448 len 16384 20/30 Write block 397 len 20480 data: 17 18 19 18 22 21/30 Read block 60 len 32768 22/30 Write block 442 len 28672 data: 12 12 12 13 16 19 20 23/30 Flush 24/30 Write block 353 len 36864 data: 14 17 17 17 16 14 14 15 15 25/30 Write block 293 len 24576 data: 14 13 12 11 13 11 26/30 Read block 244 len 36864 27/30 Read block 311 len 32768 28/30 Read block 231 len 4096 29/30 Write block 399 len 32768 data: 20 19 23 20 21 18 17 17 30/30 Write block 461 len 16384 data: 22 24 22 22 ---------------------------------------------------------------- Crucible gen:196 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 9 Done Done Done false 1024 Acked Write 6 Done Done Done false 1025 Acked Read 9 Done Done Done false 1026 Acked Read 8 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Write 8 Sent Sent Sent false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:40:22.349025118Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1966} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 192 192 192 361 361 361 F F F 1 020-039 192 192 192 359 359 359 F F F 2 040-059 196 196 196 370 370 370 F F F 3 060-079 196 196 196 370 370 370 F F F 4 080-099 196 196 196 370 370 370 F F F 5 100-119 195 195 195 367 367 367 F F F 6 120-139 194 194 194 363 363 363 F F F 7 140-159 196 196 196 370 370 370 F F F 8 160-179 195 195 195 367 367 367 F F F 9 180-199 195 195 195 367 367 367 F F F 10 200-219 193 193 193 362 362 362 F F F 11 220-239 189 189 189 354 354 354 F F F 12 240-259 193 193 193 362 362 362 F F F 13 260-279 195 195 195 367 367 367 F F F 14 280-299 195 195 195 367 367 367 T T T 15 300-319 191 191 191 358 358 358 F F F 16 320-339 190 190 190 357 357 357 F F F 17 340-359 195 195 195 367 367 367 T T T 18 360-379 192 192 192 360 360 360 T T T 19 380-399 196 196 196 371 371 371 T T T 20 400-419 196 196 196 371 371 371 T T T 21 420-439 196 196 196 370 370 370 F F F 22 440-459 196 196 196 371 371 371 F F F 23 460-479 194 194 194 364 364 364 T T T 24 480-499 194 194 194 366 366 366 F F F 25 500-519 187 187 187 350 350 350 F F F 26 520-539 196 196 196 371 371 371 F F F 27 540-559 195 195 195 367 367 367 F F F 28 560-579 196 196 196 370 370 370 F F F 29 580-599 196 196 196 370 370 370 F F F Max gen: 196, Max flush: 371 A Difference in extent metadata is expected here On loop 98 Verifying data now No change after verify Loop: 98 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 192 192 192 361 361 361 F F F 1 020-039 192 192 192 359 359 359 F F F 2 040-059 196 196 196 370 370 370 F F F 3 060-079 196 196 196 370 370 370 F F F 4 080-099 196 196 196 370 370 370 F F F 5 100-119 195 195 195 367 367 367 F F F 6 120-139 194 194 194 363 363 363 F F F 7 140-159 196 196 196 370 370 370 F F F 8 160-179 195 195 195 367 367 367 F F F 9 180-199 195 195 195 367 367 367 F F F 10 200-219 193 193 193 362 362 362 F F F 11 220-239 189 189 189 354 354 354 F F F 12 240-259 193 193 193 362 362 362 F F F 13 260-279 195 195 195 367 367 367 F F F 14 280-299 197 197 197 372 372 372 F F F 15 300-319 191 191 191 358 358 358 F F F 16 320-339 190 190 190 357 357 357 F F F 17 340-359 197 197 197 372 372 372 F F F 18 360-379 197 197 197 372 372 372 F F F 19 380-399 197 197 197 372 372 372 F F F 20 400-419 197 197 197 372 372 372 F F F 21 420-439 196 196 196 370 370 370 F F F 22 440-459 196 196 196 371 371 371 F F F 23 460-479 197 197 197 372 372 372 F F F 24 480-499 194 194 194 366 366 366 F F F 25 500-519 187 187 187 350 350 350 F F F 26 520-539 196 196 196 371 371 371 F F F 27 540-559 195 195 195 367 367 367 F F F 28 560-579 196 196 196 370 370 370 F F F 29 580-599 196 196 196 370 370 370 F F F Max gen: 197, Max flush: 372 Begin loop 99 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:26.657705378Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1974,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:26.702223619Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1974,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:26.70293184Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1974} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"26e7d501b33f47379e53601720353c2983e7654b\",\n git_commit_timestamp: \"2024-12-12T20:16:40.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:26.702966617Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1974} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:26.702980656Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1974} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:26.702994825Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1974} {"msg":"Crucible 8e364fa7-7d95-4955-94b4-da81d043e2c6 has session id: c8cb1783-b997-4a68-92fd-113b4935f4f8","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:26.703399047Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1974,"session_id":"c8cb1783-b997-4a68-92fd-113b4935f4f8"} {"msg":"Upstairs opts: Upstairs UUID: 8e364fa7-7d95-4955-94b4-da81d043e2c6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:26.703445434Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1974,"session_id":"c8cb1783-b997-4a68-92fd-113b4935f4f8"} {"msg":"Crucible stats registered with UUID: 8e364fa7-7d95-4955-94b4-da81d043e2c6","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:26.703470482Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1974,"session_id":"c8cb1783-b997-4a68-92fd-113b4935f4f8"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:26.734316766Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1974,"task":"crutest"} {"msg":"The guest has requested activation with gen:198","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:26.734352673Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1974} {"msg":"8e364fa7-7d95-4955-94b4-da81d043e2c6 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:26.734409249Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1974,"session_id":"c8cb1783-b997-4a68-92fd-113b4935f4f8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:26.734437377Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1974,"client":"0","":"downstairs","session_id":"c8cb1783-b997-4a68-92fd-113b4935f4f8"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:26.734461925Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1974,"client":"0","":"downstairs","session_id":"c8cb1783-b997-4a68-92fd-113b4935f4f8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:26.734476624Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1974,"client":"1","":"downstairs","session_id":"c8cb1783-b997-4a68-92fd-113b4935f4f8"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:26.734491214Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1974,"client":"1","":"downstairs","session_id":"c8cb1783-b997-4a68-92fd-113b4935f4f8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:26.734505073Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1974,"client":"2","":"downstairs","session_id":"c8cb1783-b997-4a68-92fd-113b4935f4f8"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:26.734519441Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1974,"client":"2","":"downstairs","session_id":"c8cb1783-b997-4a68-92fd-113b4935f4f8"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:26.734532831Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1974,"":"io task","client":"2","":"downstairs","session_id":"c8cb1783-b997-4a68-92fd-113b4935f4f8"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:26.73454682Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1974,"":"io task","client":"0","":"downstairs","session_id":"c8cb1783-b997-4a68-92fd-113b4935f4f8"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:26.734560339Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1974,"":"io task","client":"1","":"downstairs","session_id":"c8cb1783-b997-4a68-92fd-113b4935f4f8"} {"msg":"ds_connection connected from Ok(127.0.0.1:58442)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:26.734581127Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1974,"":"io task","client":"2","":"downstairs","session_id":"c8cb1783-b997-4a68-92fd-113b4935f4f8"} {"msg":"ds_connection connected from Ok(127.0.0.1:55701)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:26.7346887Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1974,"":"io task","client":"1","":"downstairs","session_id":"c8cb1783-b997-4a68-92fd-113b4935f4f8"} {"msg":"ds_connection connected from Ok(127.0.0.1:36984)","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:26.734764975Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1974,"":"io task","client":"0","":"downstairs","session_id":"c8cb1783-b997-4a68-92fd-113b4935f4f8"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:26.7354048Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1974,"client":"0","":"downstairs","session_id":"c8cb1783-b997-4a68-92fd-113b4935f4f8"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:26.735439488Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1974,"client":"1","":"downstairs","session_id":"c8cb1783-b997-4a68-92fd-113b4935f4f8"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:26.735459317Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1974,"client":"2","":"downstairs","session_id":"c8cb1783-b997-4a68-92fd-113b4935f4f8"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:26.736039977Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1974,"client":"0","":"downstairs","session_id":"c8cb1783-b997-4a68-92fd-113b4935f4f8"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:26.73613639Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1974,"client":"2","":"downstairs","session_id":"c8cb1783-b997-4a68-92fd-113b4935f4f8"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:26.736156969Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1974,"client":"1","":"downstairs","session_id":"c8cb1783-b997-4a68-92fd-113b4935f4f8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:26.736611747Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1974,"session_id":"c8cb1783-b997-4a68-92fd-113b4935f4f8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:26.736726269Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1974,"session_id":"c8cb1783-b997-4a68-92fd-113b4935f4f8"} {"msg":"[0]R flush_numbers[0..12]: [361, 359, 370, 370, 370, 367, 363, 370, 367, 367, 362, 354]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:26.736829502Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1974,"":"downstairs","":"downstairs","session_id":"c8cb1783-b997-4a68-92fd-113b4935f4f8"} {"msg":"[0]R generation[0..12]: [192, 192, 196, 196, 196, 195, 194, 196, 195, 195, 193, 189]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:26.73685935Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1974,"":"downstairs","":"downstairs","session_id":"c8cb1783-b997-4a68-92fd-113b4935f4f8"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:26.736875939Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1974,"":"downstairs","":"downstairs","session_id":"c8cb1783-b997-4a68-92fd-113b4935f4f8"} {"msg":"[1]R flush_numbers[0..12]: [361, 359, 370, 370, 370, 367, 363, 370, 367, 367, 362, 354]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:26.736893948Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1974,"":"downstairs","":"downstairs","session_id":"c8cb1783-b997-4a68-92fd-113b4935f4f8"} {"msg":"[1]R generation[0..12]: [192, 192, 196, 196, 196, 195, 194, 196, 195, 195, 193, 189]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:26.736923196Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1974,"":"downstairs","":"downstairs","session_id":"c8cb1783-b997-4a68-92fd-113b4935f4f8"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:26.736937995Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1974,"":"downstairs","":"downstairs","session_id":"c8cb1783-b997-4a68-92fd-113b4935f4f8"} {"msg":"[2]R flush_numbers[0..12]: [361, 359, 370, 370, 370, 367, 363, 370, 367, 367, 362, 354]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:26.736951784Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1974,"":"downstairs","":"downstairs","session_id":"c8cb1783-b997-4a68-92fd-113b4935f4f8"} {"msg":"[2]R generation[0..12]: [192, 192, 196, 196, 196, 195, 194, 196, 195, 195, 193, 189]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:26.736965633Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1974,"":"downstairs","":"downstairs","session_id":"c8cb1783-b997-4a68-92fd-113b4935f4f8"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:26.736979722Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1974,"":"downstairs","":"downstairs","session_id":"c8cb1783-b997-4a68-92fd-113b4935f4f8"} {"msg":"Max found gen is 198","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:26.73700845Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1974,"":"downstairs","":"downstairs","session_id":"c8cb1783-b997-4a68-92fd-113b4935f4f8"} {"msg":"Generation requested: 198 >= found:198","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-12-12T20:40:26.737023559Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1974,"":"downstairs","":"downstairs","session_id":"c8cb1783-b997-4a68-92fd-113b4935f4f8"} {"msg":"Next flush: 373","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:26.737038938Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1974,"":"downstairs","":"downstairs","session_id":"c8cb1783-b997-4a68-92fd-113b4935f4f8"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:26.737052487Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1974,"":"downstairs","":"downstairs","session_id":"c8cb1783-b997-4a68-92fd-113b4935f4f8"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:26.737065896Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1974,"session_id":"c8cb1783-b997-4a68-92fd-113b4935f4f8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:26.737077795Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1974,"client":"0","":"downstairs","session_id":"c8cb1783-b997-4a68-92fd-113b4935f4f8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:26.737091074Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1974,"client":"1","":"downstairs","session_id":"c8cb1783-b997-4a68-92fd-113b4935f4f8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:26.737104053Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1974,"client":"2","":"downstairs","session_id":"c8cb1783-b997-4a68-92fd-113b4935f4f8"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:26.737117262Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1974,"session_id":"c8cb1783-b997-4a68-92fd-113b4935f4f8"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:26.737132581Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1974,"session_id":"c8cb1783-b997-4a68-92fd-113b4935f4f8"} {"msg":"8e364fa7-7d95-4955-94b4-da81d043e2c6 is now active with session: c8cb1783-b997-4a68-92fd-113b4935f4f8","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:26.73714562Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1974,"session_id":"c8cb1783-b997-4a68-92fd-113b4935f4f8"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:26.7371576Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1974,"session_id":"c8cb1783-b997-4a68-92fd-113b4935f4f8"} {"msg":"The guest has finished waiting for activation with:198","v":0,"name":"crucible","level":30,"time":"2024-12-12T20:40:26.737170019Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1974} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 588 len 20480 data: 20 21 24 23 20 02/30 Write block 33 len 40960 data: 15 17 19 17 17 18 19 17 16 14 03/30 Read block 209 len 40960 04/30 Read block 517 len 16384 05/30 Write block 139 len 32768 data: 10 11 11 15 16 15 16 15 06/30 Flush 07/30 Write block 115 len 8192 data: 14 13 08/30 Read block 165 len 4096 09/30 Read block 165 len 24576 10/30 Read block 399 len 8192 11/30 Read block 250 len 16384 12/30 Write block 504 len 8192 data: 16 15 13/30 Write block 128 len 40960 data: 13 15 15 11 10 9 10 10 12 14 14/30 Read block 254 len 32768 15/30 Read block 109 len 20480 16/30 Flush 17/30 Write block 149 len 20480 data: 14 14 14 16 16 18/30 Flush 19/30 Write block 542 len 8192 data: 11 12 20/30 Read block 587 len 32768 21/30 Read block 553 len 4096 22/30 Read block 585 len 20480 23/30 Write block 567 len 4096 data: 10 24/30 Read block 280 len 36864 25/30 Read block 279 len 4096 26/30 Write block 387 len 36864 data: 18 16 12 15 14 11 11 14 15 27/30 Write block 401 len 20480 data: 24 21 22 19 18 28/30 Read block 142 len 8192 29/30 Read block 403 len 20480 30/30 Write block 357 len 40960 data: 17 15 15 16 16 13 14 18 19 21 ---------------------------------------------------------------- Crucible gen:198 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1018 Acked Write 2 Done Done Done false 1019 Acked Read 8 Done Done Done false 1020 Acked Read 1 Done Done Done false 1021 Acked Read 5 Done Done Done false 1022 Acked Write 1 Done Done Done false 1023 Acked Read 9 Done Done Done false 1024 Acked Read 1 Done Done Done false 1025 Acked Write 9 Done Done Done false 1026 Acked Write 5 Done Done Done false 1027 Acked Read 2 Done Done Done false 1028 Acked Read 5 Done Done Done false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 11 11 11 33 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1017 1017 1017 Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-12T20:40:26.756335317Z","hostname":"w-01JEY94CKQJCK3GEXVRRGZA3GF","pid":1974} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 192 192 192 361 361 361 F F F 1 020-039 198 198 198 373 373 373 F F F 2 040-059 198 198 198 373 373 373 F F F 3 060-079 196 196 196 370 370 370 F F F 4 080-099 196 196 196 370 370 370 F F F 5 100-119 198 198 198 374 374 374 F F F 6 120-139 198 198 198 374 374 374 F F F 7 140-159 198 198 198 375 375 375 F F F 8 160-179 195 195 195 367 367 367 F F F 9 180-199 195 195 195 367 367 367 F F F 10 200-219 193 193 193 362 362 362 F F F 11 220-239 189 189 189 354 354 354 F F F 12 240-259 193 193 193 362 362 362 F F F 13 260-279 195 195 195 367 367 367 F F F 14 280-299 197 197 197 372 372 372 F F F 15 300-319 191 191 191 358 358 358 F F F 16 320-339 190 190 190 357 357 357 F F F 17 340-359 197 197 197 372 372 372 T T T 18 360-379 197 197 197 372 372 372 T T T 19 380-399 197 197 197 372 372 372 T T T 20 400-419 197 197 197 372 372 372 T T T 21 420-439 196 196 196 370 370 370 F F F 22 440-459 196 196 196 371 371 371 F F F 23 460-479 197 197 197 372 372 372 F F F 24 480-499 194 194 194 366 366 366 F F F 25 500-519 198 198 198 374 374 374 F F F 26 520-539 196 196 196 371 371 371 F F F 27 540-559 195 195 195 367 367 367 T T T 28 560-579 196 196 196 370 370 370 T T T 29 580-599 198 198 198 373 373 373 F F F Max gen: 198, Max flush: 375 A Difference in extent metadata is expected here On loop 99 Verifying data now No change after verify Loop: 99 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 192 192 192 361 361 361 F F F 1 020-039 198 198 198 373 373 373 F F F 2 040-059 198 198 198 373 373 373 F F F 3 060-079 196 196 196 370 370 370 F F F 4 080-099 196 196 196 370 370 370 F F F 5 100-119 198 198 198 374 374 374 F F F 6 120-139 198 198 198 374 374 374 F F F 7 140-159 198 198 198 375 375 375 F F F 8 160-179 195 195 195 367 367 367 F F F 9 180-199 195 195 195 367 367 367 F F F 10 200-219 193 193 193 362 362 362 F F F 11 220-239 189 189 189 354 354 354 F F F 12 240-259 193 193 193 362 362 362 F F F 13 260-279 195 195 195 367 367 367 F F F 14 280-299 197 197 197 372 372 372 F F F 15 300-319 191 191 191 358 358 358 F F F 16 320-339 190 190 190 357 357 357 F F F 17 340-359 199 199 199 376 376 376 F F F 18 360-379 199 199 199 376 376 376 F F F 19 380-399 199 199 199 376 376 376 F F F 20 400-419 199 199 199 376 376 376 F F F 21 420-439 196 196 196 370 370 370 F F F 22 440-459 196 196 196 371 371 371 F F F 23 460-479 197 197 197 372 372 372 F F F 24 480-499 194 194 194 366 366 366 F F F 25 500-519 198 198 198 374 374 374 F F F 26 520-539 196 196 196 371 371 371 F F F 27 540-559 199 199 199 376 376 376 F F F 28 560-579 199 199 199 376 376 376 F F F 29 580-599 198 198 198 373 373 373 F F F Max gen: 199, Max flush: 376 7:20 Test duration Test completed real 7:19.854515411 user 54.387186743 sys 17.826713538 trap 0.070226087 tflt 0.039260864 dflt 0.113912728 kflt 0.005977781 lock 11:45:23.134131195 slp 35:38.811090171 lat 5.094590720 stop 0.058550263