Turn off color for downstairs dump Creating region directory at: /var/tmp/test_repair Creating dsc directory at: /tmp/dsc Downstairs region 0 created at /var/tmp/test_repair/8810 in 0.30205676 Downstairs region 1 created at /var/tmp/test_repair/8820 in 0.053710997 Downstairs region 2 created at /var/tmp/test_repair/8830 in 0.06260255 Created 3 regions /var/tmp/bins/crutest with -t 127.0.0.1:8810 -t 127.0.0.1:8820 -t 127.0.0.1:8830 1182 1183 1184 Dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:21.233116057Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1186,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:21.310842598Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1186,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:21.311927075Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1186} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"0fd98d0d5d909611f8b408ccda2559d35fe117bb\",\n git_commit_timestamp: \"2024-11-08T20:05:44.000000000Z\",\n git_branch: \"mkeeter/drop-skipped-backpressure-guard\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:21.31203727Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1186} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:21.312064239Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1186} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:21.312084098Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1186} {"msg":"Crucible 4e2f51a2-03a9-4897-8b88-f79c81b8c9d9 has session id: a44fff11-dad4-41ce-a228-aa63db42c404","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:21.312943956Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1186,"session_id":"a44fff11-dad4-41ce-a228-aa63db42c404"} {"msg":"Upstairs opts: Upstairs UUID: 4e2f51a2-03a9-4897-8b88-f79c81b8c9d9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:21.313029041Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1186,"session_id":"a44fff11-dad4-41ce-a228-aa63db42c404"} {"msg":"Crucible stats registered with UUID: 4e2f51a2-03a9-4897-8b88-f79c81b8c9d9","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:21.31305513Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1186,"session_id":"a44fff11-dad4-41ce-a228-aa63db42c404"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:21.353630557Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1186,"task":"crutest"} {"msg":"The guest has requested activation with gen:1","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:21.353698653Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1186} {"msg":"4e2f51a2-03a9-4897-8b88-f79c81b8c9d9 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:21.353729312Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1186,"session_id":"a44fff11-dad4-41ce-a228-aa63db42c404"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:21.35376669Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1186,"client":"0","":"downstairs","session_id":"a44fff11-dad4-41ce-a228-aa63db42c404"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:21.353874215Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1186,"client":"0","":"downstairs","session_id":"a44fff11-dad4-41ce-a228-aa63db42c404"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:21.353905063Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1186,"client":"1","":"downstairs","session_id":"a44fff11-dad4-41ce-a228-aa63db42c404"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:21.353930962Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1186,"client":"1","":"downstairs","session_id":"a44fff11-dad4-41ce-a228-aa63db42c404"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:21.353956201Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1186,"client":"2","":"downstairs","session_id":"a44fff11-dad4-41ce-a228-aa63db42c404"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:21.35398084Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1186,"client":"2","":"downstairs","session_id":"a44fff11-dad4-41ce-a228-aa63db42c404"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:21.354006518Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1186,"":"io task","client":"2","":"downstairs","session_id":"a44fff11-dad4-41ce-a228-aa63db42c404"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:21.354032697Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1186,"":"io task","client":"0","":"downstairs","session_id":"a44fff11-dad4-41ce-a228-aa63db42c404"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:21.354061356Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1186,"":"io task","client":"1","":"downstairs","session_id":"a44fff11-dad4-41ce-a228-aa63db42c404"} {"msg":"ds_connection connected from Ok(127.0.0.1:65507)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:21.354168291Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1186,"":"io task","client":"0","":"downstairs","session_id":"a44fff11-dad4-41ce-a228-aa63db42c404"} {"msg":"ds_connection connected from Ok(127.0.0.1:37466)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:21.354212539Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1186,"":"io task","client":"2","":"downstairs","session_id":"a44fff11-dad4-41ce-a228-aa63db42c404"} {"msg":"ds_connection connected from Ok(127.0.0.1:46579)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:21.354240057Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1186,"":"io task","client":"1","":"downstairs","session_id":"a44fff11-dad4-41ce-a228-aa63db42c404"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:21.356274928Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1186,"client":"1","":"downstairs","session_id":"a44fff11-dad4-41ce-a228-aa63db42c404"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:21.356315966Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1186,"client":"1","":"downstairs","session_id":"a44fff11-dad4-41ce-a228-aa63db42c404"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:21.356334615Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1186,"client":"2","":"downstairs","session_id":"a44fff11-dad4-41ce-a228-aa63db42c404"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:21.356349884Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1186,"client":"2","":"downstairs","session_id":"a44fff11-dad4-41ce-a228-aa63db42c404"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:21.356379573Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1186,"client":"0","":"downstairs","session_id":"a44fff11-dad4-41ce-a228-aa63db42c404"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:21.356409181Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1186,"client":"0","":"downstairs","session_id":"a44fff11-dad4-41ce-a228-aa63db42c404"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:21.357581934Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1186,"client":"0","":"downstairs","session_id":"a44fff11-dad4-41ce-a228-aa63db42c404"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:21.357621202Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1186,"client":"1","":"downstairs","session_id":"a44fff11-dad4-41ce-a228-aa63db42c404"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:21.35764936Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1186,"client":"2","":"downstairs","session_id":"a44fff11-dad4-41ce-a228-aa63db42c404"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:21.358545247Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1186,"client":"1","":"downstairs","session_id":"a44fff11-dad4-41ce-a228-aa63db42c404"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:21.358586305Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1186,"client":"1","":"downstairs","session_id":"a44fff11-dad4-41ce-a228-aa63db42c404"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:21.358628273Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1186,"session_id":"a44fff11-dad4-41ce-a228-aa63db42c404"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:21.358652771Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1186,"client":"2","":"downstairs","session_id":"a44fff11-dad4-41ce-a228-aa63db42c404"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:21.35867809Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1186,"client":"2","":"downstairs","session_id":"a44fff11-dad4-41ce-a228-aa63db42c404"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:21.358701329Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1186,"session_id":"a44fff11-dad4-41ce-a228-aa63db42c404"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:21.358722938Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1186,"client":"0","":"downstairs","session_id":"a44fff11-dad4-41ce-a228-aa63db42c404"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:21.358747047Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1186,"client":"0","":"downstairs","session_id":"a44fff11-dad4-41ce-a228-aa63db42c404"} {"msg":"[0]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:21.358860251Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1186,"":"downstairs","":"downstairs","session_id":"a44fff11-dad4-41ce-a228-aa63db42c404"} {"msg":"[0]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:21.35889089Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1186,"":"downstairs","":"downstairs","session_id":"a44fff11-dad4-41ce-a228-aa63db42c404"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:21.358911849Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1186,"":"downstairs","":"downstairs","session_id":"a44fff11-dad4-41ce-a228-aa63db42c404"} {"msg":"[1]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:21.358937317Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1186,"":"downstairs","":"downstairs","session_id":"a44fff11-dad4-41ce-a228-aa63db42c404"} {"msg":"[1]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:21.358952217Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1186,"":"downstairs","":"downstairs","session_id":"a44fff11-dad4-41ce-a228-aa63db42c404"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:21.358966696Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1186,"":"downstairs","":"downstairs","session_id":"a44fff11-dad4-41ce-a228-aa63db42c404"} {"msg":"[2]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:21.358980495Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1186,"":"downstairs","":"downstairs","session_id":"a44fff11-dad4-41ce-a228-aa63db42c404"} {"msg":"[2]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:21.358993905Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1186,"":"downstairs","":"downstairs","session_id":"a44fff11-dad4-41ce-a228-aa63db42c404"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:21.359007904Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1186,"":"downstairs","":"downstairs","session_id":"a44fff11-dad4-41ce-a228-aa63db42c404"} {"msg":"Max found gen is 1","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:21.359021173Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1186,"":"downstairs","":"downstairs","session_id":"a44fff11-dad4-41ce-a228-aa63db42c404"} {"msg":"Generation requested: 1 >= found:1","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:21.359034743Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1186,"":"downstairs","":"downstairs","session_id":"a44fff11-dad4-41ce-a228-aa63db42c404"} {"msg":"Next flush: 1","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:21.359047952Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1186,"":"downstairs","":"downstairs","session_id":"a44fff11-dad4-41ce-a228-aa63db42c404"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:21.359175086Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1186,"":"downstairs","":"downstairs","session_id":"a44fff11-dad4-41ce-a228-aa63db42c404"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:21.359195095Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1186,"session_id":"a44fff11-dad4-41ce-a228-aa63db42c404"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:21.359207514Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1186,"client":"0","":"downstairs","session_id":"a44fff11-dad4-41ce-a228-aa63db42c404"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:21.359220974Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1186,"client":"1","":"downstairs","session_id":"a44fff11-dad4-41ce-a228-aa63db42c404"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:21.359234133Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1186,"client":"2","":"downstairs","session_id":"a44fff11-dad4-41ce-a228-aa63db42c404"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:21.359250402Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1186,"session_id":"a44fff11-dad4-41ce-a228-aa63db42c404"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:21.359263072Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1186,"session_id":"a44fff11-dad4-41ce-a228-aa63db42c404"} {"msg":"4e2f51a2-03a9-4897-8b88-f79c81b8c9d9 is now active with session: a44fff11-dad4-41ce-a228-aa63db42c404","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:21.359275941Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1186,"session_id":"a44fff11-dad4-41ce-a228-aa63db42c404"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:21.359288Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW"Wait for a query_work_queue command to finish before sending IO ,"pid":1186,"session_id":"a44fff11-dad4-41ce-a228-aa63db42c404"} {"msg":"The guest has finished waiting for activation with:1","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:21.35930476Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1186} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Fill test Read and Verify all blocks (0..600 range:false) Wrote out file "/tmp/test_repair_verify.data" CLIENT: Tests done. All submitted work has been ACK'd ---------------------------------------------------------------- Crucible gen:1 GIO:true work queues: Upstairs:0 downstairs:3 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1004 Acked Read 256 Done Done Done false 1005 Acked Read 88 Done Done Done false 1006 Acked Read 256 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1003 1003 1003 Downstairs last five completed: 1003 1002 1001 1000 Upstairs last five completed: 1006 1005 1004 1003 1002 CLIENT: Up:0 ds:3 act:3 ---------------------------------------------------------------- Crucible gen:1 GIO:true work queues: Upstairs:0 downstairs:0 Downstairs last five completed: 1007 1006 1005 1004 1003 Upstairs last five completed: 1007 1006 1005 1004 1003 CLIENT: Up:0 ds:0 act:3 CLIENT: All crucible jobs finished, exiting program {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:22:29.243950633Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1186} Begin loop 1 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:29.278425388Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1188,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:29.324116185Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1188,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:29.324765504Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1188} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"0fd98d0d5d909611f8b408ccda2559d35fe117bb\",\n git_commit_timestamp: \"2024-11-08T20:05:44.000000000Z\",\n git_branch: \"mkeeter/drop-skipped-backpressure-guard\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:29.32482869Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1188} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:29.324860669Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1188} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:29.324890587Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1188} {"msg":"Crucible 50e6dab7-dd01-493f-816f-800171dbbd5a has session id: 7db3cff4-f471-4f00-b553-a63b10cfc281","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:29.325388013Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1188,"session_id":"7db3cff4-f471-4f00-b553-a63b10cfc281"} {"msg":"Upstairs opts: Upstairs UUID: 50e6dab7-dd01-493f-816f-800171dbbd5a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:29.325435501Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1188,"session_id":"7db3cff4-f471-4f00-b553-a63b10cfc281"} {"msg":"Crucible stats registered with UUID: 50e6dab7-dd01-493f-816f-800171dbbd5a","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:29.32546069Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1188,"session_id":"7db3cff4-f471-4f00-b553-a63b10cfc281"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:29.357288144Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1188,"task":"crutest"} {"msg":"The guest has requested activation with gen:2","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:29.357326342Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1188} {"msg":"50e6dab7-dd01-493f-816f-800171dbbd5a active request set","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:29.357356141Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1188,"session_id":"7db3cff4-f471-4f00-b553-a63b10cfc281"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:29.35737909Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1188,"client":"0","":"downstairs","session_id":"7db3cff4-f471-4f00-b553-a63b10cfc281"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:29.357423607Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1188,"client":"0","":"downstairs","session_id":"7db3cff4-f471-4f00-b553-a63b10cfc281"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:29.357460076Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1188,"client":"1","":"downstairs","session_id":"7db3cff4-f471-4f00-b553-a63b10cfc281"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:29.357496434Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1188,"client":"1","":"downstairs","session_id":"7db3cff4-f471-4f00-b553-a63b10cfc281"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:29.357531432Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1188,"client":"2","":"downstairs","session_id":"7db3cff4-f471-4f00-b553-a63b10cfc281"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:29.357566461Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1188,"client":"2","":"downstairs","session_id":"7db3cff4-f471-4f00-b553-a63b10cfc281"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:29.357593729Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1188,"":"io task","client":"2","":"downstairs","session_id":"7db3cff4-f471-4f00-b553-a63b10cfc281"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:29.357628107Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1188,"":"io task","client":"0","":"downstairs","session_id":"7db3cff4-f471-4f00-b553-a63b10cfc281"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:29.357652686Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1188,"":"io task","client":"1","":"downstairs","session_id":"7db3cff4-f471-4f00-b553-a63b10cfc281"} {"msg":"ds_connection connected from Ok(127.0.0.1:49485)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:29.357676985Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1188,"":"io task","client":"2","":"downstairs","session_id":"7db3cff4-f471-4f00-b553-a63b10cfc281"} {"msg":"ds_connection connected from Ok(127.0.0.1:39305)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:29.35778484Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1188,"":"io task","client":"1","":"downstairs","session_id":"7db3cff4-f471-4f00-b553-a63b10cfc281"} {"msg":"ds_connection connected from Ok(127.0.0.1:54018)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:29.357871995Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1188,"":"io task","client":"0","":"downstairs","session_id":"7db3cff4-f471-4f00-b553-a63b10cfc281"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:29.358272746Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1188,"client":"1","":"downstairs","session_id":"7db3cff4-f471-4f00-b553-a63b10cfc281"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:29.358300145Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1188,"client":"1","":"downstairs","session_id":"7db3cff4-f471-4f00-b553-a63b10cfc281"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:29.358446807Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1188,"client":"0","":"downstairs","session_id":"7db3cff4-f471-4f00-b553-a63b10cfc281"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:29.358481086Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1188,"client":"0","":"downstairs","session_id":"7db3cff4-f471-4f00-b553-a63b10cfc281"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:29.358504964Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1188,"client":"2","":"downstairs","session_id":"7db3cff4-f471-4f00-b553-a63b10cfc281"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:29.358523814Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1188,"client":"2","":"downstairs","session_id":"7db3cff4-f471-4f00-b553-a63b10cfc281"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:29.358862777Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1188,"client":"1","":"downstairs","session_id":"7db3cff4-f471-4f00-b553-a63b10cfc281"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:29.359329214Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1188,"client":"0","":"downstairs","session_id":"7db3cff4-f471-4f00-b553-a63b10cfc281"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:29.359373482Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1188,"client":"2","":"downstairs","session_id":"7db3cff4-f471-4f00-b553-a63b10cfc281"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:29.359448038Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1188,"client":"1","":"downstairs","session_id":"7db3cff4-f471-4f00-b553-a63b10cfc281"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:29.359479957Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1188,"client":"1","":"downstairs","session_id":"7db3cff4-f471-4f00-b553-a63b10cfc281"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:29.359495256Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1188,"session_id":"7db3cff4-f471-4f00-b553-a63b10cfc281"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:29.359891137Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1188,"client":"2","":"downstairs","session_id":"7db3cff4-f471-4f00-b553-a63b10cfc281"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:29.359915316Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1188,"client":"2","":"downstairs","session_id":"7db3cff4-f471-4f00-b553-a63b10cfc281"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:29.359929415Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1188,"session_id":"7db3cff4-f471-4f00-b553-a63b10cfc281"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:29.359955244Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1188,"client":"0","":"downstairs","session_id":"7db3cff4-f471-4f00-b553-a63b10cfc281"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:29.359969043Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1188,"client":"0","":"downstairs","session_id":"7db3cff4-f471-4f00-b553-a63b10cfc281"} {"msg":"[0]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:29.360044999Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1188,"":"downstairs","":"downstairs","session_id":"7db3cff4-f471-4f00-b553-a63b10cfc281"} {"msg":"[0]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:29.360063988Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1188,"":"downstairs","":"downstairs","session_id":"7db3cff4-f471-4f00-b553-a63b10cfc281"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:29.360078348Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1188,"":"downstairs","":"downstairs","session_id":"7db3cff4-f471-4f00-b553-a63b10cfc281"} {"msg":"[1]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:29.360092287Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1188,"":"downstairs","":"downstairs","session_id":"7db3cff4-f471-4f00-b553-a63b10cfc281"} {"msg":"[1]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:29.360105626Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1188,"":"downstairs","":"downstairs","session_id":"7db3cff4-f471-4f00-b553-a63b10cfc281"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:29.360119646Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1188,"":"downstairs","":"downstairs","session_id":"7db3cff4-f471-4f00-b553-a63b10cfc281"} {"msg":"[2]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:29.360132995Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1188,"":"downstairs","":"downstairs","session_id":"7db3cff4-f471-4f00-b553-a63b10cfc281"} {"msg":"[2]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:29.360146434Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1188,"":"downstairs","":"downstairs","session_id":"7db3cff4-f471-4f00-b553-a63b10cfc281"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:29.360164034Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1188,"":"downstairs","":"downstairs","session_id":"7db3cff4-f471-4f00-b553-a63b10cfc281"} {"msg":"Max found gen is 2","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:29.360177443Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1188,"":"downstairs","":"downstairs","session_id":"7db3cff4-f471-4f00-b553-a63b10cfc281"} {"msg":"Generation requested: 2 >= found:2","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:29.360190622Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1188,"":"downstairs","":"downstairs","session_id":"7db3cff4-f471-4f00-b553-a63b10cfc281"} {"msg":"Next flush: 2","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:29.360203481Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","Wait for a query_work_queue command to finish before sending IO pid":1188,"":"downstairs","":"downstairs","session_id":"7db3cff4-f471-4f00-b553-a63b10cfc281"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:29.3602249Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1188,"":"downstairs","":"downstairs","session_id":"7db3cff4-f471-4f00-b553-a63b10cfc281"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:29.36023853Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1188,"session_id":"7db3cff4-f471-4f00-b553-a63b10cfc281"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:29.360250589Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1188,"client":"0","":"downstairs","session_id":"7db3cff4-f471-4f00-b553-a63b10cfc281"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:29.360263998Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1188,"client":"1","":"downstairs","session_id":"7db3cff4-f471-4f00-b553-a63b10cfc281"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:29.360277238Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1188,"client":"2","":"downstairs","session_id":"7db3cff4-f471-4f00-b553-a63b10cfc281"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:29.360290377Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1188,"session_id":"7db3cff4-f471-4f00-b553-a63b10cfc281"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:29.360302587Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1188,"session_id":"7db3cff4-f471-4f00-b553-a63b10cfc281"} {"msg":"50e6dab7-dd01-493f-816f-800171dbbd5a is now active with session: 7db3cff4-f471-4f00-b553-a63b10cfc281","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:29.360315426Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1188,"session_id":"7db3cff4-f471-4f00-b553-a63b10cfc281"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:29.360327295Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1188,"session_id":"7db3cff4-f471-4f00-b553-a63b10cfc281"} {"msg":"The guest has finished waiting for activation with:2","v":0,"name":"Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 crucible","level":30,"time":"2024-11-08T20:22:29.360343345Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1188} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 286 len 36864 data: 2 2 2 2 2 2 2 2 2 02/30 Read block 587 len 36864 03/30 Read block 2 len 32768 04/30 Read block 353 len 20480 05/30 Read block 127 len 16384 06/30 Write block 568 len 28672 data: 2 2 2 2 2 2 2 07/30 Read block 571 len 12288 08/30 Write block 459 len 12288 data: 2 2 2 09/30 Write block 468 len 12288 data: 2 2 2 10/30 Read block 393 len 28672 11/30 Write block 334 len 36864 data: 2 2 2 2 2 2 2 2 2 12/30 Read block 549 len 20480 13/30 Flush 14/30 Write block 555 len 28672 data: 2 2 2 2 2 2 2 15/30 Read block 179 len 8192 16/30 Write block 583 len 32768 data: 2 2 2 2 2 2 2 2 17/30 Write block 257 len 16384 data: 2 2 2 2 18/30 Read block 444 len 32768 19/30 Read block 558 len 8192 20/30 Read block 533 len 28672 21/30 Write block 527 len 12288 data: 2 2 2 22/30 Write block 192 len 32768 data: 2 2 2 2 2 2 2 2 23/30 Write block 353 len 40960 data: 2 2 2 2 2 2 2 2 2 2 24/30 Write block 5 len 16384 data: 2 2 2 2 25/30 Read block 89 len 8192 26/30 Write block 519 len 16384 data: 2 2 2 2 27/30 Read block 490 len 32768 28/30 Write block 109 len 4096 data: 2 29/30 Read block 448 len 40960 30/30 Write block 292 len 12288 data: 3 3 3 ---------------------------------------------------------------- Crucible gen:2 GIO:true work queues: Upstairs:0 downstairs:17 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1013 Acked Write 7 Done Done Done false 1014 Acked Read 2 Done Done Done false 1015 Acked Write 8 Done Done Done false 1016 Acked Write 4 Done Done Done false 1017 Acked Read 8 Done Done Done false 1018 Acked Read 2 Done Done Done false 1019 Acked Read 7 Done Done Done false 1020 Acked Write 3 Done Done Done false 1021 Acked Write 8 Done Done Done false 1022 Acked Write 10 Done Done Done false 1023 Acked Write 4 Done Done Done false 1024 Acked Read 2 Done Done Done false 1025 Acked Write 4 Done Done Done false 1026 Acked Read 8 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Read 10 Done Done Done false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 16 16 16 48 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1012 1012 1012 Downstairs last five completed: 1012 1011 1010 1009 1008 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:22:29.567150007Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1188} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 1 1 1 T T T 1 020-039 1 1 1 1 1 1 F F F 2 040-059 1 1 1 1 1 1 F F F 3 060-079 1 1 1 1 1 1 F F F 4 080-099 1 1 1 1 1 1 F F F 5 100-119 1 1 1 1 1 1 T T T 6 120-139 1 1 1 1 1 1 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 1 1 1 1 1 1 F F F 9 180-199 1 1 1 1 1 1 T T T 10 200-219 1 1 1 1 1 1 F F F 11 220-239 1 1 1 1 1 1 F F F 12 240-259 1 1 1 1 1 1 T T T 13 260-279 1 1 1 1 1 1 T T T 14 280-299 2 2 2 2 2 2 T T T 15 300-319 1 1 1 1 1 1 F F F 16 320-339 2 2 2 2 2 2 F F F 17 340-359 2 2 2 2 2 2 T T T 18 360-379 1 1 1 1 1 1 T T T 19 380-399 1 1 1 1 1 1 F F F 20 400-419 1 1 1 1 1 1 F F F 21 420-439 1 1 1 1 1 1 F F F 22 440-459 2 2 2 2 2 2 F F F 23 460-479 2 2 2 2 2 2 F F F 24 480-499 1 1 1 1 1 1 F F F 25 500-519 1 1 1 1 1 1 T T T 26 520-539 1 1 1 1 1 1 T T T 27 540-559 1 1 1 1 1 1 T T T 28 560-579 2 2 2 2 2 2 T T T 29 580-599 1 1 1 1 1 1 T T T Max gen: 2, Max flush: 2 A Difference in extent metadata is expected here On loop 1 Verifying data now No change after verify Loop: 1 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 3 3 3 3 3 3 F F F 1 020-039 1 1 1 1 1 1 F F F 2 040-059 1 1 1 1 1 1 F F F 3 060-079 1 1 1 1 1 1 F F F 4 080-099 1 1 1 1 1 1 F F F 5 100-119 3 3 3 3 3 3 F F F 6 120-139 1 1 1 1 1 1 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 1 1 1 1 1 1 F F F 9 180-199 3 3 3 3 3 3 F F F 10 200-219 1 1 1 1 1 1 F F F 11 220-239 1 1 1 1 1 1 F F F 12 240-259 3 3 3 3 3 3 F F F 13 260-279 3 3 3 3 3 3 F F F 14 280-299 3 3 3 3 3 3 F F F 15 300-319 1 1 1 1 1 1 F F F 16 320-339 2 2 2 2 2 2 F F F 17 340-359 3 3 3 3 3 3 F F F 18 360-379 3 3 3 3 3 3 F F F 19 380-399 1 1 1 1 1 1 F F F 20 400-419 1 1 1 1 1 1 F F F 21 420-439 1 1 1 1 1 1 F F F 22 440-459 2 2 2 2 2 2 F F F 23 460-479 2 2 2 2 2 2 F F F 24 480-499 1 1 1 1 1 1 F F F 25 500-519 3 3 3 3 3 3 F F F 26 520-539 3 3 3 3 3 3 F F F 27 540-559 3 3 3 3 3 3 F F F 28 560-579 3 3 3 3 3 3 F F F 29 580-599 3 3 3 3 3 3 F F F Max gen: 3, Max flush: 3 Begin loop 2 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:34.034144197Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1196,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:34.078332517Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1196,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:34.078926278Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1196} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"0fd98d0d5d909611f8b408ccda2559d35fe117bb\",\n git_commit_timestamp: \"2024-11-08T20:05:44.000000000Z\",\n git_branch: \"mkeeter/drop-skipped-backpressure-guard\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:34.078963026Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1196} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:34.078978075Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1196} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:34.078993625Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1196} {"msg":"Crucible 595d3ce9-ceee-44bd-b582-2911502c8a3e has session id: fa15d3f1-3576-4426-a03d-b06155a0853d","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:34.079414044Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1196,"session_id":"fa15d3f1-3576-4426-a03d-b06155a0853d"} {"msg":"Upstairs opts: Upstairs UUID: 595d3ce9-ceee-44bd-b582-2911502c8a3e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:34.079463302Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1196,"session_id":"fa15d3f1-3576-4426-a03d-b06155a0853d"} {"msg":"Crucible stats registered with UUID: 595d3ce9-ceee-44bd-b582-2911502c8a3e","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:34.07948863Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1196,"session_id":"fa15d3f1-3576-4426-a03d-b06155a0853d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:34.110941853Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1196,"task":"crutest"} {"msg":"The guest has requested activation with gen:4","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:34.110973691Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1196} {"msg":"595d3ce9-ceee-44bd-b582-2911502c8a3e active request set","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:34.111023139Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1196,"session_id":"fa15d3f1-3576-4426-a03d-b06155a0853d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:34.111042638Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1196,"client":"0","":"downstairs","session_id":"fa15d3f1-3576-4426-a03d-b06155a0853d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:34.111068667Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1196,"client":"0","":"downstairs","session_id":"fa15d3f1-3576-4426-a03d-b06155a0853d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:34.111086446Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1196,"client":"1","":"downstairs","session_id":"fa15d3f1-3576-4426-a03d-b06155a0853d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:34.111103905Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1196,"client":"1","":"downstairs","session_id":"fa15d3f1-3576-4426-a03d-b06155a0853d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:34.111120954Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1196,"client":"2","":"downstairs","session_id":"fa15d3f1-3576-4426-a03d-b06155a0853d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:34.111138414Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1196,"client":"2","":"downstairs","session_id":"fa15d3f1-3576-4426-a03d-b06155a0853d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:34.111155023Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1196,"":"io task","client":"2","":"downstairs","session_id":"fa15d3f1-3576-4426-a03d-b06155a0853d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:34.111172412Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1196,"":"io task","client":"0","":"downstairs","session_id":"fa15d3f1-3576-4426-a03d-b06155a0853d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:34.111189451Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1196,"":"io task","client":"1","":"downstairs","session_id":"fa15d3f1-3576-4426-a03d-b06155a0853d"} {"msg":"ds_connection connected from Ok(127.0.0.1:47932)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:34.111218319Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1196,"":"io task","client":"2","":"downstairs","session_id":"fa15d3f1-3576-4426-a03d-b06155a0853d"} {"msg":"ds_connection connected from Ok(127.0.0.1:42833)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:34.111244268Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1196,"":"io task","client":"0","":"downstairs","session_id":"fa15d3f1-3576-4426-a03d-b06155a0853d"} {"msg":"ds_connection connected from Ok(127.0.0.1:47143)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:34.111462268Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1196,"":"io task","client":"1","":"downstairs","session_id":"fa15d3f1-3576-4426-a03d-b06155a0853d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:34.11203158Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1196,"client":"0","":"downstairs","session_id":"fa15d3f1-3576-4426-a03d-b06155a0853d"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:34.112071298Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1196,"client":"0","":"downstairs","session_id":"fa15d3f1-3576-4426-a03d-b06155a0853d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:34.112102556Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1196,"client":"2","":"downstairs","session_id":"fa15d3f1-3576-4426-a03d-b06155a0853d"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:34.112150664Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1196,"client":"2","":"downstairs","session_id":"fa15d3f1-3576-4426-a03d-b06155a0853d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:34.112276118Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1196,"client":"1","":"downstairs","session_id":"fa15d3f1-3576-4426-a03d-b06155a0853d"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:34.112310336Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1196,"client":"1","":"downstairs","session_id":"fa15d3f1-3576-4426-a03d-b06155a0853d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:34.112789343Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1196,"client":"0","":"downstairs","session_id":"fa15d3f1-3576-4426-a03d-b06155a0853d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:34.112825771Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1196,"client":"2","":"downstairs","session_id":"fa15d3f1-3576-4426-a03d-b06155a0853d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:34.112895968Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1196,"client":"1","":"downstairs","session_id":"fa15d3f1-3576-4426-a03d-b06155a0853d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:34.113405923Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1196,"client":"0","":"downstairs","session_id":"fa15d3f1-3576-4426-a03d-b06155a0853d"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:34.113435571Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1196,"client":"0","":"downstairs","session_id":"fa15d3f1-3576-4426-a03d-b06155a0853d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:34.11345081Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1196,"session_id":"fa15d3f1-3576-4426-a03d-b06155a0853d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:34.113518867Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1196,"client":"1","":"downstairs","session_id":"fa15d3f1-3576-4426-a03d-b06155a0853d"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:34.113539866Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1196,"client":"1","":"downstairs","session_id":"fa15d3f1-3576-4426-a03d-b06155a0853d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:34.113553785Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1196,"session_id":"fa15d3f1-3576-4426-a03d-b06155a0853d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:34.113566065Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1196,"client":"2","":"downstairs","session_id":"fa15d3f1-3576-4426-a03d-b06155a0853d"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:34.113579984Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1196,"client":"2","":"downstairs","session_id":"fa15d3f1-3576-4426-a03d-b06155a0853d"} {"msg":"[0]R flush_numbers[0..12]: [3, 1, 1, 1, 1, 3, 1, 1, 1, 3, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:34.113593983Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1196,"":"downstairs","":"downstairs","session_id":"fa15d3f1-3576-4426-a03d-b06155a0853d"} {"msg":"[0]R generation[0..12]: [3, 1, 1, 1, 1, 3, 1, 1, 1, 3, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:34.113607743Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1196,"":"downstairs","":"downstairs","session_id":"fa15d3f1-3576-4426-a03d-b06155a0853d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:34.113622052Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1196,"":"downstairs","":"downstairs","session_id":"fa15d3f1-3576-4426-a03d-b06155a0853d"} {"msg":"[1]R flush_numbers[0..12]: [3, 1, 1, 1, 1, 3, 1, 1, 1, 3, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:34.113635771Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1196,"":"downstairs","":"downstairs","session_id":"fa15d3f1-3576-4426-a03d-b06155a0853d"} {"msg":"[1]R generation[0..12]: [3, 1, 1, 1, 1, 3, 1, 1, 1, 3, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:34.113649201Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1196,"":"downstairs","":"downstairs","session_id":"fa15d3f1-3576-4426-a03d-b06155a0853d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:34.11366332Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1196,"":"downstairs","":"downstairs","session_id":"fa15d3f1-3576-4426-a03d-b06155a0853d"} {"msg":"[2]R flush_numbers[0..12]: [3, 1, 1, 1, 1, 3, 1, 1, 1, 3, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:34.113677059Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1196,"":"downstairs","":"downstairs","session_id":"fa15d3f1-3576-4426-a03d-b06155a0853d"} {"msg":"[2]R generation[0..12]: [3, 1, 1, 1, 1, 3, 1, 1, 1, 3, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:34.113690579Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1196,"":"downstairs","":"downstairs","session_id":"fa15d3f1-3576-4426-a03d-b06155a0853d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:34.113704498Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1196,"":"downstairs","":"downstairs","session_id":"fa15d3f1-3576-4426-a03d-b06155a0853d"} {"msg":"Max found gen is 4","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:34.113717827Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1196,"":"downstairs","":"downstairs","session_id":"fa15d3f1-3576-4426-a03d-b06155a0853d"} {"msg":"Generation requested: 4 >= found:4","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-08T20:22:34.113734536Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1196,"":"downstairs","":"downstairs","session_id":"fa15d3f1-3576-4426-a03d-b06155a0853d"} {"msg":"Next flush: 4","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:34.113769825Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1196,"":"downstairs","":"downstairs","session_id":"fa15d3f1-3576-4426-a03d-b06155a0853d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:34.113783784Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1196,"":"downstairs","":"downstairs","session_id":"fa15d3f1-3576-4426-a03d-b06155a0853d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:34.113797224Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1196,"session_id":"fa15d3f1-3576-4426-a03d-b06155a0853d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:34.113809363Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1196,"client":"0","":"downstairs","session_id":"fa15d3f1-3576-4426-a03d-b06155a0853d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:34.113822572Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1196,"client":"1","":"downstairs","session_id":"fa15d3f1-3576-4426-a03d-b06155a0853d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:34.113835592Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1196,"client":"2","":"downstairs","session_id":"fa15d3f1-3576-4426-a03d-b06155a0853d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:34.113848851Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1196,"session_id":"fa15d3f1-3576-4426-a03d-b06155a0853d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:34.11386122Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1196,"session_id":"fa15d3f1-3576-4426-a03d-b06155a0853d"} {"msg":"595d3ce9-ceee-44bd-b582-2911502c8a3e is now active with session: fa15d3f1-3576-4426-a03d-b06155a0853d","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:34.11387407Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1196,"session_id":"fa15d3f1-3576-4426-a03d-b06155a0853d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:34.113886049Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1196,"session_id":"fa15d3f1-3576-4426-a03d-b06155a0853d"} {"msg":"The guest has finished waiting for activation with:4","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:34.113898339Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1196} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 352 len 4096 data: 2 02/30 Flush 03/30 Write block 554 len 28672 data: 2 3 3 3 3 3 3 04/30 Flush 05/30 Write block 471 len 12288 data: 2 2 2 06/30 Flush 07/30 Write block 282 len 32768 data: 2 2 2 2 3 3 3 3 08/30 Read block 459 len 32768 09/30 Write block 72 len 32768 data: 2 2 2 2 2 2 2 2 10/30 Flush 11/30 Write block 367 len 8192 data: 2 2 12/30 Read block 482 len 32768 13/30 Write block 4 len 40960 data: 2 3 3 3 3 2 2 2 2 2 14/30 Read block 229 len 16384 15/30 Write block 137 len 20480 data: 2 2 2 2 2 16/30 Write block 423 len 12288 data: 2 2 2 17/30 Write block 429 len 20480 data: 2 2 2 2 2 18/30 Read block 531 len 12288 19/30 Write block 358 len 24576 data: 3 3 3 3 3 2 20/30 Write block 254 len 4096 data: 2 21/30 Read block 79 len 4096 22/30 Flush 23/30 Write block 83 len 12288 data: 2 2 2 24/30 Write block 421 len 4096 data: 2 25/30 Write block 571 len 12288 data: 3 3 3 26/30 Read block 463 len 4096 27/30 Write block 133 len 12288 data: 2 2 2 28/30 Read block 21 len 28672 29/30 Write block 397 len 40960 data: 2 2 2 2 2 2 2 2 2 2 30/30 Write block 404 len 24576 data: 3 3 3 2 2 2 ---------------------------------------------------------------- Crucible gen:4 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 3 Done Done Done false 1023 Acked Write 1 Done Done Done false 1024 Acked Write 3 Done Done Done false 1025 Acked Read 1 Done Done Done false 1026 Acked Write 3 Done Done Done false 1027 Acked Read 7 Done Done Done false 1028 Acked Write 10 Sent Sent Sent false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 6 6 6 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1021 1021 1021 Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:22:34.140439371Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1196} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 4 4 4 8 8 8 F F F 1 020-039 1 1 1 1 1 1 F F F 2 040-059 1 1 1 1 1 1 F F F 3 060-079 4 4 4 7 7 7 F F F 4 080-099 1 1 1 1 1 1 T T T 5 100-119 3 3 3 3 3 3 F F F 6 120-139 4 4 4 8 8 8 T T T 7 140-159 4 4 4 8 8 8 F F F 8 160-179 1 1 1 1 1 1 F F F 9 180-199 3 3 3 3 3 3 F F F 10 200-219 1 1 1 1 1 1 F F F 11 220-239 1 1 1 1 1 1 F F F 12 240-259 4 4 4 8 8 8 F F F 13 260-279 3 3 3 3 3 3 F F F 14 280-299 4 4 4 7 7 7 F F F 15 300-319 1 1 1 1 1 1 F F F 16 320-339 2 2 2 2 2 2 F F F 17 340-359 4 4 4 8 8 8 F F F 18 360-379 4 4 4 8 8 8 F F F 19 380-399 1 1 1 1 1 1 T T T 20 400-419 1 1 1 1 1 1 T T T 21 420-439 4 4 4 8 8 8 T T T 22 440-459 2 2 2 2 2 2 F F F 23 460-479 4 4 4 6 6 6 F F F 24 480-499 1 1 1 1 1 1 F F F 25 500-519 3 3 3 3 3 3 F F F 26 520-539 3 3 3 3 3 3 F F F 27 540-559 4 4 4 5 5 5 F F F 28 560-579 4 4 4 5 5 5 T T T 29 580-599 3 3 3 3 3 3 F F F Max gen: 4, Max flush: 8 A Difference in extent metadata is expected here On loop 2 Verifying data now No change after verify Loop: 2 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 4 4 4 8 8 8 F F F 1 020-039 1 1 1 1 1 1 F F F 2 040-059 1 1 1 1 1 1 F F F 3 060-079 4 4 4 7 7 7 F F F 4 080-099 5 5 5 9 9 9 F F F 5 100-119 3 3 3 3 3 3 F F F 6 120-139 5 5 5 9 9 9 F F F 7 140-159 4 4 4 8 8 8 F F F 8 160-179 1 1 1 1 1 1 F F F 9 180-199 3 3 3 3 3 3 F F F 10 200-219 1 1 1 1 1 1 F F F 11 220-239 1 1 1 1 1 1 F F F 12 240-259 4 4 4 8 8 8 F F F 13 260-279 3 3 3 3 3 3 F F F 14 280-299 4 4 4 7 7 7 F F F 15 300-319 1 1 1 1 1 1 F F F 16 320-339 2 2 2 2 2 2 F F F 17 340-359 4 4 4 8 8 8 F F F 18 360-379 4 4 4 8 8 8 F F F 19 380-399 5 5 5 9 9 9 F F F 20 400-419 5 5 5 9 9 9 F F F 21 420-439 5 5 5 9 9 9 F F F 22 440-459 2 2 2 2 2 2 F F F 23 460-479 4 4 4 6 6 6 F F F 24 480-499 1 1 1 1 1 1 F F F 25 500-519 3 3 3 3 3 3 F F F 26 520-539 3 3 3 3 3 3 F F F 27 540-559 4 4 4 5 5 5 F F F 28 560-579 5 5 5 9 9 9 F F F 29 580-599 3 3 3 3 3 3 F F F Max gen: 5, Max flush: 9 Begin loop 3 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:38.477729191Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1204,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:38.521139879Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1204,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:38.521758929Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1204} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"0fd98d0d5d909611f8b408ccda2559d35fe117bb\",\n git_commit_timestamp: \"2024-11-08T20:05:44.000000000Z\",\n git_branch: \"mkeeter/drop-skipped-backpressure-guard\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:38.521795077Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1204} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:38.521809536Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1204} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:38.521824276Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1204} {"msg":"Crucible 94ce7724-6fd9-4f27-bc22-ebce1c85a543 has session id: 534e1c72-498b-49bf-8498-b674fd6476da","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:38.522283883Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1204,"session_id":"534e1c72-498b-49bf-8498-b674fd6476da"} {"msg":"Upstairs opts: Upstairs UUID: 94ce7724-6fd9-4f27-bc22-ebce1c85a543, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:38.522330511Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1204,"session_id":"534e1c72-498b-49bf-8498-b674fd6476da"} {"msg":"Crucible stats registered with UUID: 94ce7724-6fd9-4f27-bc22-ebce1c85a543","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:38.52235281Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1204,"session_id":"534e1c72-498b-49bf-8498-b674fd6476da"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:38.553428221Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1204,"task":"crutest"} {"msg":"The guest has requested activation with gen:6","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:38.553486668Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1204} {"msg":"94ce7724-6fd9-4f27-bc22-ebce1c85a543 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:38.553522646Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1204,"session_id":"534e1c72-498b-49bf-8498-b674fd6476da"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:38.553553045Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1204,"client":"0","":"downstairs","session_id":"534e1c72-498b-49bf-8498-b674fd6476da"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:38.553600123Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1204,"client":"0","":"downstairs","session_id":"534e1c72-498b-49bf-8498-b674fd6476da"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:38.553636171Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1204,"client":"1","":"downstairs","session_id":"534e1c72-498b-49bf-8498-b674fd6476da"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:38.553671659Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1204,"client":"1","":"downstairs","session_id":"534e1c72-498b-49bf-8498-b674fd6476da"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:38.553705248Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1204,"client":"2","":"downstairs","session_id":"534e1c72-498b-49bf-8498-b674fd6476da"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:38.553739746Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1204,"client":"2","":"downstairs","session_id":"534e1c72-498b-49bf-8498-b674fd6476da"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:38.553773364Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1204,"":"io task","client":"2","":"downstairs","session_id":"534e1c72-498b-49bf-8498-b674fd6476da"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:38.553807812Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1204,"":"io task","client":"0","":"downstairs","session_id":"534e1c72-498b-49bf-8498-b674fd6476da"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:38.553843291Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1204,"":"io task","client":"1","":"downstairs","session_id":"534e1c72-498b-49bf-8498-b674fd6476da"} {"msg":"ds_connection connected from Ok(127.0.0.1:50912)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:38.554048981Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1204,"":"io task","client":"1","":"downstairs","session_id":"534e1c72-498b-49bf-8498-b674fd6476da"} {"msg":"ds_connection connected from Ok(127.0.0.1:47986)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:38.55407603Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1204,"":"io task","client":"0","":"downstairs","session_id":"534e1c72-498b-49bf-8498-b674fd6476da"} {"msg":"ds_connection connected from Ok(127.0.0.1:59858)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:38.554095449Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1204,"":"io task","client":"2","":"downstairs","session_id":"534e1c72-498b-49bf-8498-b674fd6476da"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:38.554622943Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1204,"client":"0","":"downstairs","session_id":"534e1c72-498b-49bf-8498-b674fd6476da"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:38.554650251Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1204,"client":"0","":"downstairs","session_id":"534e1c72-498b-49bf-8498-b674fd6476da"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:38.554775065Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1204,"client":"1","":"downstairs","session_id":"534e1c72-498b-49bf-8498-b674fd6476da"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:38.554849032Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1204,"client":"1","":"downstairs","session_id":"534e1c72-498b-49bf-8498-b674fd6476da"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:38.55488047Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1204,"client":"2","":"downstairs","session_id":"534e1c72-498b-49bf-8498-b674fd6476da"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:38.554937147Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1204,"client":"2","":"downstairs","session_id":"534e1c72-498b-49bf-8498-b674fd6476da"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:38.555338758Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1204,"client":"1","":"downstairs","session_id":"534e1c72-498b-49bf-8498-b674fd6476da"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:38.555369766Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1204,"client":"0","":"downstairs","session_id":"534e1c72-498b-49bf-8498-b674fd6476da"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:38.555462572Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1204,"client":"2","":"downstairs","session_id":"534e1c72-498b-49bf-8498-b674fd6476da"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:38.555919339Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1204,"client":"0","":"downstairs","session_id":"534e1c72-498b-49bf-8498-b674fd6476da"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:38.555946338Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1204,"client":"0","":"downstairs","session_id":"534e1c72-498b-49bf-8498-b674fd6476da"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:38.555960697Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1204,"session_id":"534e1c72-498b-49bf-8498-b674fd6476da"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:38.556042273Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1204,"client":"1","":"downstairs","session_id":"534e1c72-498b-49bf-8498-b674fd6476da"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:38.556075292Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1204,"client":"1","":"downstairs","session_id":"534e1c72-498b-49bf-8498-b674fd6476da"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:38.556091741Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1204,"session_id":"534e1c72-498b-49bf-8498-b674fd6476da"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:38.55610437Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1204,"client":"2","":"downstairs","session_id":"534e1c72-498b-49bf-8498-b674fd6476da"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:38.55611813Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1204,"client":"2","":"downstairs","session_id":"534e1c72-498b-49bf-8498-b674fd6476da"} {"msg":"[0]R flush_numbers[0..12]: [8, 1, 1, 7, 9, 3, 9, 8, 1, 3, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:38.556134959Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1204,"":"downstairs","":"downstairs","session_id":"534e1c72-498b-49bf-8498-b674fd6476da"} {"msg":"[0]R generation[0..12]: [4, 1, 1, 4, 5, 3, 5, 4, 1, 3, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:38.556148998Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1204,"":"downstairs","":"downstairs","session_id":"534e1c72-498b-49bf-8498-b674fd6476da"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:38.556163288Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1204,"":"downstairs","":"downstairs","session_id":"534e1c72-498b-49bf-8498-b674fd6476da"} {"msg":"[1]R flush_numbers[0..12]: [8, 1, 1, 7, 9, 3, 9, 8, 1, 3, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:38.556176957Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1204,"":"downstairs","":"downstairs","session_id":"534e1c72-498b-49bf-8498-b674fd6476da"} {"msg":"[1]R generation[0..12]: [4, 1, 1, 4, 5, 3, 5, 4, 1, 3, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:38.556190426Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1204,"":"downstairs","":"downstairs","session_id":"534e1c72-498b-49bf-8498-b674fd6476da"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:38.556204366Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1204,"":"downstairs","":"downstairs","session_id":"534e1c72-498b-49bf-8498-b674fd6476da"} {"msg":"[2]R flush_numbers[0..12]: [8, 1, 1, 7, 9, 3, 9, 8, 1, 3, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:38.556217735Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1204,"":"downstairs","":"downstairs","session_id":"534e1c72-498b-49bf-8498-b674fd6476da"} {"msg":"[2]R generation[0..12]: [4, 1, 1, 4, 5, 3, 5, 4, 1, 3, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:38.556231024Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1204,"":"downstairs","":"downstairs","session_id":"534e1c72-498b-49bf-8498-b674fd6476da"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:38.556246203Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1204,"":"downstairs","":"downstairs","session_id":"534e1c72-498b-49bf-8498-b674fd6476da"} {"msg":"Max found gen is 6","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:38.556270162Z","hostname"Wait for a query_work_queue command to finish before sending IO :"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1204,"":"downstairs","":"downstairs","session_id":"534e1c72-498b-49bf-8498-b674fd6476da"} {"msg":"Generation requested: 6 >= found:6","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:38.55630425Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1204,"":"downstairs","":"downstairs","session_id":"534e1c72-498b-49bf-8498-b674fd6476da"} {"msg":"Next flush: 10","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:38.55632502Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1204,"":"downstairs","":"downstairs","session_id":"534e1c72-498b-49bf-8498-b674fd6476da"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:38.556338679Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1204,"":"downstairs","":"downstairs","session_id":"534e1c72-498b-49bf-8498-b674fd6476da"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:38.556352148Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1204,"session_id":"534e1c72-498b-49bf-8498-b674fd6476da"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:38.556364358Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1204,"client":"0","":"downstairs","session_id":"534e1c72-498b-49bf-8498-b674fd6476da"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:38.556380727Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1204,"client":"1","":"downstairs","session_id":"534e1c72-498b-49bf-8498-b674fd6476da"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:38.556394136Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1204,"client":"2","":"downstairs","session_id":"534e1c72-498b-49bf-8498-b674fd6476da"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:38.556409945Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1204,"session_id":"534e1c72-498b-49bf-8498-b674fd6476da"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:38.556432804Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1204,"session_id":"534e1c72-498b-49bf-8498-b674fd6476da"} {"msg":"94ce7724-6fd9-4f27-bc22-ebce1c85a543 is now active with session: 534e1c72-498b-49bf-8498-b674fd6476da","v":0,"name":"crucible","level":30,"time":"Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 2024-11-08T20:22:38.556457683Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1204,"session_id":"534e1c72-498b-49bf-8498-b674fd6476da"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:38.556483502Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1204,"session_id":"534e1c72-498b-49bf-8498-b674fd6476da"} {"msg":"The guest has finished waiting for activation with:6","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:38.556506031Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1204} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 583 len 20480 data: 3 3 3 3 3 02/30 Read block 254 len 24576 03/30 Read block 76 len 24576 04/30 Flush 05/30 Write block 473 len 28672 data: 3 2 2 2 2 2 2 06/30 Read block 275 len 12288 07/30 Read block 291 len 40960 08/30 Write block 237 len 40960 data: 2 2 2 2 2 2 2 2 2 2 09/30 Write block 32 len 12288 data: 2 2 2 10/30 Read block 140 len 24576 11/30 Read block 227 len 4096 12/30 Read block 472 len 28672 13/30 Read block 76 len 24576 14/30 Write block 411 len 4096 data: 2 15/30 Write block 366 len 28672 data: 2 3 3 2 2 2 2 16/30 Write block 240 len 40960 data: 3 3 3 3 3 3 3 2 2 2 17/30 Read block 164 len 24576 18/30 Read block 100 len 20480 19/30 Write block 42 len 40960 data: 2 2 2 2 2 2 2 2 2 2 20/30 Read block 87 len 8192 21/30 Read block 48 len 36864 22/30 Flush 23/30 Write block 168 len 16384 data: 2 2 2 2 24/30 Write block 288 len 32768 data: 4 4 3 3 4 4 4 2 25/30 Write block 349 len 36864 data: 2 2 2 3 3 3 3 3 3 26/30 Read block 46 len 12288 27/30 Read block 270 len 16384 28/30 Read block 148 len 36864 29/30 Read block 558 len 28672 30/30 Write block 310 len 40960 data: 2 2 2 2 2 2 2 2 2 2 ---------------------------------------------------------------- Crucible gen:6 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 4 Done Done Done false 1023 Acked Write 8 Done Done Done false 1024 Acked Write 9 Done Done Done false 1025 Acked Read 3 Done Done Done false 1026 Acked Read 4 Done Done Done false 1027 Acked Read 9 Done Done Done false 1028 Acked Read 7 Done Done Done false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 7 7 7 21 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1021 1021 1021 Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:22:38.581819574Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1204} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 4 4 4 8 8 8 F F F 1 020-039 6 6 6 11 11 11 F F F 2 040-059 6 6 6 11 11 11 F F F 3 060-079 4 4 4 7 7 7 F F F 4 080-099 5 5 5 9 9 9 F F F 5 100-119 3 3 3 3 3 3 F F F 6 120-139 5 5 5 9 9 9 F F F 7 140-159 4 4 4 8 8 8 F F F 8 160-179 1 1 1 1 1 1 T T T 9 180-199 3 3 3 3 3 3 F F F 10 200-219 1 1 1 1 1 1 F F F 11 220-239 6 6 6 11 11 11 F F F 12 240-259 6 6 6 11 11 11 F F F 13 260-279 3 3 3 3 3 3 F F F 14 280-299 4 4 4 7 7 7 T T T 15 300-319 1 1 1 1 1 1 T T T 16 320-339 2 2 2 2 2 2 F F F 17 340-359 4 4 4 8 8 8 T T T 18 360-379 6 6 6 11 11 11 F F F 19 380-399 5 5 5 9 9 9 F F F 20 400-419 6 6 6 11 11 11 F F F 21 420-439 5 5 5 9 9 9 F F F 22 440-459 2 2 2 2 2 2 F F F 23 460-479 6 6 6 11 11 11 F F F 24 480-499 1 1 1 1 1 1 F F F 25 500-519 3 3 3 3 3 3 F F F 26 520-539 3 3 3 3 3 3 F F F 27 540-559 4 4 4 5 5 5 F F F 28 560-579 5 5 5 9 9 9 F F F 29 580-599 6 6 6 10 10 10 F F F Max gen: 6, Max flush: 11 A Difference in extent metadata is expected here On loop 3 Verifying data now No change after verify Loop: 3 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 4 4 4 8 8 8 F F F 1 020-039 6 6 6 11 11 11 F F F 2 040-059 6 6 6 11 11 11 F F F 3 060-079 4 4 4 7 7 7 F F F 4 080-099 5 5 5 9 9 9 F F F 5 100-119 3 3 3 3 3 3 F F F 6 120-139 5 5 5 9 9 9 F F F 7 140-159 4 4 4 8 8 8 F F F 8 160-179 7 7 7 12 12 12 F F F 9 180-199 3 3 3 3 3 3 F F F 10 200-219 1 1 1 1 1 1 F F F 11 220-239 6 6 6 11 11 11 F F F 12 240-259 6 6 6 11 11 11 F F F 13 260-279 3 3 3 3 3 3 F F F 14 280-299 7 7 7 12 12 12 F F F 15 300-319 7 7 7 12 12 12 F F F 16 320-339 2 2 2 2 2 2 F F F 17 340-359 7 7 7 12 12 12 F F F 18 360-379 6 6 6 11 11 11 F F F 19 380-399 5 5 5 9 9 9 F F F 20 400-419 6 6 6 11 11 11 F F F 21 420-439 5 5 5 9 9 9 F F F 22 440-459 2 2 2 2 2 2 F F F 23 460-479 6 6 6 11 11 11 F F F 24 480-499 1 1 1 1 1 1 F F F 25 500-519 3 3 3 3 3 3 F F F 26 520-539 3 3 3 3 3 3 F F F 27 540-559 4 4 4 5 5 5 F F F 28 560-579 5 5 5 9 9 9 F F F 29 580-599 6 6 6 10 10 10 F F F Max gen: 7, Max flush: 12 Begin loop 4 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:42.882786466Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1212,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:42.926832843Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1212,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:42.927473682Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1212} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"0fd98d0d5d909611f8b408ccda2559d35fe117bb\",\n git_commit_timestamp: \"2024-11-08T20:05:44.000000000Z\",\n git_branch: \"mkeeter/drop-skipped-backpressure-guard\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:42.92751101Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1212} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:42.927525139Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1212} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:42.927540129Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1212} {"msg":"Crucible e44c7de6-aac0-4947-b8f5-5c17ffdf796a has session id: e17538e9-e464-46fb-8305-c414f01bc4a2","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:42.927978337Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1212,"session_id":"e17538e9-e464-46fb-8305-c414f01bc4a2"} {"msg":"Upstairs opts: Upstairs UUID: e44c7de6-aac0-4947-b8f5-5c17ffdf796a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:42.928045154Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1212,"session_id":"e17538e9-e464-46fb-8305-c414f01bc4a2"} {"msg":"Crucible stats registered with UUID: e44c7de6-aac0-4947-b8f5-5c17ffdf796a","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:42.928074013Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1212,"session_id":"e17538e9-e464-46fb-8305-c414f01bc4a2"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:42.959667578Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1212,"task":"crutest"} {"msg":"The guest has requested activation with gen:8","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:42.959701507Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1212} {"msg":"e44c7de6-aac0-4947-b8f5-5c17ffdf796a active request set","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:42.959727606Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1212,"session_id":"e17538e9-e464-46fb-8305-c414f01bc4a2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:42.959744595Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1212,"client":"0","":"downstairs","session_id":"e17538e9-e464-46fb-8305-c414f01bc4a2"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:42.959771803Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1212,"client":"0","":"downstairs","session_id":"e17538e9-e464-46fb-8305-c414f01bc4a2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:42.959794892Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1212,"client":"1","":"downstairs","session_id":"e17538e9-e464-46fb-8305-c414f01bc4a2"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:42.959812821Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1212,"client":"1","":"downstairs","session_id":"e17538e9-e464-46fb-8305-c414f01bc4a2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:42.959830761Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1212,"client":"2","":"downstairs","session_id":"e17538e9-e464-46fb-8305-c414f01bc4a2"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:42.959907647Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1212,"client":"2","":"downstairs","session_id":"e17538e9-e464-46fb-8305-c414f01bc4a2"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:42.959949125Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1212,"":"io task","client":"2","":"downstairs","session_id":"e17538e9-e464-46fb-8305-c414f01bc4a2"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:42.959975273Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1212,"":"io task","client":"0","":"downstairs","session_id":"e17538e9-e464-46fb-8305-c414f01bc4a2"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:42.959993342Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1212,"":"io task","client":"1","":"downstairs","session_id":"e17538e9-e464-46fb-8305-c414f01bc4a2"} {"msg":"ds_connection connected from Ok(127.0.0.1:55168)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:42.960011122Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1212,"":"io task","client":"0","":"downstairs","session_id":"e17538e9-e464-46fb-8305-c414f01bc4a2"} {"msg":"ds_connection connected from Ok(127.0.0.1:58440)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:42.960230591Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1212,"":"io task","client":"2","":"downstairs","session_id":"e17538e9-e464-46fb-8305-c414f01bc4a2"} {"msg":"ds_connection connected from Ok(127.0.0.1:56044)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:42.960271229Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1212,"":"io task","client":"1","":"downstairs","session_id":"e17538e9-e464-46fb-8305-c414f01bc4a2"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:42.960753745Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1212,"client":"0","":"downstairs","session_id":"e17538e9-e464-46fb-8305-c414f01bc4a2"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:42.960785174Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1212,"client":"0","":"downstairs","session_id":"e17538e9-e464-46fb-8305-c414f01bc4a2"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:42.960801933Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1212,"client":"2","":"downstairs","session_id":"e17538e9-e464-46fb-8305-c414f01bc4a2"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:42.960819102Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1212,"client":"2","":"downstairs","session_id":"e17538e9-e464-46fb-8305-c414f01bc4a2"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:42.960842551Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1212,"client":"1","":"downstairs","session_id":"e17538e9-e464-46fb-8305-c414f01bc4a2"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:42.96085767Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1212,"client":"1","":"downstairs","session_id":"e17538e9-e464-46fb-8305-c414f01bc4a2"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:42.96147544Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1212,"client":"1","":"downstairs","session_id":"e17538e9-e464-46fb-8305-c414f01bc4a2"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:42.961504829Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1212,"client":"2","":"downstairs","session_id":"e17538e9-e464-46fb-8305-c414f01bc4a2"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:42.961520038Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1212,"client":"0","":"downstairs","session_id":"e17538e9-e464-46fb-8305-c414f01bc4a2"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:42.962027803Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1212,"client":"2","":"downstairs","session_id":"e17538e9-e464-46fb-8305-c414f01bc4a2"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:42.962057122Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1212,"client":"2","":"downstairs","session_id":"e17538e9-e464-46fb-8305-c414f01bc4a2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:42.962072011Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1212,"session_id":"e17538e9-e464-46fb-8305-c414f01bc4a2"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:42.962138378Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1212,"client":"1","":"downstairs","session_id":"e17538e9-e464-46fb-8305-c414f01bc4a2"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:42.962157967Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1212,"client":"1","":"downstairs","session_id":"e17538e9-e464-46fb-8305-c414f01bc4a2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:42.962171676Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1212,"session_id":"e17538e9-e464-46fb-8305-c414f01bc4a2"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:42.962184115Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1212,"client":"0","":"downstairs","session_id":"e17538e9-e464-46fb-8305-c414f01bc4a2"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:42.962197635Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1212,"client":"0","":"downstairs","session_id":"e17538e9-e464-46fb-8305-c414f01bc4a2"} {"msg":"[0]R flush_numbers[0..12]: [8, 11, 11, 7, 9, 3, 9, 8, 12, 3, 1, 11]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:42.962211154Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1212,"":"downstairs","":"downstairs","session_id":"e17538e9-e464-46fb-8305-c414f01bc4a2"} {"msg":"[0]R generation[0..12]: [4, 6, 6, 4, 5, 3, 5, 4, 7, 3, 1, 6]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:42.962224853Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1212,"":"downstairs","":"downstairs","session_id":"e17538e9-e464-46fb-8305-c414f01bc4a2"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:42.962239083Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1212,"":"downstairs","":"downstairs","session_id":"e17538e9-e464-46fb-8305-c414f01bc4a2"} {"msg":"[1]R flush_numbers[0..12]: [8, 11, 11, 7, 9, 3, 9, 8, 12, 3, 1, 11]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:42.962252812Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1212,"":"downstairs","":"downstairs","session_id":"e17538e9-e464-46fb-8305-c414f01bc4a2"} {"msg":"[1]R generation[0..12]: [4, 6, 6, 4, 5, 3, 5, 4, 7, 3, 1, 6]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:42.962269511Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1212,"":"downstairs","":"downstairs","session_id":"e17538e9-e464-46fb-8305-c414f01bc4a2"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:42.96228383Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1212,"":"downstairs","":"downstairs","session_id":"e17538e9-e464-46fb-8305-c414f01bc4a2"} {"msg":"[2]R flush_numbers[0..12]: [8, 11, 11, 7, 9, 3, 9, 8, 12, 3, 1, 11]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:42.96229759Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1212,"":"downstairs","":"downstairs","session_id":"e17538e9-e464-46fb-8305-c414f01bc4a2"} {"msg":"[2]R generation[0..12]: [4, 6, 6, 4, 5, 3, 5, 4, 7, 3, 1, 6]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:42.962311169Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1212,"":"downstairs","":"downstairs","session_id":"e17538e9-e464-46fb-8305-c414f01bc4a2"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:42.962325219Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1212,"":"downstairs","":"downstairs","session_id":"e17538e9-e464-46fb-8305-c414f01bc4a2"} {"msg":"Max found gen is 8","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:42.962338448Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1212,"":"downstairs","":"downstairs","session_id":"e17538e9-e464-46fb-8305-c414f01bc4a2"} {"msg":"Generation requested: 8 >= found:8","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:42.962351877Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1212,"":"downstairs","":"downstairs","session_id":"e17538e9-e464-46fb-8305-c414f01bc4a2"} {"msg":"Next flush: 13","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:42.962364737Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1212,"":"downstairs","":"downstairs","session_id":"e17538e9-e464-46fb-8305-c414f01bc4a2"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:42.962377686Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1212,"":"downstairs","":"downstairs","session_id":"e17538e9-e464-46fb-8305-c414f01bc4a2"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-08T20:22:42.962390965Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1212,"session_id":"e17538e9-e464-46fb-8305-c414f01bc4a2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:42.962412844Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1212,"client":"0","":"downstairs","session_id":"e17538e9-e464-46fb-8305-c414f01bc4a2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:42.962426134Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1212,"client":"1","":"downstairs","session_id":"e17538e9-e464-46fb-8305-c414f01bc4a2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:42.962439323Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1212,"client":"2","":"downstairs","session_id":"e17538e9-e464-46fb-8305-c414f01bc4a2"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:42.962452722Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1212,"session_id":"e17538e9-e464-46fb-8305-c414f01bc4a2"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:42.962464922Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1212,"session_id":"e17538e9-e464-46fb-8305-c414f01bc4a2"} {"msg":"e44c7de6-aac0-4947-b8f5-5c17ffdf796a is now active with session: e17538e9-e464-46fb-8305-c414f01bc4a2","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:42.962479501Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1212,"session_id":"e17538e9-e464-46fb-8305-c414f01bc4a2"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:42.96249191Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1212,"session_id":"e17538e9-e464-46fb-8305-c414f01bc4a2"} {"msg":"The guest has finished waiting for activation with:8","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:42.96250416Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1212} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 100 len 20480 data: 2 2 2 2 2 02/30 Read block 493 len 36864 03/30 Write block 535 len 12288 data: 2 2 2 04/30 Write block 179 len 20480 data: 2 2 2 2 2 05/30 Read block 230 len 32768 06/30 Read block 435 len 20480 07/30 Write block 64 len 28672 data: 2 2 2 2 2 2 2 08/30 Write block 451 len 40960 data: 2 2 2 2 2 2 2 2 3 3 09/30 Write block 68 len 16384 data: 3 3 3 2 10/30 Write block 465 len 32768 data: 2 2 2 3 3 3 3 3 11/30 Write block 372 len 12288 data: 3 2 2 12/30 Write block 411 len 24576 data: 3 2 2 2 2 2 13/30 Read block 291 len 4096 14/30 Write block 337 len 32768 data: 3 3 3 3 3 3 2 2 15/30 Flush 16/30 Write block 114 len 4096 data: 2 17/30 Write block 74 len 4096 data: 3 18/30 Read block 24 len 40960 19/30 Write block 247 len 36864 data: 3 3 3 2 2 2 2 3 2 20/30 Read block 213 len 16384 21/30 Write block 445 len 32768 data: 2 2 2 2 2 2 3 3 22/30 Write block 513 len 12288 data: 2 2 2 23/30 Read block 68 len 20480 24/30 Write block 271 len 24576 data: 2 2 2 2 2 2 25/30 Read block 208 len 4096 26/30 Read block 541 len 36864 27/30 Read block 255 len 36864 28/30 Read block 162 len 4096 29/30 Read block 415 len 20480 30/30 Read block 133 len 24576 ---------------------------------------------------------------- Crucible gen:8 GIO:true work queues: Upstairs:0 downstairs:15 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1015 Acked Write 1 Done Done Done false 1016 Acked Write 1 Done Done Done false 1017 Acked Read 10 Done Done Done false 1018 Acked Write 9 Done Done Done false 1019 Acked Read 4 Done Done Done false 1020 Acked Write 8 Done Done Done false 1021 Acked Write 3 Done Done Done false 1022 Acked Read 5 Done Done Done false 1023 Acked Write 6 Done Done Done false 1024 Acked Read 1 Done Done Done false 1025 Acked Read 9 Done Done Done false 1026 Acked Read 9 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Read 5 Done Done Done false 1029 Acked Read 6 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 15 15 14 44 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1014 1014 1014 Downstairs last five completed: 1014 1013 1012 1011 1010 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:22:42.984907015Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1212} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 4 4 4 8 8 8 F F F 1 020-039 6 6 6 11 11 11 F F F 2 040-059 6 6 6 11 11 11 F F F 3 060-079 8 8 8 13 13 13 T T T 4 080-099 5 5 5 9 9 9 F F F 5 100-119 8 8 8 13 13 13 T T T 6 120-139 5 5 5 9 9 9 F F F 7 140-159 4 4 4 8 8 8 F F F 8 160-179 8 8 8 13 13 13 F F F 9 180-199 8 8 8 13 13 13 F F F 10 200-219 1 1 1 1 1 1 F F F 11 220-239 6 6 6 11 11 11 F F F 12 240-259 6 6 6 11 11 11 T T T 13 260-279 3 3 3 3 3 3 T T T 14 280-299 7 7 7 12 12 12 F F F 15 300-319 7 7 7 12 12 12 F F F 16 320-339 8 8 8 13 13 13 F F F 17 340-359 8 8 8 13 13 13 F F F 18 360-379 8 8 8 13 13 13 F F F 19 380-399 5 5 5 9 9 9 F F F 20 400-419 8 8 8 13 13 13 F F F 21 420-439 5 5 5 9 9 9 F F F 22 440-459 8 8 8 13 13 13 T T T 23 460-479 8 8 8 13 13 13 F F F 24 480-499 1 1 1 1 1 1 F F F 25 500-519 3 3 3 3 3 3 T T T 26 520-539 8 8 8 13 13 13 F F F 27 540-559 4 4 4 5 5 5 F F F 28 560-579 5 5 5 9 9 9 F F F 29 580-599 6 6 6 10 10 10 F F F Max gen: 8, Max flush: 13 A Difference in extent metadata is expected here On loop 4 Verifying data now No change after verify Loop: 4 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 4 4 4 8 8 8 F F F 1 020-039 6 6 6 11 11 11 F F F 2 040-059 6 6 6 11 11 11 F F F 3 060-079 9 9 9 14 14 14 F F F 4 080-099 5 5 5 9 9 9 F F F 5 100-119 9 9 9 14 14 14 F F F 6 120-139 5 5 5 9 9 9 F F F 7 140-159 4 4 4 8 8 8 F F F 8 160-179 8 8 8 13 13 13 F F F 9 180-199 8 8 8 13 13 13 F F F 10 200-219 1 1 1 1 1 1 F F F 11 220-239 6 6 6 11 11 11 F F F 12 240-259 9 9 9 14 14 14 F F F 13 260-279 9 9 9 14 14 14 F F F 14 280-299 7 7 7 12 12 12 F F F 15 300-319 7 7 7 12 12 12 F F F 16 320-339 8 8 8 13 13 13 F F F 17 340-359 8 8 8 13 13 13 F F F 18 360-379 8 8 8 13 13 13 F F F 19 380-399 5 5 5 9 9 9 F F F 20 400-419 8 8 8 13 13 13 F F F 21 420-439 5 5 5 9 9 9 F F F 22 440-459 9 9 9 14 14 14 F F F 23 460-479 8 8 8 13 13 13 F F F 24 480-499 1 1 1 1 1 1 F F F 25 500-519 9 9 9 14 14 14 F F F 26 520-539 8 8 8 13 13 13 F F F 27 540-559 4 4 4 5 5 5 F F F 28 560-579 5 5 5 9 9 9 F F F 29 580-599 6 6 6 10 10 10 F F F Max gen: 9, Max flush: 14 Begin loop 5 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:47.315755128Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1220,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:47.359946008Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1220,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:47.360587457Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1220} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"0fd98d0d5d909611f8b408ccda2559d35fe117bb\",\n git_commit_timestamp: \"2024-11-08T20:05:44.000000000Z\",\n git_branch: \"mkeeter/drop-skipped-backpressure-guard\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:47.360628135Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1220} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:47.360643494Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1220} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:47.360658143Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1220} {"msg":"Crucible 372a53fc-f9bf-4f3f-bdb8-8828ec67f0fe has session id: e2fd4bb2-91d2-4a84-bef5-a6efd83657ec","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:47.361197177Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1220,"session_id":"e2fd4bb2-91d2-4a84-bef5-a6efd83657ec"} {"msg":"Upstairs opts: Upstairs UUID: 372a53fc-f9bf-4f3f-bdb8-8828ec67f0fe, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:47.361245985Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1220,"session_id":"e2fd4bb2-91d2-4a84-bef5-a6efd83657ec"} {"msg":"Crucible stats registered with UUID: 372a53fc-f9bf-4f3f-bdb8-8828ec67f0fe","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:47.361277223Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1220,"session_id":"e2fd4bb2-91d2-4a84-bef5-a6efd83657ec"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:47.392386072Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1220,"task":"crutest"} {"msg":"The guest has requested activation with gen:10","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:47.392420281Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1220} {"msg":"372a53fc-f9bf-4f3f-bdb8-8828ec67f0fe active request set","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:47.392471868Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1220,"session_id":"e2fd4bb2-91d2-4a84-bef5-a6efd83657ec"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:47.392492467Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1220,"client":"0","":"downstairs","session_id":"e2fd4bb2-91d2-4a84-bef5-a6efd83657ec"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:47.392520466Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1220,"client":"0","":"downstairs","session_id":"e2fd4bb2-91d2-4a84-bef5-a6efd83657ec"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:47.392538555Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1220,"client":"1","":"downstairs","session_id":"e2fd4bb2-91d2-4a84-bef5-a6efd83657ec"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:47.392555804Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1220,"client":"1","":"downstairs","session_id":"e2fd4bb2-91d2-4a84-bef5-a6efd83657ec"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:47.392571954Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1220,"client":"2","":"downstairs","session_id":"e2fd4bb2-91d2-4a84-bef5-a6efd83657ec"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:47.392592762Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1220,"client":"2","":"downstairs","session_id":"e2fd4bb2-91d2-4a84-bef5-a6efd83657ec"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:47.392609692Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1220,"":"io task","client":"2","":"downstairs","session_id":"e2fd4bb2-91d2-4a84-bef5-a6efd83657ec"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:47.392627071Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1220,"":"io task","client":"0","":"downstairs","session_id":"e2fd4bb2-91d2-4a84-bef5-a6efd83657ec"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:47.39264428Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1220,"":"io task","client":"1","":"downstairs","session_id":"e2fd4bb2-91d2-4a84-bef5-a6efd83657ec"} {"msg":"ds_connection connected from Ok(127.0.0.1:55302)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:47.392884008Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1220,"":"io task","client":"0","":"downstairs","session_id":"e2fd4bb2-91d2-4a84-bef5-a6efd83657ec"} {"msg":"ds_connection connected from Ok(127.0.0.1:35737)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:47.392925506Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1220,"":"io task","client":"1","":"downstairs","session_id":"e2fd4bb2-91d2-4a84-bef5-a6efd83657ec"} {"msg":"ds_connection connected from Ok(127.0.0.1:54766)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:47.392956625Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1220,"":"io task","client":"2","":"downstairs","session_id":"e2fd4bb2-91d2-4a84-bef5-a6efd83657ec"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:47.393694999Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1220,"client":"2","":"downstairs","session_id":"e2fd4bb2-91d2-4a84-bef5-a6efd83657ec"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:47.393735726Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1220,"client":"2","":"downstairs","session_id":"e2fd4bb2-91d2-4a84-bef5-a6efd83657ec"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:47.393765975Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1220,"client":"0","":"downstairs","session_id":"e2fd4bb2-91d2-4a84-bef5-a6efd83657ec"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:47.393788854Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1220,"client":"0","":"downstairs","session_id":"e2fd4bb2-91d2-4a84-bef5-a6efd83657ec"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:47.393895709Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1220,"client":"1","":"downstairs","session_id":"e2fd4bb2-91d2-4a84-bef5-a6efd83657ec"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:47.393943087Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1220,"client":"1","":"downstairs","session_id":"e2fd4bb2-91d2-4a84-bef5-a6efd83657ec"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:47.394404004Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1220,"client":"0","":"downstairs","session_id":"e2fd4bb2-91d2-4a84-bef5-a6efd83657ec"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:47.394435262Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1220,"client":"1","":"downstairs","session_id":"e2fd4bb2-91d2-4a84-bef5-a6efd83657ec"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:47.394468161Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1220,"client":"2","":"downstairs","session_id":"e2fd4bb2-91d2-4a84-bef5-a6efd83657ec"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:47.394961087Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1220,"client":"0","":"downstairs","session_id":"e2fd4bb2-91d2-4a84-bef5-a6efd83657ec"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:47.394989135Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1220,"client":"0","":"downstairs","session_id":"e2fd4bb2-91d2-4a84-bef5-a6efd83657ec"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:47.395003655Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1220,"session_id":"e2fd4bb2-91d2-4a84-bef5-a6efd83657ec"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:47.395016854Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1220,"client":"2","":"downstairs","session_id":"e2fd4bb2-91d2-4a84-bef5-a6efd83657ec"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:47.395030243Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1220,"client":"2","":"downstairs","session_id":"e2fd4bb2-91d2-4a84-bef5-a6efd83657ec"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:47.395043523Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1220,"session_id":"e2fd4bb2-91d2-4a84-bef5-a6efd83657ec"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:47.395055832Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1220,"client":"1","":"downstairs","session_id":"e2fd4bb2-91d2-4a84-bef5-a6efd83657ec"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:47.395069352Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1220,"client":"1","":"downstairs","session_id":"e2fd4bb2-91d2-4a84-bef5-a6efd83657ec"} {"msg":"[0]R flush_numbers[0..12]: [8, 11, 11, 14, 9, 14, 9, 8, 13, 13, 1, 11]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:47.395082961Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1220,"":"downstairs","":"downstairs","session_id":"e2fd4bb2-91d2-4a84-bef5-a6efd83657ec"} {"msg":"[0]R generation[0..12]: [4, 6, 6, 9, 5, 9, 5, 4, 8, 8, 1, 6]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:47.39509662Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1220,"":"downstairs","":"downstairs","session_id":"e2fd4bb2-91d2-4a84-bef5-a6efd83657ec"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:47.395110709Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1220,"":"downstairs","":"downstairs","session_id":"e2fd4bb2-91d2-4a84-bef5-a6efd83657ec"} {"msg":"[1]R flush_numbers[0..12]: [8, 11, 11, 14, 9, 14, 9, 8, 13, 13, 1, 11]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:47.395124349Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1220,"":"downstairs","":"downstairs","session_id":"e2fd4bb2-91d2-4a84-bef5-a6efd83657ec"} {"msg":"[1]R generation[0..12]: [4, 6, 6, 9, 5, 9, 5, 4, 8, 8, 1, 6]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:47.395137638Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1220,"":"downstairs","":"downstairs","session_id":"e2fd4bb2-91d2-4a84-bef5-a6efd83657ec"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:47.395151617Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1220,"":"downstairs","":"downstairs","session_id":"e2fd4bb2-91d2-4a84-bef5-a6efd83657ec"} {"msg":"[2]R flush_numbers[0..12]: [8, 11, 11, 14, 9, 14, 9, 8, 13, 13, 1, 11]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:47.395168007Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1220,"":"downstairs","":"downstairs","session_id":"e2fd4bb2-91d2-4a84-bef5-a6efd83657ec"} {"msg":"[2]R generation[0..12]: [4, 6, 6, 9, 5, 9, 5, 4, 8, 8, 1, 6]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:47.395181636Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1220,"":"downstairs","":"downstairs","session_id":"e2fd4bb2-91d2-4a84-bef5-a6efd83657ec"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:47.395195395Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1220,"":"downstairs","":"downstairs","session_id":"e2fd4bb2-91d2-4a84-bef5-a6efd83657ec"} {"msg":"Max found gen is 10","v":0,"name":"crucible","level"Wait for a query_work_queue command to finish before sending IO :30,"time":"2024-11-08T20:22:47.395216554Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1220,"":"downstairs","":"downstairs","session_id":"e2fd4bb2-91d2-4a84-bef5-a6efd83657ec"} {"msg":"Generation requested: 10 >= found:10","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:47.395230244Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1220,"":"downstairs","":"downstairs","session_id":"e2fd4bb2-91d2-4a84-bef5-a6efd83657ec"} {"msg":"Next flush: 15","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:47.395243193Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1220,"":"downstairs","":"downstairs","session_id":"e2fd4bb2-91d2-4a84-bef5-a6efd83657ec"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:47.395255982Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1220,"":"downstairs","":"downstairs","session_id":"e2fd4bb2-91d2-4a84-bef5-a6efd83657ec"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:47.395269372Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1220,"session_id":"e2fd4bb2-91d2-4a84-bef5-a6efd83657ec"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:47.395281271Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1220,"client":"0","":"downstairs","session_id":"e2fd4bb2-91d2-4a84-bef5-a6efd83657ec"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:47.39529452Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1220,"client":"1","":"downstairs","session_id":"e2fd4bb2-91d2-4a84-bef5-a6efd83657ec"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:47.39530757Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1220,"client":"2","":"downstairs","session_id":"e2fd4bb2-91d2-4a84-bef5-a6efd83657ec"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:47.395320749Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1220,"session_id":"e2fd4bb2-91d2-4a84-bef5-a6efd83657ec"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:47.395332908Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1220,"session_id":"e2fd4bb2-91d2-4a84-bef5-a6efd83657ec"} {"msg":"372a53fc-f9bf-4f3f-bdb8-8828ec67f0fe is now active with session: e2fd4bb2-91d2-4a84-bef5-a6efd83657ec","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:47.395345568Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1220,"session_id":"e2fd4bb2-91d2-4a84-bef5-a6efd83657ec"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:47.395357527Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1220,"session_id":"e2fd4bb2-91d2-4a84-bef5-a6efd83657ec"} {"msg":"Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 The guest has finished waiting for activation with:10","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:47.395375626Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1220} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 252 len 8192 data: 3 3 02/30 Read block 336 len 32768 03/30 Write block 434 len 24576 data: 2 2 2 2 2 2 04/30 Read block 514 len 40960 05/30 Write block 414 len 40960 data: 3 3 3 2 2 2 2 3 2 3 06/30 Read block 470 len 4096 07/30 Read block 1 len 20480 08/30 Read block 323 len 4096 09/30 Read block 571 len 36864 10/30 Read block 453 len 40960 11/30 Write block 247 len 24576 data: 4 4 4 3 3 4 12/30 Write block 332 len 20480 data: 2 2 3 3 3 13/30 Read block 334 len 32768 14/30 Write block 355 len 20480 data: 4 4 4 4 4 15/30 Read block 316 len 12288 16/30 Write block 168 len 12288 data: 3 3 3 17/30 Read block 199 len 24576 18/30 Write block 475 len 8192 data: 3 3 19/30 Write block 107 len 8192 data: 2 2 20/30 Write block 85 len 16384 data: 3 2 2 2 21/30 Read block 388 len 28672 22/30 Write block 418 len 12288 data: 3 3 3 23/30 Flush 24/30 Write block 494 len 8192 data: 2 2 25/30 Write block 542 len 36864 data: 2 2 2 2 2 2 2 2 2 26/30 Write block 508 len 24576 data: 2 2 2 2 2 3 27/30 Write block 493 len 24576 data: 2 3 3 2 2 2 28/30 Write block 230 len 16384 data: 2 2 2 2 29/30 Read block 186 len 28672 30/30 Write block 416 len 20480 data: 4 3 4 4 4 ---------------------------------------------------------------- Crucible gen:10 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 2 Done Done Done false 1024 Acked Write 9 Done Done Done false 1025 Acked Write 6 Done Done Done false 1026 Acked Write 6 Sent Done Done false 1027 Acked Write 4 Sent Done Done false 1028 Acked Read 7 Sent Done Done false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 4 1 1 6 Done 3 6 6 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:22:47.41780395Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1220} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 4 4 4 8 8 8 F F F 1 020-039 6 6 6 11 11 11 F F F 2 040-059 6 6 6 11 11 11 F F F 3 060-079 9 9 9 14 14 14 F F F 4 080-099 10 10 10 15 15 15 F F F 5 100-119 10 10 10 15 15 15 F F F 6 120-139 5 5 5 9 9 9 F F F 7 140-159 4 4 4 8 8 8 F F F 8 160-179 10 10 10 15 15 15 F F F 9 180-199 8 8 8 13 13 13 F F F 10 200-219 1 1 1 1 1 1 F F F 11 220-239 6 6 6 11 11 11 T T T 12 240-259 10 10 10 15 15 15 F F F 13 260-279 9 9 9 14 14 14 F F F 14 280-299 7 7 7 12 12 12 F F F 15 300-319 7 7 7 12 12 12 F F F 16 320-339 10 10 10 15 15 15 F F F 17 340-359 10 10 10 15 15 15 F F F 18 360-379 8 8 8 13 13 13 F F F 19 380-399 5 5 5 9 9 9 F F F 20 400-419 10 10 10 15 15 15 T T T 21 420-439 10 10 10 15 15 15 T T T 22 440-459 9 9 9 14 14 14 F F F 23 460-479 10 10 10 15 15 15 F F F 24 480-499 1 1 1 1 1 1 T T T 25 500-519 9 9 9 14 14 14 T T T 26 520-539 8 8 8 13 13 13 F F F 27 540-559 4 4 4 5 5 5 T T T 28 560-579 5 5 5 9 9 9 F F F 29 580-599 6 6 6 10 10 10 F F F Max gen: 10, Max flush: 15 A Difference in extent metadata is expected here On loop 5 Verifying data now No change after verify Loop: 5 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 4 4 4 8 8 8 F F F 1 020-039 6 6 6 11 11 11 F F F 2 040-059 6 6 6 11 11 11 F F F 3 060-079 9 9 9 14 14 14 F F F 4 080-099 10 10 10 15 15 15 F F F 5 100-119 10 10 10 15 15 15 F F F 6 120-139 5 5 5 9 9 9 F F F 7 140-159 4 4 4 8 8 8 F F F 8 160-179 10 10 10 15 15 15 F F F 9 180-199 8 8 8 13 13 13 F F F 10 200-219 1 1 1 1 1 1 F F F 11 220-239 11 11 11 16 16 16 F F F 12 240-259 10 10 10 15 15 15 F F F 13 260-279 9 9 9 14 14 14 F F F 14 280-299 7 7 7 12 12 12 F F F 15 300-319 7 7 7 12 12 12 F F F 16 320-339 10 10 10 15 15 15 F F F 17 340-359 10 10 10 15 15 15 F F F 18 360-379 8 8 8 13 13 13 F F F 19 380-399 5 5 5 9 9 9 F F F 20 400-419 11 11 11 16 16 16 F F F 21 420-439 11 11 11 16 16 16 F F F 22 440-459 9 9 9 14 14 14 F F F 23 460-479 10 10 10 15 15 15 F F F 24 480-499 11 11 11 16 16 16 F F F 25 500-519 11 11 11 16 16 16 F F F 26 520-539 8 8 8 13 13 13 F F F 27 540-559 11 11 11 16 16 16 F F F 28 560-579 5 5 5 9 9 9 F F F 29 580-599 6 6 6 10 10 10 F F F Max gen: 11, Max flush: 16 Begin loop 6 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:51.743096385Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1228,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:51.786862895Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1228,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:51.787568091Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1228} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"0fd98d0d5d909611f8b408ccda2559d35fe117bb\",\n git_commit_timestamp: \"2024-11-08T20:05:44.000000000Z\",\n git_branch: \"mkeeter/drop-skipped-backpressure-guard\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:51.787604949Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1228} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:51.787620318Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1228} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:51.787634498Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1228} {"msg":"Crucible 61b08f14-ac99-433e-842e-b83c10d14fef has session id: 461c74f8-121c-418d-a713-ee1a069bb117","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:51.788157562Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1228,"session_id":"461c74f8-121c-418d-a713-ee1a069bb117"} {"msg":"Upstairs opts: Upstairs UUID: 61b08f14-ac99-433e-842e-b83c10d14fef, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:51.78820772Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1228,"session_id":"461c74f8-121c-418d-a713-ee1a069bb117"} {"msg":"Crucible stats registered with UUID: 61b08f14-ac99-433e-842e-b83c10d14fef","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:51.788234928Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1228,"session_id":"461c74f8-121c-418d-a713-ee1a069bb117"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:51.819577976Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1228,"task":"crutest"} {"msg":"The guest has requested activation with gen:12","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:51.819612945Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1228} {"msg":"61b08f14-ac99-433e-842e-b83c10d14fef active request set","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:51.819648163Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1228,"session_id":"461c74f8-121c-418d-a713-ee1a069bb117"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:51.819665372Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1228,"client":"0","":"downstairs","session_id":"461c74f8-121c-418d-a713-ee1a069bb117"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:51.819694741Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1228,"client":"0","":"downstairs","session_id":"461c74f8-121c-418d-a713-ee1a069bb117"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:51.81971246Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1228,"client":"1","":"downstairs","session_id":"461c74f8-121c-418d-a713-ee1a069bb117"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:51.819729789Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1228,"client":"1","":"downstairs","session_id":"461c74f8-121c-418d-a713-ee1a069bb117"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:51.819746248Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1228,"client":"2","":"downstairs","session_id":"461c74f8-121c-418d-a713-ee1a069bb117"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:51.819763207Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1228,"client":"2","":"downstairs","session_id":"461c74f8-121c-418d-a713-ee1a069bb117"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:51.819779357Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1228,"":"io task","client":"2","":"downstairs","session_id":"461c74f8-121c-418d-a713-ee1a069bb117"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:51.819796386Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1228,"":"io task","client":"0","":"downstairs","session_id":"461c74f8-121c-418d-a713-ee1a069bb117"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:51.819823105Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1228,"":"io task","client":"1","":"downstairs","session_id":"461c74f8-121c-418d-a713-ee1a069bb117"} {"msg":"ds_connection connected from Ok(127.0.0.1:41341)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:51.819950168Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1228,"":"io task","client":"2","":"downstairs","session_id":"461c74f8-121c-418d-a713-ee1a069bb117"} {"msg":"ds_connection connected from Ok(127.0.0.1:40325)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:51.820088292Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1228,"":"io task","client":"1","":"downstairs","session_id":"461c74f8-121c-418d-a713-ee1a069bb117"} {"msg":"ds_connection connected from Ok(127.0.0.1:61544)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:51.820137089Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1228,"":"io task","client":"0","":"downstairs","session_id":"461c74f8-121c-418d-a713-ee1a069bb117"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:51.820779508Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1228,"client":"0","":"downstairs","session_id":"461c74f8-121c-418d-a713-ee1a069bb117"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:51.820808876Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1228,"client":"0","":"downstairs","session_id":"461c74f8-121c-418d-a713-ee1a069bb117"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:51.820825516Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1228,"client":"2","":"downstairs","session_id":"461c74f8-121c-418d-a713-ee1a069bb117"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:51.820839645Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1228,"client":"2","":"downstairs","session_id":"461c74f8-121c-418d-a713-ee1a069bb117"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:51.820855074Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1228,"client":"1","":"downstairs","session_id":"461c74f8-121c-418d-a713-ee1a069bb117"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:51.820880593Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1228,"client":"1","":"downstairs","session_id":"461c74f8-121c-418d-a713-ee1a069bb117"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:51.821500983Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1228,"client":"0","":"downstairs","session_id":"461c74f8-121c-418d-a713-ee1a069bb117"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:51.821531171Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1228,"client":"1","":"downstairs","session_id":"461c74f8-121c-418d-a713-ee1a069bb117"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:51.82154771Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1228,"client":"2","":"downstairs","session_id":"461c74f8-121c-418d-a713-ee1a069bb117"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:51.822099893Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1228,"client":"0","":"downstairs","session_id":"461c74f8-121c-418d-a713-ee1a069bb117"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:51.822129312Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1228,"client":"0","":"downstairs","session_id":"461c74f8-121c-418d-a713-ee1a069bb117"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:51.822144671Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1228,"session_id":"461c74f8-121c-418d-a713-ee1a069bb117"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:51.82216056Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1228,"client":"1","":"downstairs","session_id":"461c74f8-121c-418d-a713-ee1a069bb117"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:51.82217484Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1228,"client":"1","":"downstairs","session_id":"461c74f8-121c-418d-a713-ee1a069bb117"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:51.822188139Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1228,"session_id":"461c74f8-121c-418d-a713-ee1a069bb117"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:51.822200499Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1228,"client":"2","":"downstairs","session_id":"461c74f8-121c-418d-a713-ee1a069bb117"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:51.822213738Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1228,"client":"2","":"downstairs","session_id":"461c74f8-121c-418d-a713-ee1a069bb117"} {"msg":"[0]R flush_numbers[0..12]: [8, 11, 11, 14, 15, 15, 9, 8, 15, 13, 1, 16]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:51.822227497Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1228,"":"downstairs","":"downstairs","session_id":"461c74f8-121c-418d-a713-ee1a069bb117"} {"msg":"[0]R generation[0..12]: [4, 6, 6, 9, 10, 10, 5, 4, 10, 8, 1, 11]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:51.822241146Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1228,"":"downstairs","":"downstairs","session_id":"461c74f8-121c-418d-a713-ee1a069bb117"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:51.822255236Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1228,"":"downstairs","":"downstairs","session_id":"461c74f8-121c-418d-a713-ee1a069bb117"} {"msg":"[1]R flush_numbers[0..12]: [8, 11, 11, 14, 15, 15, 9, 8, 15, 13, 1, 16]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:51.822268805Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1228,"":"downstairs","":"downstairs","session_id":"461c74f8-121c-418d-a713-ee1a069bb117"} {"msg":"[1]R generation[0..12]: [4, 6, 6, 9, 10, 10, 5, 4, 10, 8, 1, 11]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:51.822282415Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1228,"":"downstairs","":"downstairs","session_id":"461c74f8-121c-418d-a713-ee1a069bb117"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:51.822296514Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1228,"":"downstairs","":"downstairs","session_id":"461c74f8-121c-418d-a713-ee1a069bb117"} {"msg":"[2]R flush_numbers[0..12]: [8, 11, 11, 14, 15, 15, 9, 8, 15, 13, 1, 16]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:51.822313323Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1228,"":"downstairs","":"downstairs","session_id":"461c74f8-121c-418d-a713-ee1a069bb117"} {"msg":"[2]R generation[0..12]: [4, 6, 6, 9, 10, 10, 5, 4, 10, 8, 1, 11]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:51.822338762Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1228,"":"downstairs","":"downstairs","session_id":"461c74f8-121c-418d-a713-ee1a069bb117"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:51.82236762Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1228,"":"downstairs","":"downstairs","session_id":"461c74f8-121c-418d-a713-ee1a069bb117"} {"msg":"Max found gen is 12","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:51.822387289Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1228,"":"downstairs","":"downstairs","session_id":"461c74f8-121c-418d-a713-ee1a069bb117"} {"msg":"Wait for a query_work_queue command to finish before sending IO Generation requested: 12 >= found:12","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:51.822424937Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1228,"":"downstairs","":"downstairs","session_id":"461c74f8-121c-418d-a713-ee1a069bb117"} {"msg":"Next flush: 17","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:51.822454356Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1228,"":"downstairs","":"downstairs","session_id":"461c74f8-121c-418d-a713-ee1a069bb117"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:51.822475355Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1228,"":"downstairs","":"downstairs","session_id":"461c74f8-121c-418d-a713-ee1a069bb117"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:51.822489264Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1228,"session_id":"461c74f8-121c-418d-a713-ee1a069bb117"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:51.822503853Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1228,"client":"0","":"downstairs","session_id":"461c74f8-121c-418d-a713-ee1a069bb117"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:51.822528282Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1228,"client":"1","":"downstairs","session_id":"461c74f8-121c-418d-a713-ee1a069bb117"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:51.822554901Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1228,"client":"2","":"downstairs","session_id":"461c74f8-121c-418d-a713-ee1a069bb117"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:51.82257728Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1228,"session_id":"461c74f8-121c-418d-a713-ee1a069bb117"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:51.822590679Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1228,"session_id":"461c74f8-121c-418d-a713-ee1a069bb117"} {"msg":"61b08f14-ac99-433e-842e-b83c10d14fef is now active with session: 461c74f8-121c-418d-a713-ee1a069bb117","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:51.822611128Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1228,"session_id":"461c74f8-121c-418d-a713-ee1a069bb117"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:51.822634027Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1228,"session_id":"461c74f8-121c-418d-a713-ee1a069bb117"} {"msg":"The guest has finished waiting for activation with:12","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:51.822658876Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1228} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 346 len 40960 data: 2 2 2 3 3 3 4 4 4 5 02/30 Write block 477 len 32768 data: 3 3 3 2 2 2 2 2 03/30 Write block 435 len 32768 data: 3 3 3 3 3 2 2 2 04/30 Write block 24 len 16384 data: 2 2 2 2 05/30 Read block 449 len 4096 06/30 Write block 533 len 24576 data: 2 2 3 3 3 2 07/30 Flush 08/30 Write block 363 len 24576 data: 3 2 2 3 4 4 09/30 Write block 285 len 24576 data: 3 4 4 5 5 4 10/30 Write block 275 len 24576 data: 3 3 2 2 2 2 11/30 Write block 432 len 32768 data: 3 3 3 4 4 4 4 4 12/30 Write block 429 len 12288 data: 3 3 3 13/30 Write block 306 len 32768 data: 2 2 2 2 3 3 3 3 14/30 Write block 308 len 24576 data: 3 3 4 4 4 4 15/30 Read block 218 len 8192 16/30 Read block 322 len 36864 17/30 Write block 297 len 32768 data: 2 2 2 2 2 2 2 2 18/30 Write block 494 len 36864 data: 4 4 3 3 3 2 2 2 2 19/30 Write block 335 len 12288 data: 4 4 4 20/30 Read block 505 len 32768 21/30 Read block 366 len 40960 22/30 Write block 466 len 20480 data: 3 3 4 4 4 23/30 Write block 338 len 36864 data: 4 4 4 4 4 3 3 2 3 24/30 Read block 503 len 12288 25/30 Read block 141 len 8192 26/30 Flush 27/30 Write block 584 len 36864 data: 4 4 4 4 3 3 3 2 2 28/30 Write block 344 len 16384 data: 4 3 4 3 29/30 Read block 145 len 40960 30/30 Write block 56 len 36864 data: 2 2 2 2 2 2 2 2 3 ---------------------------------------------------------------- Crucible gen:12 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 9 Done Sent Done false 1027 Acked Write 4 Done Sent Done false 1028 Acked Read 10 Done Sent Done false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 4 1 6 Done 3 0 3 6 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:22:51.848689644Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1228} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 4 4 4 8 8 8 F F F 1 020-039 12 12 12 17 17 17 F F F 2 040-059 6 6 6 11 11 11 T F T <--- 3 060-079 9 9 9 14 14 14 T F T <--- 4 080-099 10 10 10 15 15 15 F F F 5 100-119 10 10 10 15 15 15 F F F 6 120-139 5 5 5 9 9 9 F F F 7 140-159 4 4 4 8 8 8 F F F 8 160-179 10 10 10 15 15 15 F F F 9 180-199 8 8 8 13 13 13 F F F 10 200-219 1 1 1 1 1 1 F F F 11 220-239 11 11 11 16 16 16 F F F 12 240-259 10 10 10 15 15 15 F F F 13 260-279 12 12 12 18 18 18 F F F 14 280-299 12 12 12 18 18 18 F F F 15 300-319 12 12 12 18 18 18 F F F 16 320-339 12 12 12 18 18 18 F F F 17 340-359 12 12 12 18 18 18 T T T 18 360-379 12 12 12 18 18 18 F F F 19 380-399 5 5 5 9 9 9 F F F 20 400-419 11 11 11 16 16 16 F F F 21 420-439 12 12 12 18 18 18 F F F 22 440-459 12 12 12 17 17 17 F F F 23 460-479 12 12 12 18 18 18 F F F 24 480-499 12 12 12 18 18 18 F F F 25 500-519 12 12 12 18 18 18 F F F 26 520-539 12 12 12 17 17 17 F F F 27 540-559 11 11 11 16 16 16 F F F 28 560-579 5 5 5 9 9 9 F F F 29 580-599 6 6 6 10 10 10 T T T Max gen: 12, Max flush: 18 Error: Difference in extent metadata found! A Difference in extent metadata is expected here On loop 6 Verifying data now No change after verify Loop: 6 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 4 4 4 8 8 8 F F F 1 020-039 12 12 12 17 17 17 F F F 2 040-059 13 13 13 19 19 19 F F F 3 060-079 13 13 13 19 19 19 F F F 4 080-099 10 10 10 15 15 15 F F F 5 100-119 10 10 10 15 15 15 F F F 6 120-139 5 5 5 9 9 9 F F F 7 140-159 4 4 4 8 8 8 F F F 8 160-179 10 10 10 15 15 15 F F F 9 180-199 8 8 8 13 13 13 F F F 10 200-219 1 1 1 1 1 1 F F F 11 220-239 11 11 11 16 16 16 F F F 12 240-259 10 10 10 15 15 15 F F F 13 260-279 12 12 12 18 18 18 F F F 14 280-299 12 12 12 18 18 18 F F F 15 300-319 12 12 12 18 18 18 F F F 16 320-339 12 12 12 18 18 18 F F F 17 340-359 13 13 13 19 19 19 F F F 18 360-379 12 12 12 18 18 18 F F F 19 380-399 5 5 5 9 9 9 F F F 20 400-419 11 11 11 16 16 16 F F F 21 420-439 12 12 12 18 18 18 F F F 22 440-459 12 12 12 17 17 17 F F F 23 460-479 12 12 12 18 18 18 F F F 24 480-499 12 12 12 18 18 18 F F F 25 500-519 12 12 12 18 18 18 F F F 26 520-539 12 12 12 17 17 17 F F F 27 540-559 11 11 11 16 16 16 F F F 28 560-579 5 5 5 9 9 9 F F F 29 580-599 13 13 13 19 19 19 F F F Max gen: 13, Max flush: 19 Begin loop 7 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:56.150919055Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1236,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:56.195400971Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1236,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:56.196009291Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1236} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"0fd98d0d5d909611f8b408ccda2559d35fe117bb\",\n git_commit_timestamp: \"2024-11-08T20:05:44.000000000Z\",\n git_branch: \"mkeeter/drop-skipped-backpressure-guard\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:56.196051369Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1236} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:56.196067198Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1236} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:56.196082058Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1236} {"msg":"Crucible e57ea26d-33b4-4aac-b706-577e6a93113e has session id: 21fca65f-d507-4c27-98d5-fa1dbc9e16b8","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:56.196556515Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1236,"session_id":"21fca65f-d507-4c27-98d5-fa1dbc9e16b8"} {"msg":"Upstairs opts: Upstairs UUID: e57ea26d-33b4-4aac-b706-577e6a93113e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:56.196602362Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1236,"session_id":"21fca65f-d507-4c27-98d5-fa1dbc9e16b8"} {"msg":"Crucible stats registered with UUID: e57ea26d-33b4-4aac-b706-577e6a93113e","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:56.196627311Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1236,"session_id":"21fca65f-d507-4c27-98d5-fa1dbc9e16b8"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:56.228069345Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1236,"task":"crutest"} {"msg":"The guest has requested activation with gen:14","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:56.228102863Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1236} {"msg":"e57ea26d-33b4-4aac-b706-577e6a93113e active request set","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:56.228139421Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1236,"session_id":"21fca65f-d507-4c27-98d5-fa1dbc9e16b8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:56.22815618Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1236,"client":"0","":"downstairs","session_id":"21fca65f-d507-4c27-98d5-fa1dbc9e16b8"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:56.228184079Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1236,"client":"0","":"downstairs","session_id":"21fca65f-d507-4c27-98d5-fa1dbc9e16b8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:56.228202098Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1236,"client":"1","":"downstairs","session_id":"21fca65f-d507-4c27-98d5-fa1dbc9e16b8"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:56.228219337Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1236,"client":"1","":"downstairs","session_id":"21fca65f-d507-4c27-98d5-fa1dbc9e16b8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:56.228236196Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1236,"client":"2","":"downstairs","session_id":"21fca65f-d507-4c27-98d5-fa1dbc9e16b8"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:56.228266445Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1236,"client":"2","":"downstairs","session_id":"21fca65f-d507-4c27-98d5-fa1dbc9e16b8"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:56.228283944Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1236,"":"io task","client":"2","":"downstairs","session_id":"21fca65f-d507-4c27-98d5-fa1dbc9e16b8"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:56.228301153Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1236,"":"io task","client":"0","":"downstairs","session_id":"21fca65f-d507-4c27-98d5-fa1dbc9e16b8"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:56.228318322Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1236,"":"io task","client":"1","":"downstairs","session_id":"21fca65f-d507-4c27-98d5-fa1dbc9e16b8"} {"msg":"ds_connection connected from Ok(127.0.0.1:51295)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:56.228612308Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1236,"":"io task","client":"2","":"downstairs","session_id":"21fca65f-d507-4c27-98d5-fa1dbc9e16b8"} {"msg":"ds_connection connected from Ok(127.0.0.1:37740)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:56.228664055Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1236,"":"io task","client":"1","":"downstairs","session_id":"21fca65f-d507-4c27-98d5-fa1dbc9e16b8"} {"msg":"ds_connection connected from Ok(127.0.0.1:56125)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:56.228700774Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1236,"":"io task","client":"0","":"downstairs","session_id":"21fca65f-d507-4c27-98d5-fa1dbc9e16b8"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:56.229299874Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1236,"client":"0","":"downstairs","session_id":"21fca65f-d507-4c27-98d5-fa1dbc9e16b8"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:56.229338483Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1236,"client":"0","":"downstairs","session_id":"21fca65f-d507-4c27-98d5-fa1dbc9e16b8"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:56.229368251Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1236,"client":"1","":"downstairs","session_id":"21fca65f-d507-4c27-98d5-fa1dbc9e16b8"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:56.22939732Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1236,"client":"1","":"downstairs","session_id":"21fca65f-d507-4c27-98d5-fa1dbc9e16b8"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:56.229425878Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1236,"client":"2","":"downstairs","session_id":"21fca65f-d507-4c27-98d5-fa1dbc9e16b8"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:56.229449177Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1236,"client":"2","":"downstairs","session_id":"21fca65f-d507-4c27-98d5-fa1dbc9e16b8"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:56.229928174Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1236,"client":"2","":"downstairs","session_id":"21fca65f-d507-4c27-98d5-fa1dbc9e16b8"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:56.230155223Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1236,"client":"1","":"downstairs","session_id":"21fca65f-d507-4c27-98d5-fa1dbc9e16b8"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:56.230243368Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1236,"client":"0","":"downstairs","session_id":"21fca65f-d507-4c27-98d5-fa1dbc9e16b8"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:56.230529444Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1236,"client":"2","":"downstairs","session_id":"21fca65f-d507-4c27-98d5-fa1dbc9e16b8"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:56.230569452Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1236,"client":"2","":"downstairs","session_id":"21fca65f-d507-4c27-98d5-fa1dbc9e16b8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:56.230598331Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1236,"session_id":"21fca65f-d507-4c27-98d5-fa1dbc9e16b8"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:56.230690456Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1236,"client":"1","":"downstairs","session_id":"21fca65f-d507-4c27-98d5-fa1dbc9e16b8"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:56.230712015Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1236,"client":"1","":"downstairs","session_id":"21fca65f-d507-4c27-98d5-fa1dbc9e16b8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:56.230725855Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1236,"session_id":"21fca65f-d507-4c27-98d5-fa1dbc9e16b8"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:56.230763843Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1236,"client":"0","":"downstairs","session_id":"21fca65f-d507-4c27-98d5-fa1dbc9e16b8"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:56.230784732Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1236,"client":"0","":"downstairs","session_id":"21fca65f-d507-4c27-98d5-fa1dbc9e16b8"} {"msg":"[0]R flush_numbers[0..12]: [8, 17, 19, 19, 15, 15, 9, 8, 15, 13, 1, 16]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:56.230803351Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1236,"":"downstairs","":"downstairs","session_id":"21fca65f-d507-4c27-98d5-fa1dbc9e16b8"} {"msg":"[0]R generation[0..12]: [4, 12, 13, 13, 10, 10, 5, 4, 10, 8, 1, 11]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:56.23081736Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1236,"":"downstairs","":"downstairs","session_id":"21fca65f-d507-4c27-98d5-fa1dbc9e16b8"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:56.23083159Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1236,"":"downstairs","":"downstairs","session_id":"21fca65f-d507-4c27-98d5-fa1dbc9e16b8"} {"msg":"[1]R flush_numbers[0..12]: [8, 17, 19, 19, 15, 15, 9, 8, 15, 13, 1, 16]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:56.230845319Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1236,"":"downstairs","":"downstairs","session_id":"21fca65f-d507-4c27-98d5-fa1dbc9e16b8"} {"msg":"[1]R generation[0..12]: [4, 12, 13, 13, 10, 10, 5, 4, 10, 8, 1, 11]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:56.230858878Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1236,"":"downstairs","":"downstairs","session_id":"21fca65f-d507-4c27-98d5-fa1dbc9e16b8"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:56.230872708Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1236,"":"downstairs","":"downstairs","session_id":"21fca65f-d507-4c27-98d5-fa1dbc9e16b8"} {"msg":"[2]R flush_numbers[0..12]: [8, 17, 19, 19, 15, 15, 9, 8, 15, 13, 1, 16]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:56.230886037Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1236,"":"downstairs","":"downstairs","session_id":"21fca65f-d507-4c27-98d5-fa1dbc9e16b8"} {"msg":"[2]R generation[0..12]: [4, 12, 13, 13, 10, 10, 5, 4, 10, 8, 1, 11]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:56.230899516Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1236,"":"downstairs","":"downstairs","session_id":"21fca65f-d507-4c27-98d5-fa1dbc9e16b8"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:56.230913586Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1236,"":"downstairs","":"downstairs","session_id":"21fca65f-d507-4c27-98d5-fa1dbc9e16b8"} {"msg":"Max found gen is 14","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:56.230926425Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1236,"":"downstairs","":"downstairs","session_id":"21fca65f-d507-4c27-98d5-fa1dbc9e16b8"} {"msg":"Generation requested: 14 >= found:14","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:56.230939564Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1236,"":"downstairs","":"downstairs","session_id":"21fca65f-d507-4c27-98d5-fa1dbc9e16b8"} {"msg":"Next flush: 20","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:56.230952673Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1236,"":"downstairs","":"downstairs","session_id":"21fca65f-d507-4c27-98d5-fa1dbc9e16b8"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:56.230965753Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1236,"":"downstairs","":"downstairs","session_id":"21fca65f-d507-4c27-98d5-fa1dbc9e16b8"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:56.230978772Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1236,"session_id":"21fca65f-d507-4c27-98d5-fa1dbc9e16b8"} Wait for a query_work_queue command to finish before sending IO {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:56.231001351Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1236,"client":"0","":"downstairs","session_id":"21fca65f-d507-4c27-98d5-fa1dbc9e16b8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:56.231015321Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1236,"client":"1","":"downstairs","session_id":"21fca65f-d507-4c27-98d5-fa1dbc9e16b8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:56.23102839Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1236,"client":"2","":"downstairs","session_id":"21fca65f-d507-4c27-98d5-fa1dbc9e16b8"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:56.231041779Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1236,"session_id":"21fca65f-d507-4c27-98d5-fa1dbc9e16b8"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:56.231054268Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1236,"session_id":"21fca65f-d507-4c27-98d5-fa1dbc9e16b8"} {"msg":"e57ea26d-33b4-4aac-b706-577e6a93113e is now active with session: 21fca65f-d507-4c27-98d5-fa1dbc9e16b8","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:56.231067188Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1236,"session_id":"21fca65f-d507-4c27-98d5-fa1dbc9e16b8"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:56.231079017Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1236,"session_id":"21fca65f-d507-4c27-98d5-fa1dbc9e16b8"} {"msg":"The guest has finished waiting for activation with:14","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:22:56.231091227Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1236} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 523 len 24576 data: 2 2 2 2 3 3 02/30 Read block 506 len 28672 03/30 Read block 45 len 20480 04/30 Write block 86 len 32768 data: 3 3 3 2 2 2 2 2 05/30 Read block 52 len 20480 06/30 Write block 593 len 4096 data: 2 07/30 Write block 143 len 36864 data: 2 2 2 2 2 2 2 2 2 08/30 Read block 155 len 36864 09/30 Read block 51 len 24576 10/30 Write block 99 len 36864 data: 2 3 3 3 3 3 2 2 3 11/30 Write block 264 len 12288 data: 2 2 2 12/30 Read block 293 len 32768 13/30 Write block 134 len 16384 data: 3 3 2 3 14/30 Flush 15/30 Write block 486 len 36864 data: 2 2 2 2 2 2 2 3 5 16/30 Read block 200 len 32768 17/30 Flush 18/30 Write block 21 len 28672 data: 2 2 2 3 3 3 3 19/30 Write block 265 len 4096 data: 3 20/30 Read block 203 len 28672 21/30 Write block 575 len 4096 data: 2 22/30 Flush 23/30 Write block 357 len 24576 data: 5 5 5 4 4 4 24/30 Write block 358 len 16384 data: 6 6 5 5 25/30 Write block 116 len 36864 data: 2 2 2 2 2 2 2 2 2 26/30 Flush 27/30 Write block 577 len 8192 data: 2 2 28/30 Read block 152 len 28672 29/30 Write block 356 len 40960 data: 5 6 7 7 6 6 5 4 3 3 30/30 Read block 244 len 36864 ---------------------------------------------------------------- Crucible gen:14 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 2 Done Done Done false 1027 Acked Read 7 Done Done Done false 1028 Acked Write 10 Done Done Sent false 1029 Acked Read 9 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 2 2 Done 4 4 2 10 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:22:56.260159796Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1236} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 4 4 4 8 8 8 F F F 1 020-039 14 14 14 22 22 22 F F F 2 040-059 13 13 13 19 19 19 F F F 3 060-079 13 13 13 19 19 19 F F F 4 080-099 14 14 14 20 20 20 F F F 5 100-119 14 14 14 23 23 23 F F F 6 120-139 14 14 14 23 23 23 F F F 7 140-159 14 14 14 20 20 20 F F F 8 160-179 10 10 10 15 15 15 F F F 9 180-199 8 8 8 13 13 13 F F F 10 200-219 1 1 1 1 1 1 F F F 11 220-239 11 11 11 16 16 16 F F F 12 240-259 10 10 10 15 15 15 F F F 13 260-279 14 14 14 22 22 22 F F F 14 280-299 12 12 12 18 18 18 F F F 15 300-319 12 12 12 18 18 18 F F F 16 320-339 12 12 12 18 18 18 F F F 17 340-359 14 14 14 23 23 23 T T T 18 360-379 14 14 14 23 23 23 T T T 19 380-399 5 5 5 9 9 9 F F F 20 400-419 11 11 11 16 16 16 F F F 21 420-439 12 12 12 18 18 18 F F F 22 440-459 12 12 12 17 17 17 F F F 23 460-479 12 12 12 18 18 18 F F F 24 480-499 14 14 14 21 21 21 F F F 25 500-519 12 12 12 18 18 18 F F F 26 520-539 14 14 14 20 20 20 F F F 27 540-559 11 11 11 16 16 16 F F F 28 560-579 14 14 14 22 22 22 T T T 29 580-599 14 14 14 20 20 20 F F F Max gen: 14, Max flush: 23 A Difference in extent metadata is expected here On loop 7 Verifying data now No change after verify Loop: 7 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 4 4 4 8 8 8 F F F 1 020-039 14 14 14 22 22 22 F F F 2 040-059 13 13 13 19 19 19 F F F 3 060-079 13 13 13 19 19 19 F F F 4 080-099 14 14 14 20 20 20 F F F 5 100-119 14 14 14 23 23 23 F F F 6 120-139 14 14 14 23 23 23 F F F 7 140-159 14 14 14 20 20 20 F F F 8 160-179 10 10 10 15 15 15 F F F 9 180-199 8 8 8 13 13 13 F F F 10 200-219 1 1 1 1 1 1 F F F 11 220-239 11 11 11 16 16 16 F F F 12 240-259 10 10 10 15 15 15 F F F 13 260-279 14 14 14 22 22 22 F F F 14 280-299 12 12 12 18 18 18 F F F 15 300-319 12 12 12 18 18 18 F F F 16 320-339 12 12 12 18 18 18 F F F 17 340-359 15 15 15 24 24 24 F F F 18 360-379 15 15 15 24 24 24 F F F 19 380-399 5 5 5 9 9 9 F F F 20 400-419 11 11 11 16 16 16 F F F 21 420-439 12 12 12 18 18 18 F F F 22 440-459 12 12 12 17 17 17 F F F 23 460-479 12 12 12 18 18 18 F F F 24 480-499 14 14 14 21 21 21 F F F 25 500-519 12 12 12 18 18 18 F F F 26 520-539 14 14 14 20 20 20 F F F 27 540-559 11 11 11 16 16 16 F F F 28 560-579 15 15 15 24 24 24 F F F 29 580-599 14 14 14 20 20 20 F F F Max gen: 15, Max flush: 24 Begin loop 8 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:00.54968599Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1244,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:00.595176607Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1244,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:00.595774987Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1244} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"0fd98d0d5d909611f8b408ccda2559d35fe117bb\",\n git_commit_timestamp: \"2024-11-08T20:05:44.000000000Z\",\n git_branch: \"mkeeter/drop-skipped-backpressure-guard\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:00.595812786Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1244} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:00.595827125Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1244} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:00.595841574Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1244} {"msg":"Crucible cd37c503-9cb7-4fcd-a0e3-1e11da7ce9f3 has session id: 6a56b56c-bbad-4c78-a04b-111658aa4854","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:00.596258944Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1244,"session_id":"6a56b56c-bbad-4c78-a04b-111658aa4854"} {"msg":"Upstairs opts: Upstairs UUID: cd37c503-9cb7-4fcd-a0e3-1e11da7ce9f3, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:00.596307772Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1244,"session_id":"6a56b56c-bbad-4c78-a04b-111658aa4854"} {"msg":"Crucible stats registered with UUID: cd37c503-9cb7-4fcd-a0e3-1e11da7ce9f3","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:00.59633499Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1244,"session_id":"6a56b56c-bbad-4c78-a04b-111658aa4854"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:00.627934836Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1244,"task":"crutest"} {"msg":"The guest has requested activation with gen:16","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:00.627984293Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1244} {"msg":"cd37c503-9cb7-4fcd-a0e3-1e11da7ce9f3 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:00.628018592Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1244,"session_id":"6a56b56c-bbad-4c78-a04b-111658aa4854"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:00.628036591Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1244,"client":"0","":"downstairs","session_id":"6a56b56c-bbad-4c78-a04b-111658aa4854"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:00.6280605Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1244,"client":"0","":"downstairs","session_id":"6a56b56c-bbad-4c78-a04b-111658aa4854"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:00.628078989Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1244,"client":"1","":"downstairs","session_id":"6a56b56c-bbad-4c78-a04b-111658aa4854"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:00.628094848Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1244,"client":"1","":"downstairs","session_id":"6a56b56c-bbad-4c78-a04b-111658aa4854"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:00.628108377Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1244,"client":"2","":"downstairs","session_id":"6a56b56c-bbad-4c78-a04b-111658aa4854"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:00.628121907Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1244,"client":"2","":"downstairs","session_id":"6a56b56c-bbad-4c78-a04b-111658aa4854"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:00.628134976Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1244,"":"io task","client":"0","":"downstairs","session_id":"6a56b56c-bbad-4c78-a04b-111658aa4854"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:00.628148675Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1244,"":"io task","client":"2","":"downstairs","session_id":"6a56b56c-bbad-4c78-a04b-111658aa4854"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:00.628162255Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1244,"":"io task","client":"1","":"downstairs","session_id":"6a56b56c-bbad-4c78-a04b-111658aa4854"} {"msg":"ds_connection connected from Ok(127.0.0.1:65012)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:00.628391744Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1244,"":"io task","client":"2","":"downstairs","session_id":"6a56b56c-bbad-4c78-a04b-111658aa4854"} {"msg":"ds_connection connected from Ok(127.0.0.1:41775)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:00.628492939Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1244,"":"io task","client":"1","":"downstairs","session_id":"6a56b56c-bbad-4c78-a04b-111658aa4854"} {"msg":"ds_connection connected from Ok(127.0.0.1:47090)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:00.628528477Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1244,"":"io task","client":"0","":"downstairs","session_id":"6a56b56c-bbad-4c78-a04b-111658aa4854"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:00.629146687Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1244,"client":"0","":"downstairs","session_id":"6a56b56c-bbad-4c78-a04b-111658aa4854"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:00.629182555Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1244,"client":"0","":"downstairs","session_id":"6a56b56c-bbad-4c78-a04b-111658aa4854"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:00.629201964Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1244,"client":"1","":"downstairs","session_id":"6a56b56c-bbad-4c78-a04b-111658aa4854"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:00.629219853Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1244,"client":"1","":"downstairs","session_id":"6a56b56c-bbad-4c78-a04b-111658aa4854"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:00.629237842Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1244,"client":"2","":"downstairs","session_id":"6a56b56c-bbad-4c78-a04b-111658aa4854"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:00.629255131Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1244,"client":"2","":"downstairs","session_id":"6a56b56c-bbad-4c78-a04b-111658aa4854"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:00.629772176Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1244,"client":"0","":"downstairs","session_id":"6a56b56c-bbad-4c78-a04b-111658aa4854"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:00.629945948Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1244,"client":"1","":"downstairs","session_id":"6a56b56c-bbad-4c78-a04b-111658aa4854"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:00.629971046Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1244,"client":"2","":"downstairs","session_id":"6a56b56c-bbad-4c78-a04b-111658aa4854"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:00.630356947Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1244,"client":"0","":"downstairs","session_id":"6a56b56c-bbad-4c78-a04b-111658aa4854"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:00.630385956Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1244,"client":"0","":"downstairs","session_id":"6a56b56c-bbad-4c78-a04b-111658aa4854"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:00.630400595Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1244,"session_id":"6a56b56c-bbad-4c78-a04b-111658aa4854"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:00.630531789Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1244,"client":"1","":"downstairs","session_id":"6a56b56c-bbad-4c78-a04b-111658aa4854"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:00.630556198Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1244,"client":"1","":"downstairs","session_id":"6a56b56c-bbad-4c78-a04b-111658aa4854"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:00.630570087Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1244,"session_id":"6a56b56c-bbad-4c78-a04b-111658aa4854"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:00.630595146Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1244,"client":"2","":"downstairs","session_id":"6a56b56c-bbad-4c78-a04b-111658aa4854"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:00.630609415Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1244,"client":"2","":"downstairs","session_id":"6a56b56c-bbad-4c78-a04b-111658aa4854"} {"msg":"[0]R flush_numbers[0..12]: [8, 22, 19, 19, 20, 23, 23, 20, 15, 13, 1, 16]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:00.630665382Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1244,"":"downstairs","":"downstairs","session_id":"6a56b56c-bbad-4c78-a04b-111658aa4854"} {"msg":"[0]R generation[0..12]: [4, 14, 13, 13, 14, 14, 14, 14, 10, 8, 1, 11]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:00.630685661Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1244,"":"downstairs","":"downstairs","session_id":"6a56b56c-bbad-4c78-a04b-111658aa4854"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:00.630700111Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1244,"":"downstairs","":"downstairs","session_id":"6a56b56c-bbad-4c78-a04b-111658aa4854"} {"msg":"[1]R flush_numbers[0..12]: [8, 22, 19, 19, 20, 23, 23, 20, 15, 13, 1, 16]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:00.630714Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1244,"":"downstairs","":"downstairs","session_id":"6a56b56c-bbad-4c78-a04b-111658aa4854"} {"msg":"[1]R generation[0..12]: [4, 14, 13, 13, 14, 14, 14, 14, 10, 8, 1, 11]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:00.630730869Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1244,"":"downstairs","":"downstairs","session_id":"6a56b56c-bbad-4c78-a04b-111658aa4854"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:00.630745258Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1244,"":"downstairs","":"downstairs","session_id":"6a56b56c-bbad-4c78-a04b-111658aa4854"} {"msg":"[2]R flush_numbers[0..12]: [8, 22, 19, 19, 20, 23, 23, 20, 15, 13, 1, 16]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:00.630758998Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1244,"":"downstairs","":"downstairs","session_id":"6a56b56c-bbad-4c78-a04b-111658aa4854"} {"msg":"[2]R generation[0..12]: [4, 14, 13, 13, 14, 14, 14, 14, 10, 8, 1, 11]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:00.630772487Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1244,"":"downstairs","":"downstairs","session_id":"6a56b56c-bbad-4c78-a04b-111658aa4854"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:00.630786366Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1244,"":"downstairs","":"downstairs","session_id":"6a56b56c-bbad-4c78-a04b-111658aa4854"} {"msg":"Max found gen is 16","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:00.630799396Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1244,"":"downstairs","":"downstairs","session_id":"6a56b56c-bbad-4c78-a04b-111658aa4854"} {"msg":"Generation requested: 16 >= found:16","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:00.630812735Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1244,"":"downstairs","":"downstairs","session_id":"6a56b56c-bbad-4c78-a04b-111658aa4854"} {"msg":"Next flush: 25","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:00.630825894Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1244,"":"downstairs","":"downstairs","session_id":"6a56b56c-bbad-4c78-a04b-111658aa4854"} {"msg":"All extents match","v":0,"Wait for a query_work_queue command to finish before sending IO name":"crucible","level":30,"time":"2024-11-08T20:23:00.630847403Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1244,"":"downstairs","":"downstairs","session_id":"6a56b56c-bbad-4c78-a04b-111658aa4854"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:00.630861073Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1244,"session_id":"6a56b56c-bbad-4c78-a04b-111658aa4854"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:00.630873122Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1244,"client":"0","":"downstairs","session_id":"6a56b56c-bbad-4c78-a04b-111658aa4854"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:00.630886551Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1244,"client":"1","":"downstairs","session_id":"6a56b56c-bbad-4c78-a04b-111658aa4854"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:00.630899731Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1244,"client":"2","":"downstairs","session_id":"6a56b56c-bbad-4c78-a04b-111658aa4854"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:00.63091337Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1244,"session_id":"6a56b56c-bbad-4c78-a04b-111658aa4854"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:00.630925889Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1244,"session_id":"6a56b56c-bbad-4c78-a04b-111658aa4854"} {"msg":"cd37c503-9cb7-4fcd-a0e3-1e11da7ce9f3 is now active with session: 6a56b56c-bbad-4c78-a04b-111658aa4854","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:00.630940359Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1244,"session_id":"6a56b56c-bbad-4c78-a04b-111658aa4854"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:00.630952558Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1244,"session_id":"6a56b56c-bbad-4c78-a04b-111658aa4854"} {"msg":"The guest has finished waiting for activation with:16","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:00.630964848Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1244} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 514 len 12288 data: 3 3 2 03/30 Write block 552 len 32768 data: 2 2 3 4 4 4 4 4 04/30 Read block 98 len 36864 05/30 Read block 123 len 28672 06/30 Read block 552 len 12288 07/30 Flush 08/30 Write block 9 len 16384 data: 3 3 3 3 09/30 Flush 10/30 Write block 209 len 4096 data: 2 11/30 Read block 26 len 12288 12/30 Write block 269 len 36864 data: 2 2 3 3 3 3 4 4 3 13/30 Read block 139 len 20480 14/30 Read block 14 len 28672 15/30 Write block 585 len 16384 data: 5 5 5 4 16/30 Write block 106 len 36864 data: 3 4 3 3 2 2 2 2 3 17/30 Read block 517 len 28672 18/30 Read block 297 len 36864 19/30 Read block 106 len 20480 20/30 Write block 133 len 4096 data: 3 21/30 Write block 2 len 16384 data: 2 2 3 4 22/30 Read block 410 len 40960 23/30 Read block 386 len 16384 24/30 Read block 134 len 4096 25/30 Read block 406 len 40960 26/30 Read block 365 len 4096 27/30 Write block 402 len 24576 data: 3 3 4 4 4 3 28/30 Read block 193 len 32768 29/30 Read block 470 len 12288 30/30 Write block 204 len 16384 data: 2 2 2 2 ---------------------------------------------------------------- Crucible gen:16 GIO:true work queues: Upstairs:0 downstairs:21 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1009 Acked Write 1 Done Done Done false 1010 Acked Read 3 Done Done Done false 1011 Acked Write 9 Done Done Done false 1012 Acked Read 5 Done Done Done false 1013 Acked Read 7 Done Done Done false 1014 Acked Write 4 Done Done Done false 1015 Acked Write 9 Done Done Done false 1016 Acked Read 7 Done Done Done false 1017 Acked Read 9 Done Done Done false 1018 Acked Read 5 Done Done Done false 1019 Acked Write 1 Done Done Done false 1020 Acked Write 4 Done Done Done false 1021 Acked Read 10 Done Done Done false 1022 Acked Read 4 Done Done Done false 1023 Acked Read 1 Done Done Done false 1024 Acked Read 10 Done Done Done false 1025 Acked Read 1 Done Done Done false 1026 Acked Write 6 Done Done Done false 1027 Acked Read 8 Done Done Done false 1028 Acked Read 3 Done Done Done false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 20 20 20 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1008 1008 1008 Downstairs last five completed: 1008 1007 1006 1005 1004 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:00.653323205Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1244} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 16 16 16 27 27 27 T T T 1 020-039 14 14 14 22 22 22 F F F 2 040-059 13 13 13 19 19 19 F F F 3 060-079 13 13 13 19 19 19 F F F 4 080-099 14 14 14 20 20 20 F F F 5 100-119 14 14 14 23 23 23 T T T 6 120-139 14 14 14 23 23 23 T T T 7 140-159 14 14 14 20 20 20 F F F 8 160-179 10 10 10 15 15 15 F F F 9 180-199 8 8 8 13 13 13 F F F 10 200-219 1 1 1 1 1 1 T T T 11 220-239 11 11 11 16 16 16 F F F 12 240-259 10 10 10 15 15 15 F F F 13 260-279 14 14 14 22 22 22 T T T 14 280-299 12 12 12 18 18 18 F F F 15 300-319 12 12 12 18 18 18 F F F 16 320-339 12 12 12 18 18 18 F F F 17 340-359 15 15 15 24 24 24 F F F 18 360-379 15 15 15 24 24 24 F F F 19 380-399 5 5 5 9 9 9 F F F 20 400-419 11 11 11 16 16 16 T T T 21 420-439 12 12 12 18 18 18 F F F 22 440-459 12 12 12 17 17 17 F F F 23 460-479 12 12 12 18 18 18 F F F 24 480-499 14 14 14 21 21 21 F F F 25 500-519 16 16 16 26 26 26 F F F 26 520-539 14 14 14 20 20 20 F F F 27 540-559 16 16 16 26 26 26 F F F 28 560-579 15 15 15 24 24 24 F F F 29 580-599 14 14 14 20 20 20 T T T Max gen: 16, Max flush: 27 A Difference in extent metadata is expected here On loop 8 Verifying data now No change after verify Loop: 8 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 17 17 17 28 28 28 F F F 1 020-039 14 14 14 22 22 22 F F F 2 040-059 13 13 13 19 19 19 F F F 3 060-079 13 13 13 19 19 19 F F F 4 080-099 14 14 14 20 20 20 F F F 5 100-119 17 17 17 28 28 28 F F F 6 120-139 17 17 17 28 28 28 F F F 7 140-159 14 14 14 20 20 20 F F F 8 160-179 10 10 10 15 15 15 F F F 9 180-199 8 8 8 13 13 13 F F F 10 200-219 17 17 17 28 28 28 F F F 11 220-239 11 11 11 16 16 16 F F F 12 240-259 10 10 10 15 15 15 F F F 13 260-279 17 17 17 28 28 28 F F F 14 280-299 12 12 12 18 18 18 F F F 15 300-319 12 12 12 18 18 18 F F F 16 320-339 12 12 12 18 18 18 F F F 17 340-359 15 15 15 24 24 24 F F F 18 360-379 15 15 15 24 24 24 F F F 19 380-399 5 5 5 9 9 9 F F F 20 400-419 17 17 17 28 28 28 F F F 21 420-439 12 12 12 18 18 18 F F F 22 440-459 12 12 12 17 17 17 F F F 23 460-479 12 12 12 18 18 18 F F F 24 480-499 14 14 14 21 21 21 F F F 25 500-519 16 16 16 26 26 26 F F F 26 520-539 14 14 14 20 20 20 F F F 27 540-559 16 16 16 26 26 26 F F F 28 560-579 15 15 15 24 24 24 F F F 29 580-599 17 17 17 28 28 28 F F F Max gen: 17, Max flush: 28 Begin loop 9 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:05.006099516Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1252,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:05.05001496Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1252,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:05.050635009Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1252} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"0fd98d0d5d909611f8b408ccda2559d35fe117bb\",\n git_commit_timestamp: \"2024-11-08T20:05:44.000000000Z\",\n git_branch: \"mkeeter/drop-skipped-backpressure-guard\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:05.050671148Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1252} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:05.050685487Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1252} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:05.050700406Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1252} {"msg":"Crucible 6b4bd247-8c8b-4306-827f-61f944290b49 has session id: 67ef5d52-af32-4ccd-b98f-e998900b0b69","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:05.051165843Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1252,"session_id":"67ef5d52-af32-4ccd-b98f-e998900b0b69"} {"msg":"Upstairs opts: Upstairs UUID: 6b4bd247-8c8b-4306-827f-61f944290b49, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:05.051211791Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1252,"session_id":"67ef5d52-af32-4ccd-b98f-e998900b0b69"} {"msg":"Crucible stats registered with UUID: 6b4bd247-8c8b-4306-827f-61f944290b49","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:05.05123614Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1252,"session_id":"67ef5d52-af32-4ccd-b98f-e998900b0b69"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:05.08211998Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1252,"task":"crutest"} {"msg":"The guest has requested activation with gen:18","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:05.082156149Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1252} {"msg":"6b4bd247-8c8b-4306-827f-61f944290b49 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:05.082185547Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1252,"session_id":"67ef5d52-af32-4ccd-b98f-e998900b0b69"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:05.082204016Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1252,"client":"0","":"downstairs","session_id":"67ef5d52-af32-4ccd-b98f-e998900b0b69"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:05.082232125Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1252,"client":"0","":"downstairs","session_id":"67ef5d52-af32-4ccd-b98f-e998900b0b69"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:05.082251214Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1252,"client":"1","":"downstairs","session_id":"67ef5d52-af32-4ccd-b98f-e998900b0b69"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:05.082269043Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1252,"client":"1","":"downstairs","session_id":"67ef5d52-af32-4ccd-b98f-e998900b0b69"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:05.082289462Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1252,"client":"2","":"downstairs","session_id":"67ef5d52-af32-4ccd-b98f-e998900b0b69"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:05.082307071Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1252,"client":"2","":"downstairs","session_id":"67ef5d52-af32-4ccd-b98f-e998900b0b69"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:05.08233648Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1252,"":"io task","client":"2","":"downstairs","session_id":"67ef5d52-af32-4ccd-b98f-e998900b0b69"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:05.082369688Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1252,"":"io task","client":"0","":"downstairs","session_id":"67ef5d52-af32-4ccd-b98f-e998900b0b69"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:05.082401457Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1252,"":"io task","client":"1","":"downstairs","session_id":"67ef5d52-af32-4ccd-b98f-e998900b0b69"} {"msg":"ds_connection connected from Ok(127.0.0.1:33783)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:05.082437595Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1252,"":"io task","client":"2","":"downstairs","session_id":"67ef5d52-af32-4ccd-b98f-e998900b0b69"} {"msg":"ds_connection connected from Ok(127.0.0.1:55533)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:05.082474813Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1252,"":"io task","client":"1","":"downstairs","session_id":"67ef5d52-af32-4ccd-b98f-e998900b0b69"} {"msg":"ds_connection connected from Ok(127.0.0.1:37396)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:05.082680633Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1252,"":"io task","client":"0","":"downstairs","session_id":"67ef5d52-af32-4ccd-b98f-e998900b0b69"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:05.083258455Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1252,"client":"1","":"downstairs","session_id":"67ef5d52-af32-4ccd-b98f-e998900b0b69"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:05.083288154Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1252,"client":"1","":"downstairs","session_id":"67ef5d52-af32-4ccd-b98f-e998900b0b69"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:05.083314562Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1252,"client":"2","":"downstairs","session_id":"67ef5d52-af32-4ccd-b98f-e998900b0b69"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:05.083343361Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1252,"client":"2","":"downstairs","session_id":"67ef5d52-af32-4ccd-b98f-e998900b0b69"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:05.083372649Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1252,"client":"0","":"downstairs","session_id":"67ef5d52-af32-4ccd-b98f-e998900b0b69"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:05.083401318Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1252,"client":"0","":"downstairs","session_id":"67ef5d52-af32-4ccd-b98f-e998900b0b69"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:05.083943042Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1252,"client":"1","":"downstairs","session_id":"67ef5d52-af32-4ccd-b98f-e998900b0b69"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:05.083989289Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1252,"client":"2","":"downstairs","session_id":"67ef5d52-af32-4ccd-b98f-e998900b0b69"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:05.084024967Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1252,"client":"0","":"downstairs","session_id":"67ef5d52-af32-4ccd-b98f-e998900b0b69"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:05.084571811Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1252,"client":"1","":"downstairs","session_id":"67ef5d52-af32-4ccd-b98f-e998900b0b69"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:05.084600589Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1252,"client":"1","":"downstairs","session_id":"67ef5d52-af32-4ccd-b98f-e998900b0b69"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:05.084616629Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1252,"session_id":"67ef5d52-af32-4ccd-b98f-e998900b0b69"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:05.084629888Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1252,"client":"2","":"downstairs","session_id":"67ef5d52-af32-4ccd-b98f-e998900b0b69"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:05.084643727Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1252,"client":"2","":"downstairs","session_id":"67ef5d52-af32-4ccd-b98f-e998900b0b69"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:05.084657096Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1252,"session_id":"67ef5d52-af32-4ccd-b98f-e998900b0b69"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:05.084669526Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1252,"client":"0","":"downstairs","session_id":"67ef5d52-af32-4ccd-b98f-e998900b0b69"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:05.084682985Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1252,"client":"0","":"downstairs","session_id":"67ef5d52-af32-4ccd-b98f-e998900b0b69"} {"msg":"[0]R flush_numbers[0..12]: [28, 22, 19, 19, 20, 28, 28, 20, 15, 13, 28, 16]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:05.084696655Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1252,"":"downstairs","":"downstairs","session_id":"67ef5d52-af32-4ccd-b98f-e998900b0b69"} {"msg":"[0]R generation[0..12]: [17, 14, 13, 13, 14, 17, 17, 14, 10, 8, 17, 11]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:05.084710694Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1252,"":"downstairs","":"downstairs","session_id":"67ef5d52-af32-4ccd-b98f-e998900b0b69"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:05.084724813Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1252,"":"downstairs","":"downstairs","session_id":"67ef5d52-af32-4ccd-b98f-e998900b0b69"} {"msg":"[1]R flush_numbers[0..12]: [28, 22, 19, 19, 20, 28, 28, 20, 15, 13, 28, 16]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:05.084738663Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1252,"":"downstairs","":"downstairs","session_id":"67ef5d52-af32-4ccd-b98f-e998900b0b69"} {"msg":"[1]R generation[0..12]: [17, 14, 13, 13, 14, 17, 17, 14, 10, 8, 17, 11]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:05.084752492Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1252,"":"downstairs","":"downstairs","session_id":"67ef5d52-af32-4ccd-b98f-e998900b0b69"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:05.084776621Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1252,"":"downstairs","":"downstairs","session_id":"67ef5d52-af32-4ccd-b98f-e998900b0b69"} {"msg":"[2]R flush_numbers[0..12]: [28, 22, 19, 19, 20, 28, 28, 20, 15, 13, 28, 16]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:05.084804829Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1252,"":"downstairs","":"downstairs","session_id":"67ef5d52-af32-4ccd-b98f-e998900b0b69"} {"msg":"[2]R generation[0..12]: [17, 14, 13, 13, 14, 17, 17, 14, 10, 8, 17, 11]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:05.084831808Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1252,"":"downstairs","":"downstairs","session_id":"67ef5d52-af32-4ccd-b98f-e998900b0b69"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:05.084847447Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1252,"":"downstairs","":"downstairs","session_id":"67ef5d52-af32-4ccd-b98f-e998900b0b69"} {"msg":"Max found gen is 18","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:05.084867796Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW"Wait for a query_work_queue command to finish before sending IO ,"pid":1252,"":"downstairs","":"downstairs","session_id":"67ef5d52-af32-4ccd-b98f-e998900b0b69"} {"msg":"Generation requested: 18 >= found:18","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:05.084903395Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1252,"":"downstairs","":"downstairs","session_id":"67ef5d52-af32-4ccd-b98f-e998900b0b69"} {"msg":"Next flush: 29","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:05.084930583Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1252,"":"downstairs","":"downstairs","session_id":"67ef5d52-af32-4ccd-b98f-e998900b0b69"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:05.084947322Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1252,"":"downstairs","":"downstairs","session_id":"67ef5d52-af32-4ccd-b98f-e998900b0b69"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:05.084960832Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1252,"session_id":"67ef5d52-af32-4ccd-b98f-e998900b0b69"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:05.084979891Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1252,"client":"0","":"downstairs","session_id":"67ef5d52-af32-4ccd-b98f-e998900b0b69"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:05.08500535Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1252,"client":"1","":"downstairs","session_id":"67ef5d52-af32-4ccd-b98f-e998900b0b69"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:05.085032028Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1252,"client":"2","":"downstairs","session_id":"67ef5d52-af32-4ccd-b98f-e998900b0b69"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:05.085049477Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1252,"session_id":"67ef5d52-af32-4ccd-b98f-e998900b0b69"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:05.085064367Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1252,"session_id":"67ef5d52-af32-4ccd-b98f-e998900b0b69"} {"msg":"6b4bd247-8c8b-4306-827f-61f944290b49 is now active with session: 67ef5d52-af32-4ccd-b98f-e998900b0b69","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:05.085088975Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1252,"session_id":"67ef5d52-af32-4ccd-b98f-e998900b0b69"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:05.085116924Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1252,"session_id":"67ef5d52-af32-4ccd-b98f-e998900b0b69"} {"msg":"The guest has finished waiting for activation with:18","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:05.085139853Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1252} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 107 len 16384 data: 5 4 4 3 03/30 Write block 180 len 36864 data: 3 3 3 3 2 2 2 2 2 04/30 Flush 05/30 Write block 31 len 16384 data: 2 3 3 3 06/30 Read block 463 len 4096 07/30 Read block 24 len 20480 08/30 Read block 128 len 4096 09/30 Write block 368 len 32768 data: 5 3 3 3 4 3 3 2 10/30 Write block 358 len 16384 data: 8 8 7 7 11/30 Read block 478 len 20480 12/30 Read block 342 len 20480 13/30 Read block 451 len 28672 14/30 Write block 20 len 36864 data: 2 3 3 3 4 4 4 4 2 15/30 Read block 590 len 24576 16/30 Flush 17/30 Write block 142 len 12288 data: 2 3 3 18/30 Write block 19 len 12288 data: 2 3 4 19/30 Read block 122 len 28672 20/30 Read block 204 len 16384 21/30 Write block 248 len 20480 data: 5 5 4 4 5 22/30 Write block 235 len 36864 data: 2 2 3 3 3 4 4 4 4 23/30 Write block 299 len 16384 data: 3 3 3 3 24/30 Read block 451 len 12288 25/30 Read block 266 len 16384 26/30 Read block 142 len 40960 27/30 Write block 231 len 8192 data: 3 3 28/30 Read block 103 len 24576 29/30 Write block 310 len 32768 data: 5 5 5 5 3 3 3 3 30/30 Write block 60 len 24576 data: 3 3 3 3 4 3 ---------------------------------------------------------------- Crucible gen:18 GIO:true work queues: Upstairs:0 downstairs:14 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1016 Acked Write 3 Done Done Done false 1017 Acked Write 3 Done Done Done false 1018 Acked Read 7 Done Done Done false 1019 Acked Read 4 Done Done Done false 1020 Acked Write 5 Done Done Done false 1021 Acked Write 9 Done Done Done false 1022 Acked Write 4 Done Done Done false 1023 Acked Read 3 Done Done Done false 1024 Acked Read 4 Done Done Done false 1025 Acked Read 10 Done Done Done false 1026 Acked Write 2 Done Done Done false 1027 Acked Read 6 Done Done Done false 1028 Acked Write 8 Sent Sent Sent false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 12 12 12 36 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1015 1015 1015 Downstairs last five completed: 1015 1014 1013 1012 1011 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:05.109261154Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1252} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 17 17 17 28 28 28 T T T 1 020-039 18 18 18 31 31 31 T T T 2 040-059 13 13 13 19 19 19 F F F 3 060-079 13 13 13 19 19 19 T T T 4 080-099 14 14 14 20 20 20 F F F 5 100-119 18 18 18 30 30 30 F F F 6 120-139 17 17 17 28 28 28 F F F 7 140-159 14 14 14 20 20 20 T T T 8 160-179 10 10 10 15 15 15 F F F 9 180-199 18 18 18 30 30 30 F F F 10 200-219 17 17 17 28 28 28 F F F 11 220-239 11 11 11 16 16 16 T T T 12 240-259 10 10 10 15 15 15 T T T 13 260-279 17 17 17 28 28 28 F F F 14 280-299 12 12 12 18 18 18 T T T 15 300-319 12 12 12 18 18 18 T T T 16 320-339 12 12 12 18 18 18 F F F 17 340-359 18 18 18 31 31 31 F F F 18 360-379 18 18 18 31 31 31 F F F 19 380-399 5 5 5 9 9 9 F F F 20 400-419 17 17 17 28 28 28 F F F 21 420-439 12 12 12 18 18 18 F F F 22 440-459 12 12 12 17 17 17 F F F 23 460-479 12 12 12 18 18 18 F F F 24 480-499 14 14 14 21 21 21 F F F 25 500-519 16 16 16 26 26 26 F F F 26 520-539 14 14 14 20 20 20 F F F 27 540-559 16 16 16 26 26 26 F F F 28 560-579 15 15 15 24 24 24 F F F 29 580-599 17 17 17 28 28 28 F F F Max gen: 18, Max flush: 31 A Difference in extent metadata is expected here On loop 9 Verifying data now No change after verify Loop: 9 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 19 19 19 32 32 32 F F F 1 020-039 19 19 19 32 32 32 F F F 2 040-059 13 13 13 19 19 19 F F F 3 060-079 19 19 19 32 32 32 F F F 4 080-099 14 14 14 20 20 20 F F F 5 100-119 18 18 18 30 30 30 F F F 6 120-139 17 17 17 28 28 28 F F F 7 140-159 19 19 19 32 32 32 F F F 8 160-179 10 10 10 15 15 15 F F F 9 180-199 18 18 18 30 30 30 F F F 10 200-219 17 17 17 28 28 28 F F F 11 220-239 19 19 19 32 32 32 F F F 12 240-259 19 19 19 32 32 32 F F F 13 260-279 17 17 17 28 28 28 F F F 14 280-299 19 19 19 32 32 32 F F F 15 300-319 19 19 19 32 32 32 F F F 16 320-339 12 12 12 18 18 18 F F F 17 340-359 18 18 18 31 31 31 F F F 18 360-379 18 18 18 31 31 31 F F F 19 380-399 5 5 5 9 9 9 F F F 20 400-419 17 17 17 28 28 28 F F F 21 420-439 12 12 12 18 18 18 F F F 22 440-459 12 12 12 17 17 17 F F F 23 460-479 12 12 12 18 18 18 F F F 24 480-499 14 14 14 21 21 21 F F F 25 500-519 16 16 16 26 26 26 F F F 26 520-539 14 14 14 20 20 20 F F F 27 540-559 16 16 16 26 26 26 F F F 28 560-579 15 15 15 24 24 24 F F F 29 580-599 17 17 17 28 28 28 F F F Max gen: 19, Max flush: 32 Begin loop 10 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:09.479282454Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:09.523036095Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:09.523659355Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"0fd98d0d5d909611f8b408ccda2559d35fe117bb\",\n git_commit_timestamp: \"2024-11-08T20:05:44.000000000Z\",\n git_branch: \"mkeeter/drop-skipped-backpressure-guard\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:09.523702973Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:09.523726052Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:09.523744501Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260} {"msg":"Crucible e37448e7-d801-4249-a71a-d2a674bdf559 has session id: a64bf946-1fdc-4bf3-9d5a-84b3f5106bba","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:09.52417739Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"Upstairs opts: Upstairs UUID: e37448e7-d801-4249-a71a-d2a674bdf559, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:09.524225747Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"Crucible stats registered with UUID: e37448e7-d801-4249-a71a-d2a674bdf559","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:09.524251766Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:09.556372276Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"task":"crutest"} {"msg":"The guest has requested activation with gen:20","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:09.556406744Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260} {"msg":"e37448e7-d801-4249-a71a-d2a674bdf559 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:09.556431933Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:09.556448933Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"client":"0","":"downstairs","session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:09.556477021Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"client":"0","":"downstairs","session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:09.55649473Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"client":"1","":"downstairs","session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:09.556511699Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"client":"1","":"downstairs","session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:09.556528259Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"client":"2","":"downstairs","session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:09.556545138Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"client":"2","":"downstairs","session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:09.556561677Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"":"io task","client":"2","":"downstairs","session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:09.556583406Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"":"io task","client":"0","":"downstairs","session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:09.556601055Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"":"io task","client":"1","":"downstairs","session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"ds_connection connected from Ok(127.0.0.1:55762)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:09.556678931Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"":"io task","client":"1","":"downstairs","session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"ds_connection connected from Ok(127.0.0.1:59562)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:09.556748848Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"":"io task","client":"0","":"downstairs","session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"ds_connection connected from Ok(127.0.0.1:47487)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:09.556938089Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"":"io task","client":"2","":"downstairs","session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:09.55752864Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"client":"0","":"downstairs","session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:09.557570388Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"client":"0","":"downstairs","session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:09.557596117Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"client":"1","":"downstairs","session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:09.557611646Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"client":"1","":"downstairs","session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:09.557625615Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"client":"2","":"downstairs","session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:09.557639494Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"client":"2","":"downstairs","session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:09.558104921Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"client":"0","":"downstairs","session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:09.558243675Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"client":"1","":"downstairs","session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:09.558279943Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"client":"2","":"downstairs","session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:09.558635256Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"client":"0","":"downstairs","session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:09.558674024Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"client":"0","":"downstairs","session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:09.558691353Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:09.558764099Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"client":"1","":"downstairs","session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:09.558783748Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"client":"1","":"downstairs","session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:09.558797428Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:09.558851995Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"client":"2","":"downstairs","session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:09.558871294Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"client":"2","":"downstairs","session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"[0]R flush_numbers[0..12]: [32, 32, 19, 32, 20, 30, 28, 32, 15, 30, 28, 32]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:09.558885513Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"":"downstairs","":"downstairs","session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"[0]R generation[0..12]: [19, 19, 13, 19, 14, 18, 17, 19, 10, 18, 17, 19]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:09.558899603Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"":"downstairs","":"downstairs","session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:09.558913652Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"":"downstairs","":"downstairs","session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"[1]R flush_numbers[0..12]: [32, 32, 19, 32, 20, 30, 28, 32, 15, 30, 28, 32]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:09.558927411Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"":"downstairs","":"downstairs","session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"[1]R generation[0..12]: [19, 19, 13, 19, 14, 18, 17, 19, 10, 18, 17, 19]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:09.558940821Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"":"downstairs","":"downstairs","session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:09.55895464Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"":"downstairs","":"downstairs","session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"[2]R flush_numbers[0..12]: [32, 32, 19, 32, 20, 30, 28, 32, 15, 30, 28, 32]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:09.558968359Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"":"downstairs","":"downstairs","session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"[2]R generation[0..12]: [19, 19, 13, 19, 14, 18, 17, 19, 10, 18, 17, 19]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:09.558981869Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"":"downstairs","":"downstairs","session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:09.558998438Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"":"downstairs","":"downstairs","session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"Max found gen is 20","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:09.559011767Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"":"downstairs","":"downstairs","session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"Generation requested: 20 >= found:20","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:09.559025016Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"":"downstairs","":"downstairs","session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"Next flush: 33","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:09.559037966Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"":"downstairs","":"downstairs","session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:09.559050895Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"":"downstairs","":"downstairs","session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:09.559063975Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:09.559075994Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"client":"0","":"Wait for a query_work_queue command to finish before sending IO downstairs","session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:09.559097893Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"client":"1","":"downstairs","session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:09.559111092Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"client":"2","":"downstairs","session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:09.559124032Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:09.559136251Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"e37448e7-d801-4249-a71a-d2a674bdf559 is now active with session: a64bf946-1fdc-4bf3-9d5a-84b3f5106bba","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:09.55914905Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:09.55916093Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"The guest has finished waiting for activation with:20","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:09.559173389Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 522 len 20480 data: 3 3 3 3 3 02/30 Write block 588 len 24576 data: 5 4 4 3 3 3 03/30 Read block 439 len 12288 04/30 Read block 579 len 32768 05/30 Read block 307 len 24576 06/30 Read block 13 len 16384 07/30 Write block 22 len 40960 data: 4 4 5 5 5 5 3 2 2 3 08/30 Read block 110 len 32768 09/30 Read block 438 len 12288 10/30 Write block 31 len 8192 data: 4 4 11/30 Write block 315 len 36864 data: 4 4 4 3 3 2 2 2 2 12/30 Write block 404 len 36864 data: 5 5 5 4 3 3 2 4 3 13/30 Write block 286 len 12288 data: 5 5 6 14/30 Write block 256 len 12288 data: 2 3 3 15/30 Read block 247 len 32768 16/30 Write block 313 len 4096 data: 6 17/30 Read block 319 len 28672 18/30 Read block 341 len 36864 19/30 Read block 380 len 32768 20/30 Read block 284 len 16384 21/30 Read block 69 len 36864 22/30 Flush 23/30 Write block 119 len 24576 data: 3 3 3 3 3 3 24/30 Flush 25/30 Write block 151 len 16384 data: 3 2 2 2 26/30 Read block 552 len 24576 27/30 Read block 548 len 28672 28/30 Write block 12 len 8192 data: 4 3 29/30 Read block 508 len 32768 30/30 Write block 539 len 32768 data: 2 2 2 3 3 3 3 3 ---------------------------------------------------------------- Crucible gen:20 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 4 Done Done Done false 1025 Acked Read 6 Done Done Done false 1026 Acked Read 7 Done Done Done false 1027 Acked Write 2 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:09.58289392Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260} Wrote out file "/tmp/test_repair_verify.data" {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:09.583653093Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:09.583688191Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:09.58370413Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:09.58371725Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:09.583729729Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:09.583742359Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:09.583754788Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:09.583767137Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:09.583782866Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:09.583795646Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:09.583807765Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:09.583819935Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:09.583832084Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:09.583844443Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:09.583856673Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:09.583868812Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:09.583881202Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:09.583893301Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:09.5839055Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:09.58391795Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:09.583930389Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:09.583942929Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:09.583955278Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:09.583967728Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:09.583980217Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:09.583993926Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:09.584006496Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:09.584019015Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:09.584031374Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:09.584043664Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:09.584056103Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:09.584068443Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:09.584080812Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:09.584093211Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:09.584105431Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:09.58411773Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:09.58412999Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:09.584142389Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:09.584154738Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:09.584166988Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:09.584179187Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:09.584191297Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:09.584205296Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:09.584217725Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:09.584229825Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:09.584241904Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:09.584254123Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:09.584266293Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:09.584278372Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:09.584290582Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:09.584302801Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:09.58431502Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:09.5843272Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:09.584339469Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:09.584351669Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:09.584363848Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:09.584375908Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:09.584387947Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:09.584400087Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:09.584413976Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:09.584426255Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:09.584438325Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:09.584450564Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:09.584462773Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:09.584474853Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:09.584487022Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:09.584499091Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:09.584511251Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:09.58452348Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:09.58453568Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:09.584547779Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:09.584559899Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:09.584571948Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:09.584584047Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:09.584596277Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:09.584608576Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:09.584622316Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:09.584634515Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:09.584646664Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:09.584658744Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:09.584670773Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:09.584682993Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:09.584695112Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:09.584707111Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:09.584719351Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:09.58473137Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:09.58474349Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:09.584755689Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:09.584767809Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:09.584779928Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:09.584792027Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:09.584804167Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:09.584816116Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:09.584829835Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:09.584841965Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:09.584854274Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:09.584866504Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:09.584878673Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:09.584890763Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:09.584902912Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:09.584914951Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:09.584927161Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:09.58493917Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:09.584951489Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:09.584963649Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:09.584975778Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:09.584987868Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:09.585000067Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:09.585012306Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:09.585024666Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:09.585038345Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:09.585050615Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:09.585062774Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:09.585074974Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:09.585087003Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:09.585099092Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:09.585111422Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:09.585123361Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:09.58513542Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:09.58514757Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:09.585159689Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:09.585171989Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:09.585184368Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:09.585196657Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:09.585209017Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1260,"session_id":"a64bf946-1fdc-4bf3-9d5a-84b3f5106bba"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 19 19 19 32 32 32 T T T 1 020-039 20 20 20 33 33 33 F F F 2 040-059 13 13 13 19 19 19 F F F 3 060-079 19 19 19 32 32 32 F F F 4 080-099 14 14 14 20 20 20 F F F 5 100-119 20 20 20 34 34 34 F F F 6 120-139 20 20 20 34 34 34 F F F 7 140-159 19 19 19 32 32 32 T T T 8 160-179 10 10 10 15 15 15 F F F 9 180-199 18 18 18 30 30 30 F F F 10 200-219 17 17 17 28 28 28 F F F 11 220-239 19 19 19 32 32 32 F F F 12 240-259 20 20 20 33 33 33 F F F 13 260-279 17 17 17 28 28 28 F F F 14 280-299 20 20 20 33 33 33 F F F 15 300-319 20 20 20 33 33 33 F F F 16 320-339 20 20 20 33 33 33 F F F 17 340-359 18 18 18 31 31 31 F F F 18 360-379 18 18 18 31 31 31 F F F 19 380-399 5 5 5 9 9 9 F F F 20 400-419 20 20 20 33 33 33 F F F 21 420-439 12 12 12 18 18 18 F F F 22 440-459 12 12 12 17 17 17 F F F 23 460-479 12 12 12 18 18 18 F F F 24 480-499 14 14 14 21 21 21 F F F 25 500-519 16 16 16 26 26 26 F F F 26 520-539 20 20 20 33 33 33 T T T 27 540-559 16 16 16 26 26 26 T T T 28 560-579 15 15 15 24 24 24 F F F 29 580-599 20 20 20 33 33 33 F F F Max gen: 20, Max flush: 34 A Difference in extent metadata is expected here On loop 10 Verifying data now No change after verify Loop: 10 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 21 21 21 35 35 35 F F F 1 020-039 20 20 20 33 33 33 F F F 2 040-059 13 13 13 19 19 19 F F F 3 060-079 19 19 19 32 32 32 F F F 4 080-099 14 14 14 20 20 20 F F F 5 100-119 20 20 20 34 34 34 F F F 6 120-139 20 20 20 34 34 34 F F F 7 140-159 21 21 21 35 35 35 F F F 8 160-179 10 10 10 15 15 15 F F F 9 180-199 18 18 18 30 30 30 F F F 10 200-219 17 17 17 28 28 28 F F F 11 220-239 19 19 19 32 32 32 F F F 12 240-259 20 20 20 33 33 33 F F F 13 260-279 17 17 17 28 28 28 F F F 14 280-299 20 20 20 33 33 33 F F F 15 300-319 20 20 20 33 33 33 F F F 16 320-339 20 20 20 33 33 33 F F F 17 340-359 18 18 18 31 31 31 F F F 18 360-379 18 18 18 31 31 31 F F F 19 380-399 5 5 5 9 9 9 F F F 20 400-419 20 20 20 33 33 33 F F F 21 420-439 12 12 12 18 18 18 F F F 22 440-459 12 12 12 17 17 17 F F F 23 460-479 12 12 12 18 18 18 F F F 24 480-499 14 14 14 21 21 21 F F F 25 500-519 16 16 16 26 26 26 F F F 26 520-539 21 21 21 35 35 35 F F F 27 540-559 21 21 21 35 35 35 F F F 28 560-579 15 15 15 24 24 24 F F F 29 580-599 20 20 20 33 33 33 F F F Max gen: 21, Max flush: 35 Begin loop 11 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:13.893648036Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1268,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:13.937700233Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1268,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:13.938298164Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1268} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"0fd98d0d5d909611f8b408ccda2559d35fe117bb\",\n git_commit_timestamp: \"2024-11-08T20:05:44.000000000Z\",\n git_branch: \"mkeeter/drop-skipped-backpressure-guard\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:13.938342262Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1268} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:13.938363581Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1268} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:13.93838168Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1268} {"msg":"Crucible 167a033e-77e4-48b5-895a-b8bda575d33c has session id: a3837f3e-77e9-4b93-8912-82ec1f446ffa","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:13.938806689Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1268,"session_id":"a3837f3e-77e9-4b93-8912-82ec1f446ffa"} {"msg":"Upstairs opts: Upstairs UUID: 167a033e-77e4-48b5-895a-b8bda575d33c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:13.938845687Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1268,"session_id":"a3837f3e-77e9-4b93-8912-82ec1f446ffa"} {"msg":"Crucible stats registered with UUID: 167a033e-77e4-48b5-895a-b8bda575d33c","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:13.938860606Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1268,"session_id":"a3837f3e-77e9-4b93-8912-82ec1f446ffa"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:13.969739717Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1268,"task":"crutest"} {"msg":"The guest has requested activation with gen:22","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:13.969784195Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1268} {"msg":"167a033e-77e4-48b5-895a-b8bda575d33c active request set","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:13.969848112Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1268,"session_id":"a3837f3e-77e9-4b93-8912-82ec1f446ffa"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:13.96987642Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1268,"client":"0","":"downstairs","session_id":"a3837f3e-77e9-4b93-8912-82ec1f446ffa"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:13.969900759Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1268,"client":"0","":"downstairs","session_id":"a3837f3e-77e9-4b93-8912-82ec1f446ffa"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:13.969915509Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1268,"client":"1","":"downstairs","session_id":"a3837f3e-77e9-4b93-8912-82ec1f446ffa"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:13.969936568Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1268,"client":"1","":"downstairs","session_id":"a3837f3e-77e9-4b93-8912-82ec1f446ffa"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:13.969963366Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1268,"client":"2","":"downstairs","session_id":"a3837f3e-77e9-4b93-8912-82ec1f446ffa"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:13.969991085Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1268,"client":"2","":"downstairs","session_id":"a3837f3e-77e9-4b93-8912-82ec1f446ffa"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:13.970016464Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1268,"":"io task","client":"2","":"downstairs","session_id":"a3837f3e-77e9-4b93-8912-82ec1f446ffa"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:13.970046922Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1268,"":"io task","client":"0","":"downstairs","session_id":"a3837f3e-77e9-4b93-8912-82ec1f446ffa"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:13.970173756Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1268,"":"io task","client":"1","":"downstairs","session_id":"a3837f3e-77e9-4b93-8912-82ec1f446ffa"} {"msg":"ds_connection connected from Ok(127.0.0.1:34697)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:13.970214234Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1268,"":"io task","client":"2","":"downstairs","session_id":"a3837f3e-77e9-4b93-8912-82ec1f446ffa"} {"msg":"ds_connection connected from Ok(127.0.0.1:47191)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:13.970244762Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1268,"":"io task","client":"0","":"downstairs","session_id":"a3837f3e-77e9-4b93-8912-82ec1f446ffa"} {"msg":"ds_connection connected from Ok(127.0.0.1:59639)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:13.970272981Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1268,"":"io task","client":"1","":"downstairs","session_id":"a3837f3e-77e9-4b93-8912-82ec1f446ffa"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:13.970826844Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1268,"client":"0","":"downstairs","session_id":"a3837f3e-77e9-4b93-8912-82ec1f446ffa"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:13.970857023Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1268,"client":"0","":"downstairs","session_id":"a3837f3e-77e9-4b93-8912-82ec1f446ffa"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:13.971030754Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1268,"client":"1","":"downstairs","session_id":"a3837f3e-77e9-4b93-8912-82ec1f446ffa"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:13.971071622Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1268,"client":"1","":"downstairs","session_id":"a3837f3e-77e9-4b93-8912-82ec1f446ffa"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:13.971104371Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1268,"client":"2","":"downstairs","session_id":"a3837f3e-77e9-4b93-8912-82ec1f446ffa"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:13.97112221Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1268,"client":"2","":"downstairs","session_id":"a3837f3e-77e9-4b93-8912-82ec1f446ffa"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:13.971434114Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1268,"client":"0","":"downstairs","session_id":"a3837f3e-77e9-4b93-8912-82ec1f446ffa"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:13.971827795Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1268,"client":"1","":"downstairs","session_id":"a3837f3e-77e9-4b93-8912-82ec1f446ffa"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:13.971970768Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1268,"client":"0","":"downstairs","session_id":"a3837f3e-77e9-4b93-8912-82ec1f446ffa"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:13.972000087Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1268,"client":"0","":"downstairs","session_id":"a3837f3e-77e9-4b93-8912-82ec1f446ffa"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:13.972018066Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1268,"session_id":"a3837f3e-77e9-4b93-8912-82ec1f446ffa"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:13.972031595Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1268,"client":"2","":"downstairs","session_id":"a3837f3e-77e9-4b93-8912-82ec1f446ffa"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:13.972300592Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1268,"client":"1","":"downstairs","session_id":"a3837f3e-77e9-4b93-8912-82ec1f446ffa"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:13.972328561Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1268,"client":"1","":"downstairs","session_id":"a3837f3e-77e9-4b93-8912-82ec1f446ffa"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:13.97234335Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1268,"session_id":"a3837f3e-77e9-4b93-8912-82ec1f446ffa"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:13.972494403Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1268,"client":"2","":"downstairs","session_id":"a3837f3e-77e9-4b93-8912-82ec1f446ffa"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:13.972513342Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1268,"client":"2","":"downstairs","session_id":"a3837f3e-77e9-4b93-8912-82ec1f446ffa"} {"msg":"[0]R flush_numbers[0..12]: [35, 33, 19, 32, 20, 34, 34, 35, 15, 30, 28, 32]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:13.972529101Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1268,"":"downstairs","":"downstairs","session_id":"a3837f3e-77e9-4b93-8912-82ec1f446ffa"} {"msg":"[0]R generation[0..12]: [21, 20, 13, 19, 14, 20, 20, 21, 10, 18, 17, 19]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:13.97254334Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1268,"":"downstairs","":"downstairs","session_id":"a3837f3e-77e9-4b93-8912-82ec1f446ffa"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:13.97255748Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1268,"":"downstairs","":"downstairs","session_id":"a3837f3e-77e9-4b93-8912-82ec1f446ffa"} {"msg":"[1]R flush_numbers[0..12]: [35, 33, 19, 32, 20, 34, 34, 35, 15, 30, 28, 32]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:13.972571179Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1268,"":"downstairs","":"downstairs","session_id":"a3837f3e-77e9-4b93-8912-82ec1f446ffa"} {"msg":"[1]R generation[0..12]: [21, 20, 13, 19, 14, 20, 20, 21, 10, 18, 17, 19]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:13.972584948Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1268,"":"downstairs","":"downstairs","session_id":"a3837f3e-77e9-4b93-8912-82ec1f446ffa"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:13.972598898Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1268,"":"downstairs","":"downstairs","session_id":"a3837f3e-77e9-4b93-8912-82ec1f446ffa"} {"msg":"[2]R flush_numbers[0..12]: [35, 33, 19, 32, 20, 34, 34, 35, 15, 30, 28, 32]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:13.972612537Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1268,"":"downstairs","":"downstairs","session_id":"a3837f3e-77e9-4b93-8912-82ec1f446ffa"} {"msg":"[2]R generation[0..12]: [21, 20, 13, 19, 14, 20, 20, 21, 10, 18, 17, 19]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:13.972629056Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1268,"":"downstairs","":"downstairs","session_id":"a3837f3e-77e9-4b93-8912-82ec1f446ffa"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:13.972643075Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1268,"":"downstairs","":"downstairs","session_id":"a3837f3e-77e9-4b93-8912-82ec1f446ffa"} {"msg":"Max found gen is 22","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:13.972656005Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1268,"":"downstairs","":"downstairs","session_id":"a3837f3e-77e9-4b93-8912-82ec1f446ffa"} {"msg":"Generation requested: 22 >= found:22","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:13.972669154Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1268,"":"downstairs","":"downstairs","session_id":"a3837f3e-77e9-4b93-8912-82ec1f446ffa"} {"msg":"Next flush: 36","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:13.972682113Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1268,"":"downstairs","":"downstairs","session_id":"a3837f3e-77e9-4b93-8912-82ec1f446ffa"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:13.972695053Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1268,"":"downstairs","":"downstairs","session_id":"a3837f3e-77e9-4b93-8912-82ec1f446ffa"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-08T20:23:13.972708252Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1268,"session_id":"a3837f3e-77e9-4b93-8912-82ec1f446ffa"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:13.972726791Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1268,"client":"0","":"downstairs","session_id":"a3837f3e-77e9-4b93-8912-82ec1f446ffa"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:13.972740111Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1268,"client":"1","":"downstairs","session_id":"a3837f3e-77e9-4b93-8912-82ec1f446ffa"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:13.97275335Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1268,"client":"2","":"downstairs","session_id":"a3837f3e-77e9-4b93-8912-82ec1f446ffa"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:13.972766629Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1268,"session_id":"a3837f3e-77e9-4b93-8912-82ec1f446ffa"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:13.972778989Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1268,"session_id":"a3837f3e-77e9-4b93-8912-82ec1f446ffa"} {"msg":"167a033e-77e4-48b5-895a-b8bda575d33c is now active with session: a3837f3e-77e9-4b93-8912-82ec1f446ffa","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:13.972791738Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1268,"session_id":"a3837f3e-77e9-4b93-8912-82ec1f446ffa"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:13.972803807Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1268,"session_id":"a3837f3e-77e9-4b93-8912-82ec1f446ffa"} {"msg":"The guest has finished waiting for activation with:22","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:13.972815957Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1268} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 274 len 16384 data: 4 5 5 4 02/30 Write block 186 len 20480 data: 3 3 3 2 2 03/30 Read block 385 len 36864 04/30 Write block 272 len 12288 data: 4 4 5 05/30 Write block 332 len 36864 data: 3 3 4 5 5 5 5 5 5 06/30 Write block 410 len 24576 data: 3 5 4 3 4 4 07/30 Flush 08/30 Write block 391 len 16384 data: 2 2 2 2 09/30 Write block 284 len 36864 data: 3 4 6 6 7 6 5 4 5 10/30 Read block 291 len 12288 11/30 Flush 12/30 Write block 54 len 12288 data: 2 2 3 13/30 Read block 445 len 16384 14/30 Flush 15/30 Flush 16/30 Write block 262 len 40960 data: 2 2 3 4 3 2 2 3 3 4 17/30 Write block 180 len 28672 data: 4 4 4 4 3 3 4 18/30 Flush 19/30 Write block 341 len 12288 data: 5 5 4 20/30 Write block 183 len 20480 data: 5 4 4 5 4 21/30 Read block 202 len 36864 22/30 Read block 457 len 12288 23/30 Flush 24/30 Write block 368 len 20480 data: 6 4 4 4 5 25/30 Read block 532 len 36864 26/30 Write block 378 len 32768 data: 2 2 2 2 2 2 2 2 27/30 Read block 177 len 20480 28/30 Write block 50 len 40960 data: 3 3 2 2 3 3 4 3 3 3 29/30 Write block 227 len 16384 data: 2 2 2 3 30/30 Read block 573 len 36864 ---------------------------------------------------------------- Crucible gen:22 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 5 Done Done Done false 1024 Acked Read 9 Done Done Done false 1025 Acked Write 8 Done Done Done false 1026 Acked Read 5 Done Done Done false 1027 Acked Write 10 Done Done Done false 1028 Acked Write 4 Done Sent Done false 1029 Acked Read 9 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 2 0 3 Done 6 5 7 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:14.005583725Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1268} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 21 21 21 35 35 35 F F F 1 020-039 20 20 20 33 33 33 F F F 2 040-059 22 22 22 38 38 38 T T T 3 060-079 19 19 19 32 32 32 F F F 4 080-099 14 14 14 20 20 20 F F F 5 100-119 20 20 20 34 34 34 F F F 6 120-139 20 20 20 34 34 34 F F F 7 140-159 21 21 21 35 35 35 F F F 8 160-179 10 10 10 15 15 15 F F F 9 180-199 22 22 22 41 41 41 F F F 10 200-219 17 17 17 28 28 28 F F F 11 220-239 19 19 19 32 32 32 T T T 12 240-259 20 20 20 33 33 33 F F F 13 260-279 22 22 22 40 40 40 F F F 14 280-299 22 22 22 37 37 37 F F F 15 300-319 20 20 20 33 33 33 F F F 16 320-339 22 22 22 36 36 36 F F F 17 340-359 22 22 22 41 41 41 F F F 18 360-379 18 18 18 31 31 31 T T T 19 380-399 22 22 22 37 37 37 T T T 20 400-419 22 22 22 36 36 36 F F F 21 420-439 12 12 12 18 18 18 F F F 22 440-459 12 12 12 17 17 17 F F F 23 460-479 12 12 12 18 18 18 F F F 24 480-499 14 14 14 21 21 21 F F F 25 500-519 16 16 16 26 26 26 F F F 26 520-539 21 21 21 35 35 35 F F F 27 540-559 21 21 21 35 35 35 F F F 28 560-579 15 15 15 24 24 24 F F F 29 580-599 20 20 20 33 33 33 F F F Max gen: 22, Max flush: 41 A Difference in extent metadata is expected here On loop 11 Verifying data now No change after verify Loop: 11 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 21 21 21 35 35 35 F F F 1 020-039 20 20 20 33 33 33 F F F 2 040-059 23 23 23 42 42 42 F F F 3 060-079 19 19 19 32 32 32 F F F 4 080-099 14 14 14 20 20 20 F F F 5 100-119 20 20 20 34 34 34 F F F 6 120-139 20 20 20 34 34 34 F F F 7 140-159 21 21 21 35 35 35 F F F 8 160-179 10 10 10 15 15 15 F F F 9 180-199 22 22 22 41 41 41 F F F 10 200-219 17 17 17 28 28 28 F F F 11 220-239 23 23 23 42 42 42 F F F 12 240-259 20 20 20 33 33 33 F F F 13 260-279 22 22 22 40 40 40 F F F 14 280-299 22 22 22 37 37 37 F F F 15 300-319 20 20 20 33 33 33 F F F 16 320-339 22 22 22 36 36 36 F F F 17 340-359 22 22 22 41 41 41 F F F 18 360-379 23 23 23 42 42 42 F F F 19 380-399 23 23 23 42 42 42 F F F 20 400-419 22 22 22 36 36 36 F F F 21 420-439 12 12 12 18 18 18 F F F 22 440-459 12 12 12 17 17 17 F F F 23 460-479 12 12 12 18 18 18 F F F 24 480-499 14 14 14 21 21 21 F F F 25 500-519 16 16 16 26 26 26 F F F 26 520-539 21 21 21 35 35 35 F F F 27 540-559 21 21 21 35 35 35 F F F 28 560-579 15 15 15 24 24 24 F F F 29 580-599 20 20 20 33 33 33 F F F Max gen: 23, Max flush: 42 Begin loop 12 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:18.308820479Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1276,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:18.353665317Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1276,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:18.354359143Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1276} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"0fd98d0d5d909611f8b408ccda2559d35fe117bb\",\n git_commit_timestamp: \"2024-11-08T20:05:44.000000000Z\",\n git_branch: \"mkeeter/drop-skipped-backpressure-guard\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:18.354395871Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1276} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:18.354410101Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1276} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:18.3544244Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1276} {"msg":"Crucible 62ac5c1f-5ff8-4283-b652-752f9fcdb3ba has session id: 3318d024-6fa8-401a-a4ef-059049b6ead0","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:18.35483744Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1276,"session_id":"3318d024-6fa8-401a-a4ef-059049b6ead0"} {"msg":"Upstairs opts: Upstairs UUID: 62ac5c1f-5ff8-4283-b652-752f9fcdb3ba, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:18.354909856Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1276,"session_id":"3318d024-6fa8-401a-a4ef-059049b6ead0"} {"msg":"Crucible stats registered with UUID: 62ac5c1f-5ff8-4283-b652-752f9fcdb3ba","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:18.354935955Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1276,"session_id":"3318d024-6fa8-401a-a4ef-059049b6ead0"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:18.386777849Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1276,"task":"crutest"} {"msg":"The guest has requested activation with gen:24","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:18.386818087Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1276} {"msg":"62ac5c1f-5ff8-4283-b652-752f9fcdb3ba active request set","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:18.386848945Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1276,"session_id":"3318d024-6fa8-401a-a4ef-059049b6ead0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:18.386865835Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1276,"client":"0","":"downstairs","session_id":"3318d024-6fa8-401a-a4ef-059049b6ead0"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:18.386893013Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1276,"client":"0","":"downstairs","session_id":"3318d024-6fa8-401a-a4ef-059049b6ead0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:18.386910642Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1276,"client":"1","":"downstairs","session_id":"3318d024-6fa8-401a-a4ef-059049b6ead0"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:18.386931951Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1276,"client":"1","":"downstairs","session_id":"3318d024-6fa8-401a-a4ef-059049b6ead0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:18.38696401Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1276,"client":"2","":"downstairs","session_id":"3318d024-6fa8-401a-a4ef-059049b6ead0"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:18.386997228Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1276,"client":"2","":"downstairs","session_id":"3318d024-6fa8-401a-a4ef-059049b6ead0"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:18.387020937Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1276,"":"io task","client":"2","":"downstairs","session_id":"3318d024-6fa8-401a-a4ef-059049b6ead0"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:18.387052695Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1276,"":"io task","client":"0","":"downstairs","session_id":"3318d024-6fa8-401a-a4ef-059049b6ead0"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:18.387085734Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1276,"":"io task","client":"1","":"downstairs","session_id":"3318d024-6fa8-401a-a4ef-059049b6ead0"} {"msg":"ds_connection connected from Ok(127.0.0.1:49900)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:18.387119602Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1276,"":"io task","client":"2","":"downstairs","session_id":"3318d024-6fa8-401a-a4ef-059049b6ead0"} {"msg":"ds_connection connected from Ok(127.0.0.1:63185)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:18.387248886Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1276,"":"io task","client":"1","":"downstairs","session_id":"3318d024-6fa8-401a-a4ef-059049b6ead0"} {"msg":"ds_connection connected from Ok(127.0.0.1:38329)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:18.387304843Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1276,"":"io task","client":"0","":"downstairs","session_id":"3318d024-6fa8-401a-a4ef-059049b6ead0"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:18.3879843Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1276,"client":"0","":"downstairs","session_id":"3318d024-6fa8-401a-a4ef-059049b6ead0"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:18.388013008Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1276,"client":"0","":"downstairs","session_id":"3318d024-6fa8-401a-a4ef-059049b6ead0"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:18.388027648Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1276,"client":"1","":"downstairs","session_id":"3318d024-6fa8-401a-a4ef-059049b6ead0"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:18.388041407Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1276,"client":"1","":"downstairs","session_id":"3318d024-6fa8-401a-a4ef-059049b6ead0"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:18.388054816Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1276,"client":"2","":"downstairs","session_id":"3318d024-6fa8-401a-a4ef-059049b6ead0"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:18.388068216Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1276,"client":"2","":"downstairs","session_id":"3318d024-6fa8-401a-a4ef-059049b6ead0"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:18.388678896Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1276,"client":"1","":"downstairs","session_id":"3318d024-6fa8-401a-a4ef-059049b6ead0"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:18.388708454Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1276,"client":"0","":"downstairs","session_id":"3318d024-6fa8-401a-a4ef-059049b6ead0"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:18.388723834Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1276,"client":"2","":"downstairs","session_id":"3318d024-6fa8-401a-a4ef-059049b6ead0"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:18.389229579Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1276,"client":"1","":"downstairs","session_id":"3318d024-6fa8-401a-a4ef-059049b6ead0"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:18.389258458Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1276,"client":"1","":"downstairs","session_id":"3318d024-6fa8-401a-a4ef-059049b6ead0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:18.389273937Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1276,"session_id":"3318d024-6fa8-401a-a4ef-059049b6ead0"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:18.389344813Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1276,"client":"0","":"downstairs","session_id":"3318d024-6fa8-401a-a4ef-059049b6ead0"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:18.389365842Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1276,"client":"0","":"downstairs","session_id":"3318d024-6fa8-401a-a4ef-059049b6ead0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:18.389383441Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1276,"session_id":"3318d024-6fa8-401a-a4ef-059049b6ead0"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:18.389396161Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1276,"client":"2","":"downstairs","session_id":"3318d024-6fa8-401a-a4ef-059049b6ead0"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:18.38940967Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1276,"client":"2","":"downstairs","session_id":"3318d024-6fa8-401a-a4ef-059049b6ead0"} {"msg":"[0]R flush_numbers[0..12]: [35, 33, 42, 32, 20, 34, 34, 35, 15, 41, 28, 42]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:18.389423599Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1276,"":"downstairs","":"downstairs","session_id":"3318d024-6fa8-401a-a4ef-059049b6ead0"} {"msg":"[0]R generation[0..12]: [21, 20, 23, 19, 14, 20, 20, 21, 10, 22, 17, 23]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:18.389437349Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1276,"":"downstairs","":"downstairs","session_id":"3318d024-6fa8-401a-a4ef-059049b6ead0"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:18.389451578Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1276,"":"downstairs","":"downstairs","session_id":"3318d024-6fa8-401a-a4ef-059049b6ead0"} {"msg":"[1]R flush_numbers[0..12]: [35, 33, 42, 32, 20, 34, 34, 35, 15, 41, 28, 42]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:18.389465257Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1276,"":"downstairs","":"downstairs","session_id":"3318d024-6fa8-401a-a4ef-059049b6ead0"} {"msg":"[1]R generation[0..12]: [21, 20, 23, 19, 14, 20, 20, 21, 10, 22, 17, 23]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:18.389478847Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1276,"":"downstairs","":"downstairs","session_id":"3318d024-6fa8-401a-a4ef-059049b6ead0"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:18.389492636Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1276,"":"downstairs","":"downstairs","session_id":"3318d024-6fa8-401a-a4ef-059049b6ead0"} {"msg":"[2]R flush_numbers[0..12]: [35, 33, 42, 32, 20, 34, 34, 35, 15, 41, 28, 42]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:18.389506486Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1276,"":"downstairs","":"downstairs","session_id":"3318d024-6fa8-401a-a4ef-059049b6ead0"} {"msg":"[2]R generation[0..12]: [21, 20, 23, 19, 14, 20, 20, 21, 10, 22, 17, 23]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:18.389530784Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1276,"":"downstairs","":"downstairs","session_id":"3318d024-6fa8-401a-a4ef-059049b6ead0"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:18.389559123Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1276,"":"downstairs","":"downstairs","session_id":"3318d024-6fa8-401a-a4ef-059049b6ead0"} {"msg":"Max found gen is 24","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:18.389584532Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1276,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO 3318d024-6fa8-401a-a4ef-059049b6ead0"} {"msg":"Generation requested: 24 >= found:24","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:18.38961835Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1276,"":"downstairs","":"downstairs","session_id":"3318d024-6fa8-401a-a4ef-059049b6ead0"} {"msg":"Next flush: 43","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:18.389650418Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1276,"":"downstairs","":"downstairs","session_id":"3318d024-6fa8-401a-a4ef-059049b6ead0"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:18.389677507Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1276,"":"downstairs","":"downstairs","session_id":"3318d024-6fa8-401a-a4ef-059049b6ead0"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:18.389705216Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1276,"session_id":"3318d024-6fa8-401a-a4ef-059049b6ead0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:18.389720485Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1276,"client":"0","":"downstairs","session_id":"3318d024-6fa8-401a-a4ef-059049b6ead0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:18.389734504Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1276,"client":"1","":"downstairs","session_id":"3318d024-6fa8-401a-a4ef-059049b6ead0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:18.389758843Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1276,"client":"2","":"downstairs","session_id":"3318d024-6fa8-401a-a4ef-059049b6ead0"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:18.389785222Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1276,"session_id":"3318d024-6fa8-401a-a4ef-059049b6ead0"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:18.389809751Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1276,"session_id":"3318d024-6fa8-401a-a4ef-059049b6ead0"} {"msg":"62ac5c1f-5ff8-4283-b652-752f9fcdb3ba is now active with session: 3318d024-6fa8-401a-a4ef-059049b6ead0","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:18.38982473Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1276,"session_id":"3318d024-6fa8-401a-a4ef-059049b6ead0"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:18.389843549Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1276,"session_id":"3318d024-6fa8-401a-a4ef-059049b6ead0"} {"msg":"The guest has finished waiting for activation with:24","v":0,"name":"crucible","level":30Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2024-11-08T20:23:18.389866078Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1276} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 258 len 36864 data: 4 3 3 2 3 3 4 5 4 03/30 Read block 27 len 12288 04/30 Flush 05/30 Write block 388 len 12288 data: 2 2 2 06/30 Read block 144 len 16384 07/30 Write block 274 len 36864 data: 6 6 6 5 3 3 3 2 3 08/30 Read block 229 len 24576 09/30 Write block 93 len 4096 data: 3 10/30 Flush 11/30 Write block 149 len 28672 data: 3 3 4 3 3 3 2 12/30 Write block 138 len 40960 data: 3 3 3 3 3 4 4 3 3 3 13/30 Read block 564 len 36864 14/30 Read block 463 len 16384 15/30 Read block 195 len 36864 16/30 Read block 31 len 16384 17/30 Read block 207 len 36864 18/30 Write block 552 len 4096 data: 3 19/30 Read block 222 len 36864 20/30 Read block 313 len 8192 21/30 Read block 346 len 16384 22/30 Write block 325 len 8192 data: 2 2 23/30 Read block 35 len 32768 24/30 Write block 225 len 12288 data: 2 2 3 25/30 Write block 268 len 4096 data: 3 26/30 Read block 36 len 28672 27/30 Read block 450 len 32768 28/30 Read block 448 len 16384 29/30 Read block 159 len 28672 30/30 Write block 254 len 8192 data: 4 3 ---------------------------------------------------------------- Crucible gen:24 GIO:true work queues: Upstairs:0 downstairs:20 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1010 Acked Write 7 Done Done Done false 1011 Acked Write 10 Done Done Done false 1012 Acked Read 9 Done Done Done false 1013 Acked Read 4 Done Done Done false 1014 Acked Read 9 Done Done Done false 1015 Acked Read 4 Done Done Done false 1016 Acked Read 9 Done Done Done false 1017 Acked Write 1 Done Done Done false 1018 Acked Read 9 Done Done Done false 1019 Acked Read 2 Done Done Done false 1020 Acked Read 4 Done Done Done false 1021 Acked Write 2 Done Done Done false 1022 Acked Read 8 Done Done Done false 1023 Acked Write 3 Done Done Done false 1024 Acked Write 1 Done Done Done false 1025 Acked Read 7 Done Done Done false 1026 Acked Read 8 Done Done Done false 1027 Acked Read 4 Done Done Done false 1028 Acked Read 7 Done Done Done false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 19 19 19 57 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1009 1009 1009 Downstairs last five completed: 1009 1008 1007 1006 1005 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:18.413390728Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1276} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 21 21 21 35 35 35 F F F 1 020-039 20 20 20 33 33 33 F F F 2 040-059 23 23 23 42 42 42 F F F 3 060-079 19 19 19 32 32 32 F F F 4 080-099 24 24 24 45 45 45 F F F 5 100-119 20 20 20 34 34 34 F F F 6 120-139 20 20 20 34 34 34 T T T 7 140-159 21 21 21 35 35 35 T T T 8 160-179 10 10 10 15 15 15 F F F 9 180-199 22 22 22 41 41 41 F F F 10 200-219 17 17 17 28 28 28 F F F 11 220-239 23 23 23 42 42 42 T T T 12 240-259 24 24 24 44 44 44 T T T 13 260-279 24 24 24 45 45 45 T T T 14 280-299 24 24 24 45 45 45 F F F 15 300-319 20 20 20 33 33 33 F F F 16 320-339 22 22 22 36 36 36 T T T 17 340-359 22 22 22 41 41 41 F F F 18 360-379 23 23 23 42 42 42 F F F 19 380-399 24 24 24 45 45 45 F F F 20 400-419 22 22 22 36 36 36 F F F 21 420-439 12 12 12 18 18 18 F F F 22 440-459 12 12 12 17 17 17 F F F 23 460-479 12 12 12 18 18 18 F F F 24 480-499 14 14 14 21 21 21 F F F 25 500-519 16 16 16 26 26 26 F F F 26 520-539 21 21 21 35 35 35 F F F 27 540-559 21 21 21 35 35 35 T T T 28 560-579 15 15 15 24 24 24 F F F 29 580-599 20 20 20 33 33 33 F F F Max gen: 24, Max flush: 45 A Difference in extent metadata is expected here On loop 12 Verifying data now No change after verify Loop: 12 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 21 21 21 35 35 35 F F F 1 020-039 20 20 20 33 33 33 F F F 2 040-059 23 23 23 42 42 42 F F F 3 060-079 19 19 19 32 32 32 F F F 4 080-099 24 24 24 45 45 45 F F F 5 100-119 20 20 20 34 34 34 F F F 6 120-139 25 25 25 46 46 46 F F F 7 140-159 25 25 25 46 46 46 F F F 8 160-179 10 10 10 15 15 15 F F F 9 180-199 22 22 22 41 41 41 F F F 10 200-219 17 17 17 28 28 28 F F F 11 220-239 25 25 25 46 46 46 F F F 12 240-259 25 25 25 46 46 46 F F F 13 260-279 25 25 25 46 46 46 F F F 14 280-299 24 24 24 45 45 45 F F F 15 300-319 20 20 20 33 33 33 F F F 16 320-339 25 25 25 46 46 46 F F F 17 340-359 22 22 22 41 41 41 F F F 18 360-379 23 23 23 42 42 42 F F F 19 380-399 24 24 24 45 45 45 F F F 20 400-419 22 22 22 36 36 36 F F F 21 420-439 12 12 12 18 18 18 F F F 22 440-459 12 12 12 17 17 17 F F F 23 460-479 12 12 12 18 18 18 F F F 24 480-499 14 14 14 21 21 21 F F F 25 500-519 16 16 16 26 26 26 F F F 26 520-539 21 21 21 35 35 35 F F F 27 540-559 25 25 25 46 46 46 F F F 28 560-579 15 15 15 24 24 24 F F F 29 580-599 20 20 20 33 33 33 F F F Max gen: 25, Max flush: 46 Begin loop 13 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:22.762347128Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1284,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:22.807436973Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1284,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:22.808079102Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1284} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"0fd98d0d5d909611f8b408ccda2559d35fe117bb\",\n git_commit_timestamp: \"2024-11-08T20:05:44.000000000Z\",\n git_branch: \"mkeeter/drop-skipped-backpressure-guard\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:22.808141529Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1284} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:22.808172908Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1284} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:22.808203146Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1284} {"msg":"Crucible d07f1e34-9ec6-40cb-8f22-04f37eab75ec has session id: 40da0a10-45e3-499f-92d3-611e292632cc","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:22.808710721Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1284,"session_id":"40da0a10-45e3-499f-92d3-611e292632cc"} {"msg":"Upstairs opts: Upstairs UUID: d07f1e34-9ec6-40cb-8f22-04f37eab75ec, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:22.808761249Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1284,"session_id":"40da0a10-45e3-499f-92d3-611e292632cc"} {"msg":"Crucible stats registered with UUID: d07f1e34-9ec6-40cb-8f22-04f37eab75ec","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:22.808780908Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1284,"session_id":"40da0a10-45e3-499f-92d3-611e292632cc"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:22.840496578Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1284,"task":"crutest"} {"msg":"The guest has requested activation with gen:26","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:22.840528836Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1284} {"msg":"d07f1e34-9ec6-40cb-8f22-04f37eab75ec active request set","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:22.840564894Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1284,"session_id":"40da0a10-45e3-499f-92d3-611e292632cc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:22.840581513Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1284,"client":"0","":"downstairs","session_id":"40da0a10-45e3-499f-92d3-611e292632cc"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:22.840610352Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1284,"client":"0","":"downstairs","session_id":"40da0a10-45e3-499f-92d3-611e292632cc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:22.840627861Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1284,"client":"1","":"downstairs","session_id":"40da0a10-45e3-499f-92d3-611e292632cc"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:22.84064499Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1284,"client":"1","":"downstairs","session_id":"40da0a10-45e3-499f-92d3-611e292632cc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:22.84066196Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1284,"client":"2","":"downstairs","session_id":"40da0a10-45e3-499f-92d3-611e292632cc"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:22.840678609Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1284,"client":"2","":"downstairs","session_id":"40da0a10-45e3-499f-92d3-611e292632cc"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:22.840695288Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1284,"":"io task","client":"2","":"downstairs","session_id":"40da0a10-45e3-499f-92d3-611e292632cc"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:22.840712387Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1284,"":"io task","client":"0","":"downstairs","session_id":"40da0a10-45e3-499f-92d3-611e292632cc"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:22.840729496Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1284,"":"io task","client":"1","":"downstairs","session_id":"40da0a10-45e3-499f-92d3-611e292632cc"} {"msg":"ds_connection connected from Ok(127.0.0.1:39144)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:22.840910177Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1284,"":"io task","client":"0","":"downstairs","session_id":"40da0a10-45e3-499f-92d3-611e292632cc"} {"msg":"ds_connection connected from Ok(127.0.0.1:61295)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:22.840962285Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1284,"":"io task","client":"1","":"downstairs","session_id":"40da0a10-45e3-499f-92d3-611e292632cc"} {"msg":"ds_connection connected from Ok(127.0.0.1:55637)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:22.840998433Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1284,"":"io task","client":"2","":"downstairs","session_id":"40da0a10-45e3-499f-92d3-611e292632cc"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:22.841787395Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1284,"client":"2","":"downstairs","session_id":"40da0a10-45e3-499f-92d3-611e292632cc"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:22.841818893Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1284,"client":"2","":"downstairs","session_id":"40da0a10-45e3-499f-92d3-611e292632cc"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:22.841835122Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1284,"client":"0","":"downstairs","session_id":"40da0a10-45e3-499f-92d3-611e292632cc"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:22.841849302Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1284,"client":"0","":"downstairs","session_id":"40da0a10-45e3-499f-92d3-611e292632cc"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:22.841862751Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1284,"client":"1","":"downstairs","session_id":"40da0a10-45e3-499f-92d3-611e292632cc"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:22.841914528Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1284,"client":"1","":"downstairs","session_id":"40da0a10-45e3-499f-92d3-611e292632cc"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:22.842478811Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1284,"client":"0","":"downstairs","session_id":"40da0a10-45e3-499f-92d3-611e292632cc"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:22.842625004Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1284,"client":"1","":"downstairs","session_id":"40da0a10-45e3-499f-92d3-611e292632cc"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:22.842654322Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1284,"client":"2","":"downstairs","session_id":"40da0a10-45e3-499f-92d3-611e292632cc"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:22.842961237Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1284,"client":"0","":"downstairs","session_id":"40da0a10-45e3-499f-92d3-611e292632cc"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:22.843016645Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1284,"client":"0","":"downstairs","session_id":"40da0a10-45e3-499f-92d3-611e292632cc"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:22.843044473Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1284,"session_id":"40da0a10-45e3-499f-92d3-611e292632cc"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:22.843139948Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1284,"client":"2","":"downstairs","session_id":"40da0a10-45e3-499f-92d3-611e292632cc"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:22.843169687Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1284,"client":"2","":"downstairs","session_id":"40da0a10-45e3-499f-92d3-611e292632cc"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:22.843184996Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1284,"session_id":"40da0a10-45e3-499f-92d3-611e292632cc"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:22.843199836Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1284,"client":"1","":"downstairs","session_id":"40da0a10-45e3-499f-92d3-611e292632cc"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:22.843216635Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1284,"client":"1","":"downstairs","session_id":"40da0a10-45e3-499f-92d3-611e292632cc"} {"msg":"[0]R flush_numbers[0..12]: [35, 33, 42, 32, 45, 34, 46, 46, 15, 41, 28, 46]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:22.843260073Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1284,"":"downstairs","":"downstairs","session_id":"40da0a10-45e3-499f-92d3-611e292632cc"} {"msg":"[0]R generation[0..12]: [21, 20, 23, 19, 24, 20, 25, 25, 10, 22, 17, 25]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:22.843278382Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1284,"":"downstairs","":"downstairs","session_id":"40da0a10-45e3-499f-92d3-611e292632cc"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:22.843292861Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1284,"":"downstairs","":"downstairs","session_id":"40da0a10-45e3-499f-92d3-611e292632cc"} {"msg":"[1]R flush_numbers[0..12]: [35, 33, 42, 32, 45, 34, 46, 46, 15, 41, 28, 46]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:22.8433067Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1284,"":"downstairs","":"downstairs","session_id":"40da0a10-45e3-499f-92d3-611e292632cc"} {"msg":"[1]R generation[0..12]: [21, 20, 23, 19, 24, 20, 25, 25, 10, 22, 17, 25]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:22.84332034Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1284,"":"downstairs","":"downstairs","session_id":"40da0a10-45e3-499f-92d3-611e292632cc"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:22.843334769Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1284,"":"downstairs","":"downstairs","session_id":"40da0a10-45e3-499f-92d3-611e292632cc"} {"msg":"[2]R flush_numbers[0..12]: [35, 33, 42, 32, 45, 34, 46, 46, 15, 41, 28, 46]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:22.843348398Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1284,"":"downstairs","":"downstairs","session_id":"40da0a10-45e3-499f-92d3-611e292632cc"} {"msg":"[2]R generation[0..12]: [21, 20, 23, 19, 24, 20, 25, 25, 10, 22, 17, 25]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:22.843362098Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1284,"":"downstairs","":"downstairs","session_id":"40da0a10-45e3-499f-92d3-611e292632cc"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:22.843375977Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1284,"":"downstairs","":"downstairs","session_id":"40da0a10-45e3-499f-92d3-611e292632cc"} {"msg":"Max found gen is 26","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:22.843389026Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1284,"":"downstairs","":"downstairs","session_id":"40da0a10-45e3-499f-92d3-611e292632cc"} {"msg":"Generation requested: 26 >= found:26","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:22.843402286Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1284,"":"downstairs","":"downstairs","session_id":"40da0a10-45e3-499f-92d3-611e292632cc"} {"msg":"Next flush: 47","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:22.843415415Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1284,"":"downstairs","":"downstairs","session_id":"40da0a10-45e3-499f-92d3-611e292632cc"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:22.843428485Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1284,"":"downstairs","":"downstairs","session_id":"40da0a10-45e3-499f-92d3-611e292632cc"} Wait for a query_work_queue command to finish before sending IO {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:22.843453753Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1284,"session_id":"40da0a10-45e3-499f-92d3-611e292632cc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:22.843466692Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1284,"client":"0","":"downstairs","session_id":"40da0a10-45e3-499f-92d3-611e292632cc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:22.843480872Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1284,"client":"1","":"downstairs","session_id":"40da0a10-45e3-499f-92d3-611e292632cc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:22.843494051Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1284,"client":"2","":"downstairs","session_id":"40da0a10-45e3-499f-92d3-611e292632cc"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:22.843507221Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1284,"session_id":"40da0a10-45e3-499f-92d3-611e292632cc"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:22.84351936Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1284,"session_id":"40da0a10-45e3-499f-92d3-611e292632cc"} {"msg":"d07f1e34-9ec6-40cb-8f22-04f37eab75ec is now active with session: 40da0a10-45e3-499f-92d3-611e292632cc","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:22.843532069Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1284,"session_id":"40da0a10-45e3-499f-92d3-611e292632cc"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:22.843544009Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1284,"session_id":"40da0a10-45e3-499f-92d3-611e292632cc"} {"msg":"The guest has finished waiting for activation with:26","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:22.843556198Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1284} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 543 len 8192 data: 4 4 02/30 Write block 446 len 36864 data: 3 3 3 3 3 4 4 3 3 03/30 Read block 512 len 8192 04/30 Read block 523 len 32768 05/30 Read block 486 len 24576 06/30 Write block 218 len 8192 data: 2 2 07/30 Write block 19 len 20480 data: 3 4 5 5 5 08/30 Read block 498 len 4096 09/30 Flush 10/30 Write block 24 len 12288 data: 6 6 6 11/30 Write block 369 len 4096 data: 5 12/30 Read block 44 len 20480 13/30 Write block 421 len 12288 data: 4 3 4 14/30 Read block 351 len 4096 15/30 Read block 134 len 12288 16/30 Write block 137 len 24576 data: 4 4 4 4 4 4 17/30 Write block 185 len 40960 data: 5 6 5 4 3 3 2 3 3 3 18/30 Flush 19/30 Write block 214 len 20480 data: 2 2 2 2 3 20/30 Read block 54 len 16384 21/30 Read block 123 len 40960 22/30 Read block 382 len 32768 23/30 Flush 24/30 Write block 227 len 20480 data: 4 3 3 4 4 25/30 Read block 91 len 40960 26/30 Read block 507 len 28672 27/30 Read block 364 len 24576 28/30 Read block 576 len 16384 29/30 Read block 60 len 20480 30/30 Read block 235 len 8192 ---------------------------------------------------------------- Crucible gen:26 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 5 Done Done Done false 1024 Acked Read 10 Done Done Done false 1025 Acked Read 7 Done Done Done false 1026 Acked Read 6 Done Done Done false 1027 Acked Read 4 Done Done Done false 1028 Acked Read 5 Done Done Done false 1029 Acked Read 2 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 0 1 Done 7 6 7 20 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:22.869033703Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1284} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 26 26 26 47 47 47 F F F 1 020-039 26 26 26 48 48 48 F F F 2 040-059 23 23 23 42 42 42 F F F 3 060-079 19 19 19 32 32 32 F F F 4 080-099 24 24 24 45 45 45 F F F 5 100-119 20 20 20 34 34 34 F F F 6 120-139 26 26 26 48 48 48 F F F 7 140-159 26 26 26 48 48 48 F F F 8 160-179 10 10 10 15 15 15 F F F 9 180-199 26 26 26 48 48 48 F F F 10 200-219 26 26 26 49 49 49 F F F 11 220-239 25 25 25 46 46 46 T T T 12 240-259 25 25 25 46 46 46 F F F 13 260-279 25 25 25 46 46 46 F F F 14 280-299 24 24 24 45 45 45 F F F 15 300-319 20 20 20 33 33 33 F F F 16 320-339 25 25 25 46 46 46 F F F 17 340-359 22 22 22 41 41 41 F F F 18 360-379 26 26 26 48 48 48 F F F 19 380-399 24 24 24 45 45 45 F F F 20 400-419 22 22 22 36 36 36 F F F 21 420-439 26 26 26 48 48 48 F F F 22 440-459 26 26 26 47 47 47 F F F 23 460-479 12 12 12 18 18 18 F F F 24 480-499 14 14 14 21 21 21 F F F 25 500-519 16 16 16 26 26 26 F F F 26 520-539 21 21 21 35 35 35 F F F 27 540-559 26 26 26 47 47 47 F F F 28 560-579 15 15 15 24 24 24 F F F 29 580-599 20 20 20 33 33 33 F F F Max gen: 26, Max flush: 49 A Difference in extent metadata is expected here On loop 13 Verifying data now No change after verify Loop: 13 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 26 26 26 47 47 47 F F F 1 020-039 26 26 26 48 48 48 F F F 2 040-059 23 23 23 42 42 42 F F F 3 060-079 19 19 19 32 32 32 F F F 4 080-099 24 24 24 45 45 45 F F F 5 100-119 20 20 20 34 34 34 F F F 6 120-139 26 26 26 48 48 48 F F F 7 140-159 26 26 26 48 48 48 F F F 8 160-179 10 10 10 15 15 15 F F F 9 180-199 26 26 26 48 48 48 F F F 10 200-219 26 26 26 49 49 49 F F F 11 220-239 27 27 27 50 50 50 F F F 12 240-259 25 25 25 46 46 46 F F F 13 260-279 25 25 25 46 46 46 F F F 14 280-299 24 24 24 45 45 45 F F F 15 300-319 20 20 20 33 33 33 F F F 16 320-339 25 25 25 46 46 46 F F F 17 340-359 22 22 22 41 41 41 F F F 18 360-379 26 26 26 48 48 48 F F F 19 380-399 24 24 24 45 45 45 F F F 20 400-419 22 22 22 36 36 36 F F F 21 420-439 26 26 26 48 48 48 F F F 22 440-459 26 26 26 47 47 47 F F F 23 460-479 12 12 12 18 18 18 F F F 24 480-499 14 14 14 21 21 21 F F F 25 500-519 16 16 16 26 26 26 F F F 26 520-539 21 21 21 35 35 35 F F F 27 540-559 26 26 26 47 47 47 F F F 28 560-579 15 15 15 24 24 24 F F F 29 580-599 20 20 20 33 33 33 F F F Max gen: 27, Max flush: 50 Begin loop 14 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:27.130154425Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1292,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:27.173763293Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1292,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:27.174183013Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1292,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:27.206841467Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1292,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:27.207409779Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1292} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"0fd98d0d5d909611f8b408ccda2559d35fe117bb\",\n git_commit_timestamp: \"2024-11-08T20:05:44.000000000Z\",\n git_branch: \"mkeeter/drop-skipped-backpressure-guard\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:27.207456057Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1292} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:27.207474606Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1292} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:27.207492635Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1292} {"msg":"Crucible f1966c81-e4c0-49af-bfad-80c47d381d7c has session id: bad83c78-3279-4c37-9a69-461da94055aa","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:27.207967492Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1292,"session_id":"bad83c78-3279-4c37-9a69-461da94055aa"} {"msg":"Upstairs opts: Upstairs UUID: f1966c81-e4c0-49af-bfad-80c47d381d7c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:27.20800328Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1292,"session_id":"bad83c78-3279-4c37-9a69-461da94055aa"} {"msg":"Crucible stats registered with UUID: f1966c81-e4c0-49af-bfad-80c47d381d7c","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:27.208020369Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1292,"session_id":"bad83c78-3279-4c37-9a69-461da94055aa"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:27.238871711Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1292,"task":"crutest"} {"msg":"The guest has requested activation with gen:28","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:27.23890174Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1292} {"msg":"f1966c81-e4c0-49af-bfad-80c47d381d7c active request set","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:27.238971107Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1292,"session_id":"bad83c78-3279-4c37-9a69-461da94055aa"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:27.238998985Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1292,"client":"0","":"downstairs","session_id":"bad83c78-3279-4c37-9a69-461da94055aa"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:27.239029414Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1292,"client":"0","":"downstairs","session_id":"bad83c78-3279-4c37-9a69-461da94055aa"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:27.239062062Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1292,"client":"1","":"downstairs","session_id":"bad83c78-3279-4c37-9a69-461da94055aa"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:27.23909618Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1292,"client":"1","":"downstairs","session_id":"bad83c78-3279-4c37-9a69-461da94055aa"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:27.239131419Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1292,"client":"2","":"downstairs","session_id":"bad83c78-3279-4c37-9a69-461da94055aa"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:27.239167407Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1292,"client":"2","":"downstairs","session_id":"bad83c78-3279-4c37-9a69-461da94055aa"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:27.239204575Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1292,"":"io task","client":"2","":"downstairs","session_id":"bad83c78-3279-4c37-9a69-461da94055aa"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:27.239240164Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1292,"":"io task","client":"0","":"downstairs","session_id":"bad83c78-3279-4c37-9a69-461da94055aa"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:27.239274362Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1292,"":"io task","client":"1","":"downstairs","session_id":"bad83c78-3279-4c37-9a69-461da94055aa"} {"msg":"ds_connection connected from Ok(127.0.0.1:54421)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:27.23930784Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1292,"":"io task","client":"0","":"downstairs","session_id":"bad83c78-3279-4c37-9a69-461da94055aa"} {"msg":"ds_connection connected from Ok(127.0.0.1:37002)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:27.239335799Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1292,"":"io task","client":"1","":"downstairs","session_id":"bad83c78-3279-4c37-9a69-461da94055aa"} {"msg":"ds_connection connected from Ok(127.0.0.1:53374)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:27.239483361Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1292,"":"io task","client":"2","":"downstairs","session_id":"bad83c78-3279-4c37-9a69-461da94055aa"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:27.240103551Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1292,"client":"1","":"downstairs","session_id":"bad83c78-3279-4c37-9a69-461da94055aa"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:27.24013207Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1292,"client":"1","":"downstairs","session_id":"bad83c78-3279-4c37-9a69-461da94055aa"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:27.240146839Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1292,"client":"2","":"downstairs","session_id":"bad83c78-3279-4c37-9a69-461da94055aa"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:27.240160568Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1292,"client":"2","":"downstairs","session_id":"bad83c78-3279-4c37-9a69-461da94055aa"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:27.240178498Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1292,"client":"0","":"downstairs","session_id":"bad83c78-3279-4c37-9a69-461da94055aa"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:27.240241384Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1292,"client":"0","":"downstairs","session_id":"bad83c78-3279-4c37-9a69-461da94055aa"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:27.240807087Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1292,"client":"1","":"downstairs","session_id":"bad83c78-3279-4c37-9a69-461da94055aa"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:27.240846995Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1292,"client":"2","":"downstairs","session_id":"bad83c78-3279-4c37-9a69-461da94055aa"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:27.240879593Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1292,"client":"0","":"downstairs","session_id":"bad83c78-3279-4c37-9a69-461da94055aa"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:27.241375289Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1292,"client":"1","":"downstairs","session_id":"bad83c78-3279-4c37-9a69-461da94055aa"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:27.241402138Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1292,"client":"1","":"downstairs","session_id":"bad83c78-3279-4c37-9a69-461da94055aa"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:27.241417267Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1292,"session_id":"bad83c78-3279-4c37-9a69-461da94055aa"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:27.241491303Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1292,"client":"2","":"downstairs","session_id":"bad83c78-3279-4c37-9a69-461da94055aa"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:27.241511633Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1292,"client":"2","":"downstairs","session_id":"bad83c78-3279-4c37-9a69-461da94055aa"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:27.241525592Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1292,"session_id":"bad83c78-3279-4c37-9a69-461da94055aa"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:27.241537901Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1292,"client":"0","":"downstairs","session_id":"bad83c78-3279-4c37-9a69-461da94055aa"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:27.24155132Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1292,"client":"0","":"downstairs","session_id":"bad83c78-3279-4c37-9a69-461da94055aa"} {"msg":"[0]R flush_numbers[0..12]: [47, 48, 42, 32, 45, 34, 48, 48, 15, 48, 49, 50]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:27.24156512Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1292,"":"downstairs","":"downstairs","session_id":"bad83c78-3279-4c37-9a69-461da94055aa"} {"msg":"[0]R generation[0..12]: [26, 26, 23, 19, 24, 20, 26, 26, 10, 26, 26, 27]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:27.241578769Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1292,"":"downstairs","":"downstairs","session_id":"bad83c78-3279-4c37-9a69-461da94055aa"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:27.241596008Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1292,"":"downstairs","":"downstairs","session_id":"bad83c78-3279-4c37-9a69-461da94055aa"} {"msg":"[1]R flush_numbers[0..12]: [47, 48, 42, 32, 45, 34, 48, 48, 15, 48, 49, 50]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:27.241609968Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1292,"":"downstairs","":"downstairs","session_id":"bad83c78-3279-4c37-9a69-461da94055aa"} {"msg":"[1]R generation[0..12]: [26, 26, 23, 19, 24, 20, 26, 26, 10, 26, 26, 27]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:27.241623417Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1292,"":"downstairs","":"downstairs","session_id":"bad83c78-3279-4c37-9a69-461da94055aa"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:27.241637466Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1292,"":"downstairs","":"downstairs","session_id":"bad83c78-3279-4c37-9a69-461da94055aa"} {"msg":"[2]R flush_numbers[0..12]: [47, 48, 42, 32, 45, 34, 48, 48, 15, 48, 49, 50]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:27.241651066Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1292,"":"downstairs","":"downstairs","session_id":"bad83c78-3279-4c37-9a69-461da94055aa"} {"msg":"[2]R generation[0..12]: [26, 26, 23, 19, 24, 20, 26, 26, 10, 26, 26, 27]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:27.241664515Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1292,"":"downstairs","":"downstairs","session_id":"bad83c78-3279-4c37-9a69-461da94055aa"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:27.241678364Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1292,"":"downstairs","":"downstairs","session_id":"bad83c78-3279-4c37-9a69-461da94055aa"} {"msg":"Max found gen is 28","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:27.241691294Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1292,"":"downstairs","":"downstairs","session_id":"bad83c78-3279-4c37-9a69-461da94055aa"} {"msg":"Generation requested: 28 >= found:28","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:27.241704513Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1292,"":"downstairs","":"downstairs","session_id":"bad83c78-3279-4c37-9a69-461da94055aa"} {"msg":"Next flush: 51","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:27.241717472Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1292,"":"downstairs","":"downstairs","session_id":"bad83c78-3279-4c37-9a69-461da94055aa"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:27.241730492Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1292,"":"downstairs","":"downstairs","session_id":"bad83c78-3279-4c37-9a69-461da94055aa"} Wait for a query_work_queue command to finish before sending IO {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:27.241750691Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1292,"session_id":"bad83c78-3279-4c37-9a69-461da94055aa"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:27.24176279Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1292,"client":"0","":"downstairs","session_id":"bad83c78-3279-4c37-9a69-461da94055aa"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:27.24177606Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1292,"client":"1","":"downstairs","session_id":"bad83c78-3279-4c37-9a69-461da94055aa"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:27.241790869Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1292,"client":"2","":"downstairs","session_id":"bad83c78-3279-4c37-9a69-461da94055aa"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:27.241804138Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1292,"session_id":"bad83c78-3279-4c37-9a69-461da94055aa"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:27.241816307Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1292,"session_id":"bad83c78-3279-4c37-9a69-461da94055aa"} {"msg":"f1966c81-e4c0-49af-bfad-80c47d381d7c is now active with session: bad83c78-3279-4c37-9a69-461da94055aa","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:27.241829147Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1292,"session_id":"bad83c78-3279-4c37-9a69-461da94055aa"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:27.241841226Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1292,"session_id":"bad83c78-3279-4c37-9a69-461da94055aa"} {"msg":"The guest has finished waiting for activation with:28","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:27.241853456Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1292} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 177 len 8192 data: 2 2 02/30 Write block 474 len 20480 data: 3 4 4 4 4 03/30 Read block 60 len 8192 04/30 Write block 237 len 32768 data: 4 4 4 5 5 5 5 4 05/30 Write block 75 len 8192 data: 3 3 06/30 Write block 187 len 20480 data: 6 5 4 4 3 07/30 Read block 156 len 20480 08/30 Write block 425 len 32768 data: 3 2 2 2 4 4 4 4 09/30 Read block 221 len 32768 10/30 Read block 289 len 40960 11/30 Read block 360 len 40960 12/30 Write block 183 len 32768 data: 6 5 6 7 7 6 5 5 13/30 Read block 175 len 40960 14/30 Read block 410 len 20480 15/30 Read block 498 len 36864 16/30 Write block 528 len 8192 data: 4 3 17/30 Write block 122 len 20480 data: 4 4 4 2 2 18/30 Read block 553 len 8192 19/30 Write block 277 len 28672 data: 6 4 4 4 3 4 3 20/30 Read block 376 len 28672 21/30 Write block 346 len 8192 data: 5 4 22/30 Write block 39 len 8192 data: 2 2 23/30 Flush 24/30 Write block 498 len 4096 data: 4 25/30 Write block 250 len 16384 data: 5 5 6 4 26/30 Write block 123 len 8192 data: 5 5 27/30 Read block 454 len 4096 28/30 Read block 109 len 4096 29/30 Write block 74 len 20480 data: 4 4 4 3 3 30/30 Write block 379 len 4096 data: 3 ---------------------------------------------------------------- Crucible gen:28 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 1 Done Done Done false 1024 Acked Write 4 Done Done Done false 1025 Acked Write 2 Done Done Done false 1026 Acked Read 1 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Write 5 Sent Sent Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:27.262533715Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1292} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 26 26 26 47 47 47 F F F 1 020-039 28 28 28 51 51 51 F F F 2 040-059 28 28 28 51 51 51 F F F 3 060-079 28 28 28 51 51 51 T T T 4 080-099 24 24 24 45 45 45 F F F 5 100-119 20 20 20 34 34 34 F F F 6 120-139 28 28 28 51 51 51 T T T 7 140-159 26 26 26 48 48 48 F F F 8 160-179 28 28 28 51 51 51 F F F 9 180-199 28 28 28 51 51 51 F F F 10 200-219 26 26 26 49 49 49 F F F 11 220-239 28 28 28 51 51 51 F F F 12 240-259 28 28 28 51 51 51 T T T 13 260-279 28 28 28 51 51 51 F F F 14 280-299 28 28 28 51 51 51 F F F 15 300-319 20 20 20 33 33 33 F F F 16 320-339 25 25 25 46 46 46 F F F 17 340-359 28 28 28 51 51 51 F F F 18 360-379 26 26 26 48 48 48 T T T 19 380-399 24 24 24 45 45 45 F F F 20 400-419 22 22 22 36 36 36 F F F 21 420-439 28 28 28 51 51 51 F F F 22 440-459 26 26 26 47 47 47 F F F 23 460-479 28 28 28 51 51 51 F F F 24 480-499 14 14 14 21 21 21 T T T 25 500-519 16 16 16 26 26 26 F F F 26 520-539 28 28 28 51 51 51 F F F 27 540-559 26 26 26 47 47 47 F F F 28 560-579 15 15 15 24 24 24 F F F 29 580-599 20 20 20 33 33 33 F F F Max gen: 28, Max flush: 51 A Difference in extent metadata is expected here On loop 14 Verifying data now No change after verify Loop: 14 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 26 26 26 47 47 47 F F F 1 020-039 28 28 28 51 51 51 F F F 2 040-059 28 28 28 51 51 51 F F F 3 060-079 29 29 29 52 52 52 F F F 4 080-099 24 24 24 45 45 45 F F F 5 100-119 20 20 20 34 34 34 F F F 6 120-139 29 29 29 52 52 52 F F F 7 140-159 26 26 26 48 48 48 F F F 8 160-179 28 28 28 51 51 51 F F F 9 180-199 28 28 28 51 51 51 F F F 10 200-219 26 26 26 49 49 49 F F F 11 220-239 28 28 28 51 51 51 F F F 12 240-259 29 29 29 52 52 52 F F F 13 260-279 28 28 28 51 51 51 F F F 14 280-299 28 28 28 51 51 51 F F F 15 300-319 20 20 20 33 33 33 F F F 16 320-339 25 25 25 46 46 46 F F F 17 340-359 28 28 28 51 51 51 F F F 18 360-379 29 29 29 52 52 52 F F F 19 380-399 24 24 24 45 45 45 F F F 20 400-419 22 22 22 36 36 36 F F F 21 420-439 28 28 28 51 51 51 F F F 22 440-459 26 26 26 47 47 47 F F F 23 460-479 28 28 28 51 51 51 F F F 24 480-499 29 29 29 52 52 52 F F F 25 500-519 16 16 16 26 26 26 F F F 26 520-539 28 28 28 51 51 51 F F F 27 540-559 26 26 26 47 47 47 F F F 28 560-579 15 15 15 24 24 24 F F F 29 580-599 20 20 20 33 33 33 F F F Max gen: 29, Max flush: 52 Begin loop 15 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:31.609184567Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1300,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:31.654834956Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1300,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:31.655524942Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1300} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"0fd98d0d5d909611f8b408ccda2559d35fe117bb\",\n git_commit_timestamp: \"2024-11-08T20:05:44.000000000Z\",\n git_branch: \"mkeeter/drop-skipped-backpressure-guard\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:31.65556211Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1300} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:31.655576689Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1300} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:31.655591069Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1300} {"msg":"Crucible fc031896-7e57-48e9-b91a-29f074dcd822 has session id: 50b2d67c-05e7-4ceb-a054-bc61c5ddec11","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:31.656060836Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1300,"session_id":"50b2d67c-05e7-4ceb-a054-bc61c5ddec11"} {"msg":"Upstairs opts: Upstairs UUID: fc031896-7e57-48e9-b91a-29f074dcd822, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:31.656110054Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1300,"session_id":"50b2d67c-05e7-4ceb-a054-bc61c5ddec11"} {"msg":"Crucible stats registered with UUID: fc031896-7e57-48e9-b91a-29f074dcd822","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:31.656135132Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1300,"session_id":"50b2d67c-05e7-4ceb-a054-bc61c5ddec11"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:31.688187306Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1300,"task":"crutest"} {"msg":"The guest has requested activation with gen:30","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:31.688232424Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1300} {"msg":"fc031896-7e57-48e9-b91a-29f074dcd822 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:31.68829796Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1300,"session_id":"50b2d67c-05e7-4ceb-a054-bc61c5ddec11"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:31.688326699Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1300,"client":"0","":"downstairs","session_id":"50b2d67c-05e7-4ceb-a054-bc61c5ddec11"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:31.688351578Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1300,"client":"0","":"downstairs","session_id":"50b2d67c-05e7-4ceb-a054-bc61c5ddec11"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:31.688367857Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1300,"client":"1","":"downstairs","session_id":"50b2d67c-05e7-4ceb-a054-bc61c5ddec11"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:31.688381996Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1300,"client":"1","":"downstairs","session_id":"50b2d67c-05e7-4ceb-a054-bc61c5ddec11"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:31.688399475Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1300,"client":"2","":"downstairs","session_id":"50b2d67c-05e7-4ceb-a054-bc61c5ddec11"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:31.688413485Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1300,"client":"2","":"downstairs","session_id":"50b2d67c-05e7-4ceb-a054-bc61c5ddec11"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:31.688428234Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1300,"":"io task","client":"0","":"downstairs","session_id":"50b2d67c-05e7-4ceb-a054-bc61c5ddec11"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:31.688452213Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1300,"":"io task","client":"2","":"downstairs","session_id":"50b2d67c-05e7-4ceb-a054-bc61c5ddec11"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:31.688473772Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1300,"":"io task","client":"1","":"downstairs","session_id":"50b2d67c-05e7-4ceb-a054-bc61c5ddec11"} {"msg":"ds_connection connected from Ok(127.0.0.1:46502)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:31.688750178Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1300,"":"io task","client":"1","":"downstairs","session_id":"50b2d67c-05e7-4ceb-a054-bc61c5ddec11"} {"msg":"ds_connection connected from Ok(127.0.0.1:47247)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:31.688799956Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1300,"":"io task","client":"2","":"downstairs","session_id":"50b2d67c-05e7-4ceb-a054-bc61c5ddec11"} {"msg":"ds_connection connected from Ok(127.0.0.1:61345)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:31.688821985Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1300,"":"io task","client":"0","":"downstairs","session_id":"50b2d67c-05e7-4ceb-a054-bc61c5ddec11"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:31.689432455Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1300,"client":"0","":"downstairs","session_id":"50b2d67c-05e7-4ceb-a054-bc61c5ddec11"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:31.689465573Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1300,"client":"0","":"downstairs","session_id":"50b2d67c-05e7-4ceb-a054-bc61c5ddec11"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:31.689480603Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1300,"client":"2","":"downstairs","session_id":"50b2d67c-05e7-4ceb-a054-bc61c5ddec11"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:31.689494592Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1300,"client":"2","":"downstairs","session_id":"50b2d67c-05e7-4ceb-a054-bc61c5ddec11"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:31.689518281Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1300,"client":"1","":"downstairs","session_id":"50b2d67c-05e7-4ceb-a054-bc61c5ddec11"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:31.689546519Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1300,"client":"1","":"downstairs","session_id":"50b2d67c-05e7-4ceb-a054-bc61c5ddec11"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:31.690136691Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1300,"client":"2","":"downstairs","session_id":"50b2d67c-05e7-4ceb-a054-bc61c5ddec11"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:31.690264704Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1300,"client":"1","":"downstairs","session_id":"50b2d67c-05e7-4ceb-a054-bc61c5ddec11"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:31.690289713Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1300,"client":"0","":"downstairs","session_id":"50b2d67c-05e7-4ceb-a054-bc61c5ddec11"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:31.690653005Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1300,"client":"2","":"downstairs","session_id":"50b2d67c-05e7-4ceb-a054-bc61c5ddec11"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:31.690681254Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1300,"client":"2","":"downstairs","session_id":"50b2d67c-05e7-4ceb-a054-bc61c5ddec11"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:31.690696483Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1300,"session_id":"50b2d67c-05e7-4ceb-a054-bc61c5ddec11"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:31.690865115Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1300,"client":"1","":"downstairs","session_id":"50b2d67c-05e7-4ceb-a054-bc61c5ddec11"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:31.690924402Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1300,"client":"1","":"downstairs","session_id":"50b2d67c-05e7-4ceb-a054-bc61c5ddec11"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:31.690941531Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1300,"session_id":"50b2d67c-05e7-4ceb-a054-bc61c5ddec11"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:31.69095444Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1300,"client":"0","":"downstairs","session_id":"50b2d67c-05e7-4ceb-a054-bc61c5ddec11"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:31.69096806Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1300,"client":"0","":"downstairs","session_id":"50b2d67c-05e7-4ceb-a054-bc61c5ddec11"} {"msg":"[0]R flush_numbers[0..12]: [47, 51, 51, 52, 45, 34, 52, 48, 51, 51, 49, 51]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:31.690981919Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1300,"":"downstairs","":"downstairs","session_id":"50b2d67c-05e7-4ceb-a054-bc61c5ddec11"} {"msg":"[0]R generation[0..12]: [26, 28, 28, 29, 24, 20, 29, 26, 28, 28, 26, 28]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:31.691011308Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1300,"":"downstairs","":"downstairs","session_id":"50b2d67c-05e7-4ceb-a054-bc61c5ddec11"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:31.691028007Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1300,"":"downstairs","":"downstairs","session_id":"50b2d67c-05e7-4ceb-a054-bc61c5ddec11"} {"msg":"[1]R flush_numbers[0..12]: [47, 51, 51, 52, 45, 34, 52, 48, 51, 51, 49, 51]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:31.691044626Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1300,"":"downstairs","":"downstairs","session_id":"50b2d67c-05e7-4ceb-a054-bc61c5ddec11"} {"msg":"[1]R generation[0..12]: [26, 28, 28, 29, 24, 20, 29, 26, 28, 28, 26, 28]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:31.691069805Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1300,"":"downstairs","":"downstairs","session_id":"50b2d67c-05e7-4ceb-a054-bc61c5ddec11"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:31.691098613Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1300,"":"downstairs","":"downstairs","session_id":"50b2d67c-05e7-4ceb-a054-bc61c5ddec11"} {"msg":"[2]R flush_numbers[0..12]: [47, 51, 51, 52, 45, 34, 52, 48, 51, 51, 49, 51]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:31.691121772Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1300,"":"downstairs","":"downstairs","session_id":"50b2d67c-05e7-4ceb-a054-bc61c5ddec11"} {"msg":"[2]R generation[0..12]: [26, 28, 28, 29, 24, 20, 29, 26, 28, 28, 26, 28]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:31.691142461Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1300Wait for a query_work_queue command to finish before sending IO ,"":"downstairs","":"downstairs","session_id":"50b2d67c-05e7-4ceb-a054-bc61c5ddec11"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:31.69117938Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1300,"":"downstairs","":"downstairs","session_id":"50b2d67c-05e7-4ceb-a054-bc61c5ddec11"} {"msg":"Max found gen is 30","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:31.691207368Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1300,"":"downstairs","":"downstairs","session_id":"50b2d67c-05e7-4ceb-a054-bc61c5ddec11"} {"msg":"Generation requested: 30 >= found:30","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:31.691224127Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1300,"":"downstairs","":"downstairs","session_id":"50b2d67c-05e7-4ceb-a054-bc61c5ddec11"} {"msg":"Next flush: 53","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:31.691237817Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1300,"":"downstairs","":"downstairs","session_id":"50b2d67c-05e7-4ceb-a054-bc61c5ddec11"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:31.691261236Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1300,"":"downstairs","":"downstairs","session_id":"50b2d67c-05e7-4ceb-a054-bc61c5ddec11"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:31.691287154Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1300,"session_id":"50b2d67c-05e7-4ceb-a054-bc61c5ddec11"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:31.691311123Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1300,"client":"0","":"downstairs","session_id":"50b2d67c-05e7-4ceb-a054-bc61c5ddec11"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:31.691326212Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1300,"client":"1","":"downstairs","session_id":"50b2d67c-05e7-4ceb-a054-bc61c5ddec11"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:31.691346301Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1300,"client":"2","":"downstairs","session_id":"50b2d67c-05e7-4ceb-a054-bc61c5ddec11"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:31.69137175Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1300,"session_id":"50b2d67c-05e7-4ceb-a054-bc61c5ddec11"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:31.691396859Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1300,"session_id":"50b2d67c-05e7-4ceb-a054-bc61c5ddec11"} {"msg":"fc031896-7e57-48e9-b91a-29f074dcd822 is now active with session: 50b2d67c-05e7-4ceb-a054-bc61c5ddec11","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:31.691415628Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1300,"session_id":"50b2d67c-05e7-4ceb-a054-bc61c5ddec11"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:31.691427857Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1300,"session_id":"50b2d67c-05e7-4ceb-a054-bc61c5ddec11"} {"msg":"The guest has finished waiting for activation with:30","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:31.691453056Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1300} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 74 len 16384 data: 5 5 5 4 02/30 Read block 356 len 20480 03/30 Read block 592 len 16384 04/30 Write block 287 len 24576 data: 7 8 7 6 5 6 05/30 Read block 537 len 16384 06/30 Write block 426 len 20480 data: 3 3 3 5 5 07/30 Write block 530 len 8192 data: 2 2 08/30 Read block 289 len 28672 09/30 Write block 143 len 12288 data: 5 5 4 10/30 Read block 442 len 40960 11/30 Read block 292 len 20480 12/30 Read block 180 len 32768 13/30 Read block 330 len 20480 14/30 Write block 301 len 12288 data: 4 4 3 15/30 Read block 66 len 16384 16/30 Write block 366 len 28672 data: 4 5 7 6 5 5 6 17/30 Read block 34 len 40960 18/30 Flush 19/30 Write block 86 len 8192 data: 4 4 20/30 Read block 115 len 16384 21/30 Read block 72 len 20480 22/30 Flush 23/30 Write block 429 len 16384 data: 6 6 5 5 24/30 Read block 157 len 12288 25/30 Write block 342 len 40960 data: 6 5 5 4 6 5 3 4 4 4 26/30 Flush 27/30 Write block 355 len 40960 data: 6 6 7 9 9 8 8 6 5 4 28/30 Write block 221 len 36864 data: 2 2 2 2 3 3 5 4 4 29/30 Read block 555 len 24576 30/30 Read block 232 len 24576 ---------------------------------------------------------------- Crucible gen:30 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 10 Done Done Done false 1027 Acked Write 9 Done Done Done false 1028 Acked Read 6 Done Done Done false 1029 Acked Read 6 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:31.721216892Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1300} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 26 26 26 47 47 47 F F F 1 020-039 28 28 28 51 51 51 F F F 2 040-059 28 28 28 51 51 51 F F F 3 060-079 30 30 30 53 53 53 F F F 4 080-099 30 30 30 54 54 54 F F F 5 100-119 20 20 20 34 34 34 F F F 6 120-139 29 29 29 52 52 52 F F F 7 140-159 30 30 30 53 53 53 F F F 8 160-179 28 28 28 51 51 51 F F F 9 180-199 28 28 28 51 51 51 F F F 10 200-219 26 26 26 49 49 49 F F F 11 220-239 28 28 28 51 51 51 T T T 12 240-259 29 29 29 52 52 52 F F F 13 260-279 28 28 28 51 51 51 F F F 14 280-299 30 30 30 53 53 53 F F F 15 300-319 30 30 30 53 53 53 F F F 16 320-339 25 25 25 46 46 46 F F F 17 340-359 30 30 30 55 55 55 T T T 18 360-379 30 30 30 53 53 53 T T T 19 380-399 24 24 24 45 45 45 F F F 20 400-419 22 22 22 36 36 36 F F F 21 420-439 30 30 30 55 55 55 F F F 22 440-459 26 26 26 47 47 47 F F F 23 460-479 28 28 28 51 51 51 F F F 24 480-499 29 29 29 52 52 52 F F F 25 500-519 16 16 16 26 26 26 F F F 26 520-539 30 30 30 53 53 53 F F F 27 540-559 26 26 26 47 47 47 F F F 28 560-579 15 15 15 24 24 24 F F F 29 580-599 20 20 20 33 33 33 F F F Max gen: 30, Max flush: 55 A Difference in extent metadata is expected here On loop 15 Verifying data now No change after verify Loop: 15 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 26 26 26 47 47 47 F F F 1 020-039 28 28 28 51 51 51 F F F 2 040-059 28 28 28 51 51 51 F F F 3 060-079 30 30 30 53 53 53 F F F 4 080-099 30 30 30 54 54 54 F F F 5 100-119 20 20 20 34 34 34 F F F 6 120-139 29 29 29 52 52 52 F F F 7 140-159 30 30 30 53 53 53 F F F 8 160-179 28 28 28 51 51 51 F F F 9 180-199 28 28 28 51 51 51 F F F 10 200-219 26 26 26 49 49 49 F F F 11 220-239 31 31 31 56 56 56 F F F 12 240-259 29 29 29 52 52 52 F F F 13 260-279 28 28 28 51 51 51 F F F 14 280-299 30 30 30 53 53 53 F F F 15 300-319 30 30 30 53 53 53 F F F 16 320-339 25 25 25 46 46 46 F F F 17 340-359 31 31 31 56 56 56 F F F 18 360-379 31 31 31 56 56 56 F F F 19 380-399 24 24 24 45 45 45 F F F 20 400-419 22 22 22 36 36 36 F F F 21 420-439 30 30 30 55 55 55 F F F 22 440-459 26 26 26 47 47 47 F F F 23 460-479 28 28 28 51 51 51 F F F 24 480-499 29 29 29 52 52 52 F F F 25 500-519 16 16 16 26 26 26 F F F 26 520-539 30 30 30 53 53 53 F F F 27 540-559 26 26 26 47 47 47 F F F 28 560-579 15 15 15 24 24 24 F F F 29 580-599 20 20 20 33 33 33 F F F Max gen: 31, Max flush: 56 Begin loop 16 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:36.021094337Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1308,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:36.065008091Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1308,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:36.065617121Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1308} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"0fd98d0d5d909611f8b408ccda2559d35fe117bb\",\n git_commit_timestamp: \"2024-11-08T20:05:44.000000000Z\",\n git_branch: \"mkeeter/drop-skipped-backpressure-guard\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:36.065654429Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1308} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:36.065669819Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1308} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:36.065685348Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1308} {"msg":"Crucible 434bedfd-417d-4361-8949-7efcaadc8f2e has session id: d52158c9-70bc-46dc-8a9a-061cc853afce","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:36.066113537Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1308,"session_id":"d52158c9-70bc-46dc-8a9a-061cc853afce"} {"msg":"Upstairs opts: Upstairs UUID: 434bedfd-417d-4361-8949-7efcaadc8f2e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:36.066162284Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1308,"session_id":"d52158c9-70bc-46dc-8a9a-061cc853afce"} {"msg":"Crucible stats registered with UUID: 434bedfd-417d-4361-8949-7efcaadc8f2e","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:36.066186893Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1308,"session_id":"d52158c9-70bc-46dc-8a9a-061cc853afce"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:36.097112522Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1308,"task":"crutest"} {"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:36.0971463Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1308} {"msg":"434bedfd-417d-4361-8949-7efcaadc8f2e active request set","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:36.097180388Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1308,"session_id":"d52158c9-70bc-46dc-8a9a-061cc853afce"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:36.097197488Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1308,"client":"0","":"downstairs","session_id":"d52158c9-70bc-46dc-8a9a-061cc853afce"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:36.097223796Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1308,"client":"0","":"downstairs","session_id":"d52158c9-70bc-46dc-8a9a-061cc853afce"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:36.097241325Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1308,"client":"1","":"downstairs","session_id":"d52158c9-70bc-46dc-8a9a-061cc853afce"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:36.097258305Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1308,"client":"1","":"downstairs","session_id":"d52158c9-70bc-46dc-8a9a-061cc853afce"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:36.097275054Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1308,"client":"2","":"downstairs","session_id":"d52158c9-70bc-46dc-8a9a-061cc853afce"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:36.097292033Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1308,"client":"2","":"downstairs","session_id":"d52158c9-70bc-46dc-8a9a-061cc853afce"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:36.097320112Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1308,"":"io task","client":"2","":"downstairs","session_id":"d52158c9-70bc-46dc-8a9a-061cc853afce"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:36.09735207Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1308,"":"io task","client":"0","":"downstairs","session_id":"d52158c9-70bc-46dc-8a9a-061cc853afce"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:36.097386309Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1308,"":"io task","client":"1","":"downstairs","session_id":"d52158c9-70bc-46dc-8a9a-061cc853afce"} {"msg":"ds_connection connected from Ok(127.0.0.1:40914)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:36.097437356Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1308,"":"io task","client":"1","":"downstairs","session_id":"d52158c9-70bc-46dc-8a9a-061cc853afce"} {"msg":"ds_connection connected from Ok(127.0.0.1:47631)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:36.097609318Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1308,"":"io task","client":"2","":"downstairs","session_id":"d52158c9-70bc-46dc-8a9a-061cc853afce"} {"msg":"ds_connection connected from Ok(127.0.0.1:61188)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:36.09776609Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1308,"":"io task","client":"0","":"downstairs","session_id":"d52158c9-70bc-46dc-8a9a-061cc853afce"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:36.098250106Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1308,"client":"0","":"downstairs","session_id":"d52158c9-70bc-46dc-8a9a-061cc853afce"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:36.098290444Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1308,"client":"0","":"downstairs","session_id":"d52158c9-70bc-46dc-8a9a-061cc853afce"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:36.098322653Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1308,"client":"1","":"downstairs","session_id":"d52158c9-70bc-46dc-8a9a-061cc853afce"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:36.098352991Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1308,"client":"1","":"downstairs","session_id":"d52158c9-70bc-46dc-8a9a-061cc853afce"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:36.09838182Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1308,"client":"2","":"downstairs","session_id":"d52158c9-70bc-46dc-8a9a-061cc853afce"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:36.098413468Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1308,"client":"2","":"downstairs","session_id":"d52158c9-70bc-46dc-8a9a-061cc853afce"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:36.098929453Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1308,"client":"1","":"downstairs","session_id":"d52158c9-70bc-46dc-8a9a-061cc853afce"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:36.098961072Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1308,"client":"0","":"downstairs","session_id":"d52158c9-70bc-46dc-8a9a-061cc853afce"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:36.098978121Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1308,"client":"2","":"downstairs","session_id":"d52158c9-70bc-46dc-8a9a-061cc853afce"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:36.099477736Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1308,"client":"1","":"downstairs","session_id":"d52158c9-70bc-46dc-8a9a-061cc853afce"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:36.099506645Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1308,"client":"1","":"downstairs","session_id":"d52158c9-70bc-46dc-8a9a-061cc853afce"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:36.099521834Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1308,"session_id":"d52158c9-70bc-46dc-8a9a-061cc853afce"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:36.09959809Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1308,"client":"0","":"downstairs","session_id":"d52158c9-70bc-46dc-8a9a-061cc853afce"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:36.099637288Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1308,"client":"0","":"downstairs","session_id":"d52158c9-70bc-46dc-8a9a-061cc853afce"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:36.099658677Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1308,"session_id":"d52158c9-70bc-46dc-8a9a-061cc853afce"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:36.099671527Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1308,"client":"2","":"downstairs","session_id":"d52158c9-70bc-46dc-8a9a-061cc853afce"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:36.099685136Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1308,"client":"2","":"downstairs","session_id":"d52158c9-70bc-46dc-8a9a-061cc853afce"} {"msg":"[0]R flush_numbers[0..12]: [47, 51, 51, 53, 54, 34, 52, 53, 51, 51, 49, 56]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:36.099698855Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1308,"":"downstairs","":"downstairs","session_id":"d52158c9-70bc-46dc-8a9a-061cc853afce"} {"msg":"[0]R generation[0..12]: [26, 28, 28, 30, 30, 20, 29, 30, 28, 28, 26, 31]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:36.099712775Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1308,"":"downstairs","":"downstairs","session_id":"d52158c9-70bc-46dc-8a9a-061cc853afce"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:36.099727144Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1308,"":"downstairs","":"downstairs","session_id":"d52158c9-70bc-46dc-8a9a-061cc853afce"} {"msg":"[1]R flush_numbers[0..12]: [47, 51, 51, 53, 54, 34, 52, 53, 51, 51, 49, 56]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:36.099741013Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1308,"":"downstairs","":"downstairs","session_id":"d52158c9-70bc-46dc-8a9a-061cc853afce"} {"msg":"[1]R generation[0..12]: [26, 28, 28, 30, 30, 20, 29, 30, 28, 28, 26, 31]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:36.099754663Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1308,"":"downstairs","":"downstairs","session_id":"d52158c9-70bc-46dc-8a9a-061cc853afce"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:36.099768822Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1308,"":"downstairs","":"downstairs","session_id":"d52158c9-70bc-46dc-8a9a-061cc853afce"} {"msg":"[2]R flush_numbers[0..12]: [47, 51, 51, 53, 54, 34, 52, 53, 51, 51, 49, 56]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:36.099782491Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1308,"":"downstairs","":"downstairs","session_id":"d52158c9-70bc-46dc-8a9a-061cc853afce"} {"msg":"[2]R generation[0..12]: [26, 28, 28, 30, 30, 20, 29, 30, 28, 28, 26, 31]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:36.099796181Z","Wait for a query_work_queue command to finish before sending IO hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1308,"":"downstairs","":"downstairs","session_id":"d52158c9-70bc-46dc-8a9a-061cc853afce"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:36.099821219Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1308,"":"downstairs","":"downstairs","session_id":"d52158c9-70bc-46dc-8a9a-061cc853afce"} {"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:36.099845708Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1308,"":"downstairs","":"downstairs","session_id":"d52158c9-70bc-46dc-8a9a-061cc853afce"} {"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:36.099874417Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1308,"":"downstairs","":"downstairs","session_id":"d52158c9-70bc-46dc-8a9a-061cc853afce"} {"msg":"Next flush: 57","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:36.099895986Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1308,"":"downstairs","":"downstairs","session_id":"d52158c9-70bc-46dc-8a9a-061cc853afce"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:36.099910055Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1308,"":"downstairs","":"downstairs","session_id":"d52158c9-70bc-46dc-8a9a-061cc853afce"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:36.099923465Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1308,"session_id":"d52158c9-70bc-46dc-8a9a-061cc853afce"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:36.099935684Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1308,"client":"0","":"downstairs","session_id":"d52158c9-70bc-46dc-8a9a-061cc853afce"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:36.099949003Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1308,"client":"1","":"downstairs","session_id":"d52158c9-70bc-46dc-8a9a-061cc853afce"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:36.099962212Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1308,"client":"2","":"downstairs","session_id":"d52158c9-70bc-46dc-8a9a-061cc853afce"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:36.099975652Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1308,"session_id":"d52158c9-70bc-46dc-8a9a-061cc853afce"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2024-11-08T20:23:36.099988091Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1308,"session_id":"d52158c9-70bc-46dc-8a9a-061cc853afce"} {"msg":"434bedfd-417d-4361-8949-7efcaadc8f2e is now active with session: d52158c9-70bc-46dc-8a9a-061cc853afce","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:36.10001793Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1308,"session_id":"d52158c9-70bc-46dc-8a9a-061cc853afce"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:36.100031679Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1308,"session_id":"d52158c9-70bc-46dc-8a9a-061cc853afce"} {"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:36.100044288Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1308} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 76 len 32768 data: 6 5 4 3 2 2 2 3 02/30 Write block 186 len 8192 data: 8 8 03/30 Write block 204 len 12288 data: 3 3 3 04/30 Read block 173 len 24576 05/30 Write block 30 len 28672 data: 3 5 5 4 4 2 2 06/30 Flush 07/30 Write block 95 len 8192 data: 2 2 08/30 Flush 09/30 Write block 585 len 12288 data: 6 6 6 10/30 Flush 11/30 Write block 535 len 20480 data: 4 4 4 3 3 12/30 Write block 466 len 4096 data: 4 13/30 Flush 14/30 Flush 15/30 Write block 177 len 40960 data: 3 3 3 5 5 5 7 6 7 9 16/30 Write block 177 len 16384 data: 4 4 4 6 17/30 Flush 18/30 Write block 535 len 40960 data: 5 5 5 4 4 3 3 4 5 5 19/30 Write block 548 len 8192 data: 3 3 20/30 Read block 175 len 40960 21/30 Read block 499 len 36864 22/30 Write block 16 len 20480 data: 2 2 2 4 5 23/30 Write block 72 len 24576 data: 3 3 6 6 7 6 24/30 Write block 76 len 20480 data: 8 7 5 4 3 25/30 Read block 285 len 32768 26/30 Read block 51 len 20480 27/30 Write block 221 len 20480 data: 3 3 3 3 4 28/30 Write block 516 len 4096 data: 3 29/30 Read block 127 len 12288 30/30 Write block 130 len 40960 data: 2 2 2 4 4 4 3 5 5 5 ---------------------------------------------------------------- Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:13 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1017 Acked Write 10 Done Done Done false 1018 Acked Write 2 Done Done Done false 1019 Acked Read 10 Done Done Done false 1020 Acked Read 9 Done Done Done false 1021 Acked Write 5 Done Done Done false 1022 Acked Write 6 Done Done Done false 1023 Acked Write 5 Done Done Done false 1024 Acked Read 8 Done Done Done false 1025 Acked Read 5 Done Done Done false 1026 Acked Write 5 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Read 3 Done Done Done false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 12 12 12 36 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1016 1016 1016 Downstairs last five completed: 1016 1015 1014 1013 1012 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:36.130927119Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1308} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 26 26 26 47 47 47 T T T 1 020-039 32 32 32 57 57 57 T T T 2 040-059 28 28 28 51 51 51 F F F 3 060-079 32 32 32 57 57 57 T T T 4 080-099 32 32 32 58 58 58 T T T 5 100-119 20 20 20 34 34 34 F F F 6 120-139 29 29 29 52 52 52 T T T 7 140-159 30 30 30 53 53 53 F F F 8 160-179 32 32 32 62 62 62 F F F 9 180-199 32 32 32 62 62 62 F F F 10 200-219 32 32 32 57 57 57 F F F 11 220-239 31 31 31 56 56 56 T T T 12 240-259 29 29 29 52 52 52 F F F 13 260-279 28 28 28 51 51 51 F F F 14 280-299 30 30 30 53 53 53 F F F 15 300-319 30 30 30 53 53 53 F F F 16 320-339 25 25 25 46 46 46 F F F 17 340-359 31 31 31 56 56 56 F F F 18 360-379 31 31 31 56 56 56 F F F 19 380-399 24 24 24 45 45 45 F F F 20 400-419 22 22 22 36 36 36 F F F 21 420-439 30 30 30 55 55 55 F F F 22 440-459 26 26 26 47 47 47 F F F 23 460-479 32 32 32 60 60 60 F F F 24 480-499 29 29 29 52 52 52 F F F 25 500-519 16 16 16 26 26 26 T T T 26 520-539 32 32 32 60 60 60 T T T 27 540-559 26 26 26 47 47 47 T T T 28 560-579 15 15 15 24 24 24 F F F 29 580-599 32 32 32 59 59 59 F F F Max gen: 32, Max flush: 62 A Difference in extent metadata is expected here On loop 16 Verifying data now No change after verify Loop: 16 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 33 33 33 63 63 63 F F F 1 020-039 33 33 33 63 63 63 F F F 2 040-059 28 28 28 51 51 51 F F F 3 060-079 33 33 33 63 63 63 F F F 4 080-099 33 33 33 63 63 63 F F F 5 100-119 20 20 20 34 34 34 F F F 6 120-139 33 33 33 63 63 63 F F F 7 140-159 30 30 30 53 53 53 F F F 8 160-179 32 32 32 62 62 62 F F F 9 180-199 32 32 32 62 62 62 F F F 10 200-219 32 32 32 57 57 57 F F F 11 220-239 33 33 33 63 63 63 F F F 12 240-259 29 29 29 52 52 52 F F F 13 260-279 28 28 28 51 51 51 F F F 14 280-299 30 30 30 53 53 53 F F F 15 300-319 30 30 30 53 53 53 F F F 16 320-339 25 25 25 46 46 46 F F F 17 340-359 31 31 31 56 56 56 F F F 18 360-379 31 31 31 56 56 56 F F F 19 380-399 24 24 24 45 45 45 F F F 20 400-419 22 22 22 36 36 36 F F F 21 420-439 30 30 30 55 55 55 F F F 22 440-459 26 26 26 47 47 47 F F F 23 460-479 32 32 32 60 60 60 F F F 24 480-499 29 29 29 52 52 52 F F F 25 500-519 33 33 33 63 63 63 F F F 26 520-539 33 33 33 63 63 63 F F F 27 540-559 33 33 33 63 63 63 F F F 28 560-579 15 15 15 24 24 24 F F F 29 580-599 32 32 32 59 59 59 F F F Max gen: 33, Max flush: 63 Begin loop 17 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:40.503773311Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1316,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:40.549368232Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1316,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:40.549944824Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1316} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"0fd98d0d5d909611f8b408ccda2559d35fe117bb\",\n git_commit_timestamp: \"2024-11-08T20:05:44.000000000Z\",\n git_branch: \"mkeeter/drop-skipped-backpressure-guard\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:40.549981922Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1316} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:40.549996422Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1316} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:40.550011921Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1316} {"msg":"Crucible 4e23e763-dbb3-4350-9441-c174e1ff1c8c has session id: 3a86cdff-8603-465c-86d3-783422aaaf71","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:40.550519206Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1316,"session_id":"3a86cdff-8603-465c-86d3-783422aaaf71"} {"msg":"Upstairs opts: Upstairs UUID: 4e23e763-dbb3-4350-9441-c174e1ff1c8c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:40.550565334Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1316,"session_id":"3a86cdff-8603-465c-86d3-783422aaaf71"} {"msg":"Crucible stats registered with UUID: 4e23e763-dbb3-4350-9441-c174e1ff1c8c","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:40.550594403Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1316,"session_id":"3a86cdff-8603-465c-86d3-783422aaaf71"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:40.581700652Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1316,"task":"crutest"} {"msg":"The guest has requested activation with gen:34","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:40.58174053Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1316} {"msg":"4e23e763-dbb3-4350-9441-c174e1ff1c8c active request set","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:40.581769839Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1316,"session_id":"3a86cdff-8603-465c-86d3-783422aaaf71"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:40.581790728Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1316,"client":"0","":"downstairs","session_id":"3a86cdff-8603-465c-86d3-783422aaaf71"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:40.581836076Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1316,"client":"0","":"downstairs","session_id":"3a86cdff-8603-465c-86d3-783422aaaf71"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:40.581873344Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1316,"client":"1","":"downstairs","session_id":"3a86cdff-8603-465c-86d3-783422aaaf71"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:40.581910322Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1316,"client":"1","":"downstairs","session_id":"3a86cdff-8603-465c-86d3-783422aaaf71"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:40.58194524Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1316,"client":"2","":"downstairs","session_id":"3a86cdff-8603-465c-86d3-783422aaaf71"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:40.581981129Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1316,"client":"2","":"downstairs","session_id":"3a86cdff-8603-465c-86d3-783422aaaf71"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:40.582017367Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1316,"":"io task","client":"2","":"downstairs","session_id":"3a86cdff-8603-465c-86d3-783422aaaf71"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:40.582052705Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1316,"":"io task","client":"0","":"downstairs","session_id":"3a86cdff-8603-465c-86d3-783422aaaf71"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:40.582087744Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1316,"":"io task","client":"1","":"downstairs","session_id":"3a86cdff-8603-465c-86d3-783422aaaf71"} {"msg":"ds_connection connected from Ok(127.0.0.1:59946)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:40.582119332Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1316,"":"io task","client":"2","":"downstairs","session_id":"3a86cdff-8603-465c-86d3-783422aaaf71"} {"msg":"ds_connection connected from Ok(127.0.0.1:61446)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:40.58215247Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1316,"":"io task","client":"1","":"downstairs","session_id":"3a86cdff-8603-465c-86d3-783422aaaf71"} {"msg":"ds_connection connected from Ok(127.0.0.1:42719)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:40.582256025Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1316,"":"io task","client":"0","":"downstairs","session_id":"3a86cdff-8603-465c-86d3-783422aaaf71"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:40.582931652Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1316,"client":"0","":"downstairs","session_id":"3a86cdff-8603-465c-86d3-783422aaaf71"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:40.582960921Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1316,"client":"0","":"downstairs","session_id":"3a86cdff-8603-465c-86d3-783422aaaf71"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:40.58297606Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1316,"client":"1","":"downstairs","session_id":"3a86cdff-8603-465c-86d3-783422aaaf71"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:40.582989839Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1316,"client":"1","":"downstairs","session_id":"3a86cdff-8603-465c-86d3-783422aaaf71"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:40.583003369Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1316,"client":"2","":"downstairs","session_id":"3a86cdff-8603-465c-86d3-783422aaaf71"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:40.583017128Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1316,"client":"2","":"downstairs","session_id":"3a86cdff-8603-465c-86d3-783422aaaf71"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:40.583521873Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1316,"client":"0","":"downstairs","session_id":"3a86cdff-8603-465c-86d3-783422aaaf71"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:40.58358036Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1316,"client":"1","":"downstairs","session_id":"3a86cdff-8603-465c-86d3-783422aaaf71"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:40.583666586Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1316,"client":"2","":"downstairs","session_id":"3a86cdff-8603-465c-86d3-783422aaaf71"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:40.584119484Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1316,"client":"0","":"downstairs","session_id":"3a86cdff-8603-465c-86d3-783422aaaf71"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:40.584149223Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1316,"client":"0","":"downstairs","session_id":"3a86cdff-8603-465c-86d3-783422aaaf71"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:40.584164792Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1316,"session_id":"3a86cdff-8603-465c-86d3-783422aaaf71"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:40.584177841Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1316,"client":"1","":"downstairs","session_id":"3a86cdff-8603-465c-86d3-783422aaaf71"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:40.584191821Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1316,"client":"1","":"downstairs","session_id":"3a86cdff-8603-465c-86d3-783422aaaf71"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:40.58420591Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1316,"session_id":"3a86cdff-8603-465c-86d3-783422aaaf71"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:40.584218549Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1316,"client":"2","":"downstairs","session_id":"3a86cdff-8603-465c-86d3-783422aaaf71"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:40.584235788Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1316,"client":"2","":"downstairs","session_id":"3a86cdff-8603-465c-86d3-783422aaaf71"} {"msg":"[0]R flush_numbers[0..12]: [63, 63, 51, 63, 63, 34, 63, 53, 62, 62, 57, 63]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:40.584263957Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1316,"":"downstairs","":"downstairs","session_id":"3a86cdff-8603-465c-86d3-783422aaaf71"} {"msg":"[0]R generation[0..12]: [33, 33, 28, 33, 33, 20, 33, 30, 32, 32, 32, 33]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:40.584278796Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1316,"":"downstairs","":"downstairs","session_id":"3a86cdff-8603-465c-86d3-783422aaaf71"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:40.584293586Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1316,"":"downstairs","":"downstairs","session_id":"3a86cdff-8603-465c-86d3-783422aaaf71"} {"msg":"[1]R flush_numbers[0..12]: [63, 63, 51, 63, 63, 34, 63, 53, 62, 62, 57, 63]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:40.584307335Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1316,"":"downstairs","":"downstairs","session_id":"3a86cdff-8603-465c-86d3-783422aaaf71"} {"msg":"[1]R generation[0..12]: [33, 33, 28, 33, 33, 20, 33, 30, 32, 32, 32, 33]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:40.584320844Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1316,"":"downstairs","":"downstairs","session_id":"3a86cdff-8603-465c-86d3-783422aaaf71"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:40.584334803Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1316,"":"downstairs","":"downstairs","session_id":"3a86cdff-8603-465c-86d3-783422aaaf71"} {"msg":"[2]R flush_numbers[0..12]: [63, 63, 51, 63, 63, 34, 63, 53, 62, 62, 57, 63]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:40.584348483Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1316,"":"downstairs","":"downstairs","session_id":"3a86cdff-8603-465c-86d3-783422aaaf71"} {"msg":"[2]R generation[0..12]: [33, 33, 28, 33, 33, 20, 33, 30, 32, 32, 32, 33]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:40.584362102Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1316,"":"downstairs","":"downstairs","session_id":"3a86cdff-8603-465c-86d3-783422aaaf71"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:40.584376041Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1316,"":"downstairs","":"downstairs","session_id":"3a86cdff-8603-465c-86d3-783422aaaf71"} {"msg":"Max found gen is 34","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:40.584389231Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1316,"":"downstairs","":"downstairs","session_id":"3a86cdff-8603-465c-86d3-783422aaaf71"} {"msg":"Generation requested: 34 >= found:34","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:40.58440268Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1316,"":"downstairs","":"downstairs","session_id":"3a86cdff-8603-465c-86d3-783422aaaf71"} {"msg":"Next flush: 64","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:40.58441584Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1316,"":"downstairs","":"downstairs","session_id":"3a86cdff-8603-465c-86d3-783422aaaf71"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:40.584428869Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1316,"":"downstairs","":"downstairs","session_id":"3a86cdff-8603-465c-86d3-783422aaaf71"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:40.584444178Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1316,"session_id":"3a86cdff-8603-465c-86d3-783422aaaf71"} {"msg":"Transition from WaitQuorum to Active"Wait for a query_work_queue command to finish before sending IO ,"v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:40.584465057Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1316,"client":"0","":"downstairs","session_id":"3a86cdff-8603-465c-86d3-783422aaaf71"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:40.584478696Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1316,"client":"1","":"downstairs","session_id":"3a86cdff-8603-465c-86d3-783422aaaf71"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:40.584491906Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1316,"client":"2","":"downstairs","session_id":"3a86cdff-8603-465c-86d3-783422aaaf71"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:40.584505285Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1316,"session_id":"3a86cdff-8603-465c-86d3-783422aaaf71"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:40.584517765Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1316,"session_id":"3a86cdff-8603-465c-86d3-783422aaaf71"} {"msg":"4e23e763-dbb3-4350-9441-c174e1ff1c8c is now active with session: 3a86cdff-8603-465c-86d3-783422aaaf71","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:40.584530464Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1316,"session_id":"3a86cdff-8603-465c-86d3-783422aaaf71"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:40.584542444Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1316,"session_id":"3a86cdff-8603-465c-86d3-783422aaaf71"} {"msg":"The guest has finished waiting for activation with:34","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:40.584554763Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1316} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 250 len 24576 data: 6 6 7 5 5 4 02/30 Write block 147 len 20480 data: 4 3 4 4 5 03/30 Read block 218 len 40960 04/30 Write block 204 len 40960 data: 4 4 4 3 2 3 2 2 2 2 05/30 Write block 113 len 32768 data: 3 4 2 3 3 3 4 4 06/30 Read block 306 len 28672 07/30 Write block 548 len 28672 data: 4 4 3 2 4 3 4 08/30 Read block 546 len 40960 09/30 Read block 147 len 28672 10/30 Write block 473 len 28672 data: 4 4 5 5 5 5 4 11/30 Write block 571 len 24576 data: 4 4 4 3 3 2 12/30 Read block 210 len 16384 13/30 Write block 508 len 40960 data: 3 3 3 3 3 4 4 4 4 2 14/30 Write block 247 len 32768 data: 5 6 6 7 7 8 6 6 15/30 Read block 87 len 32768 16/30 Read block 80 len 36864 17/30 Read block 378 len 12288 18/30 Read block 539 len 24576 19/30 Flush 20/30 Write block 218 len 16384 data: 4 3 2 4 21/30 Read block 591 len 4096 22/30 Read block 481 len 20480 23/30 Read block 499 len 8192 24/30 Flush 25/30 Write block 405 len 40960 data: 6 6 5 4 4 4 6 5 4 5 26/30 Write block 328 len 24576 data: 2 2 2 2 4 4 27/30 Read block 421 len 16384 28/30 Read block 367 len 36864 29/30 Read block 286 len 16384 30/30 Read block 531 len 28672 ---------------------------------------------------------------- Crucible gen:34 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 10 Done Done Done false 1025 Acked Write 6 Done Done Done false 1026 Acked Read 4 Done Done Done false 1027 Acked Read 9 Done Done Done false 1028 Acked Read 4 Done Done Done false 1029 Acked Read 7 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 6 5 5 16 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:40.609930663Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1316} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 33 33 33 63 63 63 F F F 1 020-039 33 33 33 63 63 63 F F F 2 040-059 28 28 28 51 51 51 F F F 3 060-079 33 33 33 63 63 63 F F F 4 080-099 33 33 33 63 63 63 F F F 5 100-119 34 34 34 64 64 64 F F F 6 120-139 34 34 34 64 64 64 F F F 7 140-159 34 34 34 64 64 64 F F F 8 160-179 32 32 32 62 62 62 F F F 9 180-199 32 32 32 62 62 62 F F F 10 200-219 34 34 34 65 65 65 F F F 11 220-239 34 34 34 65 65 65 F F F 12 240-259 34 34 34 64 64 64 F F F 13 260-279 28 28 28 51 51 51 F F F 14 280-299 30 30 30 53 53 53 F F F 15 300-319 30 30 30 53 53 53 F F F 16 320-339 25 25 25 46 46 46 T T T 17 340-359 31 31 31 56 56 56 F F F 18 360-379 31 31 31 56 56 56 F F F 19 380-399 24 24 24 45 45 45 F F F 20 400-419 22 22 22 36 36 36 T T T 21 420-439 30 30 30 55 55 55 F F F 22 440-459 26 26 26 47 47 47 F F F 23 460-479 34 34 34 64 64 64 F F F 24 480-499 29 29 29 52 52 52 F F F 25 500-519 34 34 34 64 64 64 F F F 26 520-539 33 33 33 63 63 63 F F F 27 540-559 34 34 34 64 64 64 F F F 28 560-579 34 34 34 64 64 64 F F F 29 580-599 32 32 32 59 59 59 F F F Max gen: 34, Max flush: 65 A Difference in extent metadata is expected here On loop 17 Verifying data now No change after verify Loop: 17 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 33 33 33 63 63 63 F F F 1 020-039 33 33 33 63 63 63 F F F 2 040-059 28 28 28 51 51 51 F F F 3 060-079 33 33 33 63 63 63 F F F 4 080-099 33 33 33 63 63 63 F F F 5 100-119 34 34 34 64 64 64 F F F 6 120-139 34 34 34 64 64 64 F F F 7 140-159 34 34 34 64 64 64 F F F 8 160-179 32 32 32 62 62 62 F F F 9 180-199 32 32 32 62 62 62 F F F 10 200-219 34 34 34 65 65 65 F F F 11 220-239 34 34 34 65 65 65 F F F 12 240-259 34 34 34 64 64 64 F F F 13 260-279 28 28 28 51 51 51 F F F 14 280-299 30 30 30 53 53 53 F F F 15 300-319 30 30 30 53 53 53 F F F 16 320-339 35 35 35 66 66 66 F F F 17 340-359 31 31 31 56 56 56 F F F 18 360-379 31 31 31 56 56 56 F F F 19 380-399 24 24 24 45 45 45 F F F 20 400-419 35 35 35 66 66 66 F F F 21 420-439 30 30 30 55 55 55 F F F 22 440-459 26 26 26 47 47 47 F F F 23 460-479 34 34 34 64 64 64 F F F 24 480-499 29 29 29 52 52 52 F F F 25 500-519 34 34 34 64 64 64 F F F 26 520-539 33 33 33 63 63 63 F F F 27 540-559 34 34 34 64 64 64 F F F 28 560-579 34 34 34 64 64 64 F F F 29 580-599 32 32 32 59 59 59 F F F Max gen: 35, Max flush: 66 Begin loop 18 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:44.882133081Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1324,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:44.926150699Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1324,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:44.926842186Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1324} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"0fd98d0d5d909611f8b408ccda2559d35fe117bb\",\n git_commit_timestamp: \"2024-11-08T20:05:44.000000000Z\",\n git_branch: \"mkeeter/drop-skipped-backpressure-guard\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:44.926871974Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1324} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:44.926884954Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1324} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:44.926898873Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1324} {"msg":"Crucible f5a81304-f5d3-4bfe-97bb-25565222d384 has session id: 82ef6610-0c26-4b49-a285-db2739bead10","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:44.927343861Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1324,"session_id":"82ef6610-0c26-4b49-a285-db2739bead10"} {"msg":"Upstairs opts: Upstairs UUID: f5a81304-f5d3-4bfe-97bb-25565222d384, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:44.927391779Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1324,"session_id":"82ef6610-0c26-4b49-a285-db2739bead10"} {"msg":"Crucible stats registered with UUID: f5a81304-f5d3-4bfe-97bb-25565222d384","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:44.927417228Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1324,"session_id":"82ef6610-0c26-4b49-a285-db2739bead10"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:44.959426333Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1324,"task":"crutest"} {"msg":"The guest has requested activation with gen:36","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:44.959458691Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1324} {"msg":"f5a81304-f5d3-4bfe-97bb-25565222d384 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:44.95949115Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1324,"session_id":"82ef6610-0c26-4b49-a285-db2739bead10"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:44.959508019Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1324,"client":"0","":"downstairs","session_id":"82ef6610-0c26-4b49-a285-db2739bead10"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:44.959541498Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1324,"client":"0","":"downstairs","session_id":"82ef6610-0c26-4b49-a285-db2739bead10"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:44.959560267Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1324,"client":"1","":"downstairs","session_id":"82ef6610-0c26-4b49-a285-db2739bead10"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:44.959577476Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1324,"client":"1","":"downstairs","session_id":"82ef6610-0c26-4b49-a285-db2739bead10"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:44.959594175Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1324,"client":"2","":"downstairs","session_id":"82ef6610-0c26-4b49-a285-db2739bead10"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:44.959611474Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1324,"client":"2","":"downstairs","session_id":"82ef6610-0c26-4b49-a285-db2739bead10"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:44.959625474Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1324,"":"io task","client":"2","":"downstairs","session_id":"82ef6610-0c26-4b49-a285-db2739bead10"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:44.959639413Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1324,"":"io task","client":"0","":"downstairs","session_id":"82ef6610-0c26-4b49-a285-db2739bead10"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:44.959653282Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1324,"":"io task","client":"1","":"downstairs","session_id":"82ef6610-0c26-4b49-a285-db2739bead10"} {"msg":"ds_connection connected from Ok(127.0.0.1:43362)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:44.959789535Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1324,"":"io task","client":"0","":"downstairs","session_id":"82ef6610-0c26-4b49-a285-db2739bead10"} {"msg":"ds_connection connected from Ok(127.0.0.1:53713)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:44.959831393Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1324,"":"io task","client":"1","":"downstairs","session_id":"82ef6610-0c26-4b49-a285-db2739bead10"} {"msg":"ds_connection connected from Ok(127.0.0.1:37614)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:44.959861312Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1324,"":"io task","client":"2","":"downstairs","session_id":"82ef6610-0c26-4b49-a285-db2739bead10"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:44.960594036Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1324,"client":"0","":"downstairs","session_id":"82ef6610-0c26-4b49-a285-db2739bead10"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:44.960624405Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1324,"client":"0","":"downstairs","session_id":"82ef6610-0c26-4b49-a285-db2739bead10"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:44.960646664Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1324,"client":"1","":"downstairs","session_id":"82ef6610-0c26-4b49-a285-db2739bead10"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:44.960673532Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1324,"client":"1","":"downstairs","session_id":"82ef6610-0c26-4b49-a285-db2739bead10"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:44.960698111Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1324,"client":"2","":"downstairs","session_id":"82ef6610-0c26-4b49-a285-db2739bead10"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:44.960727769Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1324,"client":"2","":"downstairs","session_id":"82ef6610-0c26-4b49-a285-db2739bead10"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:44.961262663Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1324,"client":"2","":"downstairs","session_id":"82ef6610-0c26-4b49-a285-db2739bead10"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:44.961408146Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1324,"client":"1","":"downstairs","session_id":"82ef6610-0c26-4b49-a285-db2739bead10"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:44.961484223Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1324,"client":"0","":"downstairs","session_id":"82ef6610-0c26-4b49-a285-db2739bead10"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:44.961789168Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1324,"client":"2","":"downstairs","session_id":"82ef6610-0c26-4b49-a285-db2739bead10"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:44.961817896Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1324,"client":"2","":"downstairs","session_id":"82ef6610-0c26-4b49-a285-db2739bead10"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:44.961833795Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1324,"session_id":"82ef6610-0c26-4b49-a285-db2739bead10"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:44.961923311Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1324,"client":"1","":"downstairs","session_id":"82ef6610-0c26-4b49-a285-db2739bead10"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:44.96194367Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1324,"client":"1","":"downstairs","session_id":"82ef6610-0c26-4b49-a285-db2739bead10"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:44.96195772Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1324,"session_id":"82ef6610-0c26-4b49-a285-db2739bead10"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:44.962031946Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1324,"client":"0","":"downstairs","session_id":"82ef6610-0c26-4b49-a285-db2739bead10"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:44.962052115Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1324,"client":"0","":"downstairs","session_id":"82ef6610-0c26-4b49-a285-db2739bead10"} {"msg":"[0]R flush_numbers[0..12]: [63, 63, 51, 63, 63, 64, 64, 64, 62, 62, 65, 65]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:44.962066184Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1324,"":"downstairs","":"downstairs","session_id":"82ef6610-0c26-4b49-a285-db2739bead10"} {"msg":"[0]R generation[0..12]: [33, 33, 28, 33, 33, 34, 34, 34, 32, 32, 34, 34]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:44.962080003Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1324,"":"downstairs","":"downstairs","session_id":"82ef6610-0c26-4b49-a285-db2739bead10"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:44.962094033Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1324,"":"downstairs","":"downstairs","session_id":"82ef6610-0c26-4b49-a285-db2739bead10"} {"msg":"[1]R flush_numbers[0..12]: [63, 63, 51, 63, 63, 64, 64, 64, 62, 62, 65, 65]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:44.962111162Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1324,"":"downstairs","":"downstairs","session_id":"82ef6610-0c26-4b49-a285-db2739bead10"} {"msg":"[1]R generation[0..12]: [33, 33, 28, 33, 33, 34, 34, 34, 32, 32, 34, 34]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:44.962125421Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1324,"":"downstairs","":"downstairs","session_id":"82ef6610-0c26-4b49-a285-db2739bead10"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:44.962139571Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1324,"":"downstairs","":"downstairs","session_id":"82ef6610-0c26-4b49-a285-db2739bead10"} {"msg":"[2]R flush_numbers[0..12]: [63, 63, 51, 63, 63, 64, 64, 64, 62, 62, 65, 65]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:44.96215309Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1324,"":"downstairs","":"downstairs","session_id":"82ef6610-0c26-4b49-a285-db2739bead10"} {"msg":"[2]R generation[0..12]: [33, 33, 28, 33, 33, 34, 34, 34, 32, 32, 34, 34]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:44.962166789Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1324,"":"downstairs","":"downstairs","session_id":"82ef6610-0c26-4b49-a285-db2739bead10"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:44.962180738Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1324,"":"downstairs","":"downstairs","session_id":"82ef6610-0c26-4b49-a285-db2739bead10"} {"msg":"Max found gen is 36","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:44.962193678Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1324,"":"downstairs","":"downstairs","session_id":"82ef6610-0c26-4b49-a285-db2739bead10"} {"msg":"Generation requested: 36 >= found:36","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:44.962207127Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1324,"":"downstairs","":"downstairs","session_id":"82ef6610-0c26-4b49-a285-db2739bead10"} {"msg":"Next flush: 67","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:44.962220137Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1324,"":"downstairs","":"downstairs","session_id":"82ef6610-0c26-4b49-a285-db2739bead10"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:44.962233016Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1324,"":"downstairs","":"downstairs","session_id":"82ef6610-0c26-4b49-a285-db2739bead10"} Wait for a query_work_queue command to finish before sending IO {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:44.962254655Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1324,"session_id":"82ef6610-0c26-4b49-a285-db2739bead10"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:44.962266994Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1324,"client":"0","":"downstairs","session_id":"82ef6610-0c26-4b49-a285-db2739bead10"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:44.962280304Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1324,"client":"1","":"downstairs","session_id":"82ef6610-0c26-4b49-a285-db2739bead10"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:44.962293673Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1324,"client":"2","":"downstairs","session_id":"82ef6610-0c26-4b49-a285-db2739bead10"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:44.962307032Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1324,"session_id":"82ef6610-0c26-4b49-a285-db2739bead10"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:44.962321352Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1324,"session_id":"82ef6610-0c26-4b49-a285-db2739bead10"} {"msg":"f5a81304-f5d3-4bfe-97bb-25565222d384 is now active with session: 82ef6610-0c26-4b49-a285-db2739bead10","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:44.962334341Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1324,"session_id":"82ef6610-0c26-4b49-a285-db2739bead10"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:44.96234631Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1324,"session_id":"82ef6610-0c26-4b49-a285-db2739bead10"} {"msg":"The guest has finished waiting for activation with:36","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:44.96235891Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1324} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 203 len 36864 data: 2 5 5 5 4 3 4 3 3 02/30 Read block 370 len 40960 03/30 Write block 334 len 20480 data: 5 6 6 6 6 04/30 Flush 05/30 Write block 585 len 4096 data: 7 06/30 Read block 143 len 28672 07/30 Write block 578 len 32768 data: 3 2 2 2 2 4 5 8 08/30 Read block 55 len 4096 09/30 Write block 82 len 32768 data: 3 4 3 4 5 5 4 3 10/30 Read block 317 len 36864 11/30 Read block 177 len 24576 12/30 Write block 168 len 16384 data: 4 4 4 3 13/30 Write block 382 len 24576 data: 3 3 3 3 2 2 14/30 Read block 395 len 16384 15/30 Read block 304 len 24576 16/30 Flush 17/30 Write block 132 len 28672 data: 3 5 5 5 4 6 6 18/30 Write block 504 len 4096 data: 2 19/30 Write block 38 len 28672 data: 2 3 3 2 3 3 3 20/30 Read block 80 len 32768 21/30 Write block 21 len 24576 data: 6 6 6 7 7 7 22/30 Flush 23/30 Write block 249 len 8192 data: 7 8 24/30 Read block 23 len 32768 25/30 Read block 484 len 16384 26/30 Write block 164 len 12288 data: 2 2 2 27/30 Write block 468 len 16384 data: 5 5 5 4 28/30 Read block 260 len 20480 29/30 Write block 353 len 8192 data: 5 5 30/30 Write block 141 len 24576 data: 5 5 6 6 5 4 ---------------------------------------------------------------- Crucible gen:36 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 2 Done Done Done false 1023 Acked Read 8 Done Done Done false 1024 Acked Read 4 Done Done Done false 1025 Acked Write 3 Done Done Done false 1026 Acked Write 4 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Write 2 Done Done Done false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 7 7 7 21 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1021 1021 1021 Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:44.990054696Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1324} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 33 33 33 63 63 63 F F F 1 020-039 36 36 36 69 69 69 F F F 2 040-059 36 36 36 69 69 69 F F F 3 060-079 33 33 33 63 63 63 F F F 4 080-099 36 36 36 68 68 68 F F F 5 100-119 34 34 34 64 64 64 F F F 6 120-139 36 36 36 69 69 69 F F F 7 140-159 34 34 34 64 64 64 T T T 8 160-179 36 36 36 68 68 68 T T T 9 180-199 32 32 32 62 62 62 F F F 10 200-219 36 36 36 67 67 67 F F F 11 220-239 34 34 34 65 65 65 F F F 12 240-259 34 34 34 64 64 64 T T T 13 260-279 28 28 28 51 51 51 F F F 14 280-299 30 30 30 53 53 53 F F F 15 300-319 30 30 30 53 53 53 F F F 16 320-339 36 36 36 67 67 67 F F F 17 340-359 31 31 31 56 56 56 T T T 18 360-379 31 31 31 56 56 56 F F F 19 380-399 36 36 36 68 68 68 F F F 20 400-419 35 35 35 66 66 66 F F F 21 420-439 30 30 30 55 55 55 F F F 22 440-459 26 26 26 47 47 47 F F F 23 460-479 34 34 34 64 64 64 T T T 24 480-499 29 29 29 52 52 52 F F F 25 500-519 36 36 36 69 69 69 F F F 26 520-539 33 33 33 63 63 63 F F F 27 540-559 34 34 34 64 64 64 F F F 28 560-579 36 36 36 68 68 68 F F F 29 580-599 36 36 36 68 68 68 F F F Max gen: 36, Max flush: 69 A Difference in extent metadata is expected here On loop 18 Verifying data now No change after verify Loop: 18 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 33 33 33 63 63 63 F F F 1 020-039 36 36 36 69 69 69 F F F 2 040-059 36 36 36 69 69 69 F F F 3 060-079 33 33 33 63 63 63 F F F 4 080-099 36 36 36 68 68 68 F F F 5 100-119 34 34 34 64 64 64 F F F 6 120-139 36 36 36 69 69 69 F F F 7 140-159 37 37 37 70 70 70 F F F 8 160-179 37 37 37 70 70 70 F F F 9 180-199 32 32 32 62 62 62 F F F 10 200-219 36 36 36 67 67 67 F F F 11 220-239 34 34 34 65 65 65 F F F 12 240-259 37 37 37 70 70 70 F F F 13 260-279 28 28 28 51 51 51 F F F 14 280-299 30 30 30 53 53 53 F F F 15 300-319 30 30 30 53 53 53 F F F 16 320-339 36 36 36 67 67 67 F F F 17 340-359 37 37 37 70 70 70 F F F 18 360-379 31 31 31 56 56 56 F F F 19 380-399 36 36 36 68 68 68 F F F 20 400-419 35 35 35 66 66 66 F F F 21 420-439 30 30 30 55 55 55 F F F 22 440-459 26 26 26 47 47 47 F F F 23 460-479 37 37 37 70 70 70 F F F 24 480-499 29 29 29 52 52 52 F F F 25 500-519 36 36 36 69 69 69 F F F 26 520-539 33 33 33 63 63 63 F F F 27 540-559 34 34 34 64 64 64 F F F 28 560-579 36 36 36 68 68 68 F F F 29 580-599 36 36 36 68 68 68 F F F Max gen: 37, Max flush: 70 Begin loop 19 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:49.307707725Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1332,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:49.35056742Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1332,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:49.350967911Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1332,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:49.382882291Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1332,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:49.383471472Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1332} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"0fd98d0d5d909611f8b408ccda2559d35fe117bb\",\n git_commit_timestamp: \"2024-11-08T20:05:44.000000000Z\",\n git_branch: \"mkeeter/drop-skipped-backpressure-guard\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:49.38350717Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1332} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:49.38352219Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1332} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:49.383537399Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1332} {"msg":"Crucible 6690b422-d324-4e06-98fa-508fdf824df8 has session id: 6d1d43f6-aa2f-46a8-a328-0066842e56cf","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:49.383947169Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1332,"session_id":"6d1d43f6-aa2f-46a8-a328-0066842e56cf"} {"msg":"Upstairs opts: Upstairs UUID: 6690b422-d324-4e06-98fa-508fdf824df8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:49.383990167Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1332,"session_id":"6d1d43f6-aa2f-46a8-a328-0066842e56cf"} {"msg":"Crucible stats registered with UUID: 6690b422-d324-4e06-98fa-508fdf824df8","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:49.384024205Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1332,"session_id":"6d1d43f6-aa2f-46a8-a328-0066842e56cf"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:49.414771353Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1332,"task":"crutest"} {"msg":"The guest has requested activation with gen:38","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:49.414802351Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1332} {"msg":"6690b422-d324-4e06-98fa-508fdf824df8 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:49.414850289Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1332,"session_id":"6d1d43f6-aa2f-46a8-a328-0066842e56cf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:49.414869638Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1332,"client":"0","":"downstairs","session_id":"6d1d43f6-aa2f-46a8-a328-0066842e56cf"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:49.414897906Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1332,"client":"0","":"downstairs","session_id":"6d1d43f6-aa2f-46a8-a328-0066842e56cf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:49.414915626Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1332,"client":"1","":"downstairs","session_id":"6d1d43f6-aa2f-46a8-a328-0066842e56cf"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:49.414932994Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1332,"client":"1","":"downstairs","session_id":"6d1d43f6-aa2f-46a8-a328-0066842e56cf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:49.414953324Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1332,"client":"2","":"downstairs","session_id":"6d1d43f6-aa2f-46a8-a328-0066842e56cf"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:49.414970673Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1332,"client":"2","":"downstairs","session_id":"6d1d43f6-aa2f-46a8-a328-0066842e56cf"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:49.414987172Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1332,"":"io task","client":"2","":"downstairs","session_id":"6d1d43f6-aa2f-46a8-a328-0066842e56cf"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:49.41502707Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1332,"":"io task","client":"0","":"downstairs","session_id":"6d1d43f6-aa2f-46a8-a328-0066842e56cf"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:49.415061308Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1332,"":"io task","client":"1","":"downstairs","session_id":"6d1d43f6-aa2f-46a8-a328-0066842e56cf"} {"msg":"ds_connection connected from Ok(127.0.0.1:52115)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:49.415317756Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1332,"":"io task","client":"2","":"downstairs","session_id":"6d1d43f6-aa2f-46a8-a328-0066842e56cf"} {"msg":"ds_connection connected from Ok(127.0.0.1:49813)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:49.415370483Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1332,"":"io task","client":"0","":"downstairs","session_id":"6d1d43f6-aa2f-46a8-a328-0066842e56cf"} {"msg":"ds_connection connected from Ok(127.0.0.1:61811)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:49.415411461Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1332,"":"io task","client":"1","":"downstairs","session_id":"6d1d43f6-aa2f-46a8-a328-0066842e56cf"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:49.416015422Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1332,"client":"1","":"downstairs","session_id":"6d1d43f6-aa2f-46a8-a328-0066842e56cf"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:49.41605106Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1332,"client":"1","":"downstairs","session_id":"6d1d43f6-aa2f-46a8-a328-0066842e56cf"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:49.416079889Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1332,"client":"2","":"downstairs","session_id":"6d1d43f6-aa2f-46a8-a328-0066842e56cf"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:49.416107247Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1332,"client":"2","":"downstairs","session_id":"6d1d43f6-aa2f-46a8-a328-0066842e56cf"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:49.416135186Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1332,"client":"0","":"downstairs","session_id":"6d1d43f6-aa2f-46a8-a328-0066842e56cf"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:49.416157355Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1332,"client":"0","":"downstairs","session_id":"6d1d43f6-aa2f-46a8-a328-0066842e56cf"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:49.416599873Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1332,"client":"1","":"downstairs","session_id":"6d1d43f6-aa2f-46a8-a328-0066842e56cf"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:49.416718877Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1332,"client":"0","":"downstairs","session_id":"6d1d43f6-aa2f-46a8-a328-0066842e56cf"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:49.416801653Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1332,"client":"2","":"downstairs","session_id":"6d1d43f6-aa2f-46a8-a328-0066842e56cf"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:49.417247901Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1332,"client":"1","":"downstairs","session_id":"6d1d43f6-aa2f-46a8-a328-0066842e56cf"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:49.41727626Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1332,"client":"1","":"downstairs","session_id":"6d1d43f6-aa2f-46a8-a328-0066842e56cf"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:49.417290809Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1332,"session_id":"6d1d43f6-aa2f-46a8-a328-0066842e56cf"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:49.417311048Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1332,"client":"0","":"downstairs","session_id":"6d1d43f6-aa2f-46a8-a328-0066842e56cf"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:49.417325028Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1332,"client":"0","":"downstairs","session_id":"6d1d43f6-aa2f-46a8-a328-0066842e56cf"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:49.417338517Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1332,"session_id":"6d1d43f6-aa2f-46a8-a328-0066842e56cf"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:49.417408824Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1332,"client":"2","":"downstairs","session_id":"6d1d43f6-aa2f-46a8-a328-0066842e56cf"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:49.417428543Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1332,"client":"2","":"downstairs","session_id":"6d1d43f6-aa2f-46a8-a328-0066842e56cf"} {"msg":"[0]R flush_numbers[0..12]: [63, 69, 69, 63, 68, 64, 69, 70, 70, 62, 67, 65]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:49.417443542Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1332,"":"downstairs","":"downstairs","session_id":"6d1d43f6-aa2f-46a8-a328-0066842e56cf"} {"msg":"[0]R generation[0..12]: [33, 36, 36, 33, 36, 34, 36, 37, 37, 32, 36, 34]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:49.417457521Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1332,"":"downstairs","":"downstairs","session_id":"6d1d43f6-aa2f-46a8-a328-0066842e56cf"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:49.41747167Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1332,"":"downstairs","":"downstairs","session_id":"6d1d43f6-aa2f-46a8-a328-0066842e56cf"} {"msg":"[1]R flush_numbers[0..12]: [63, 69, 69, 63, 68, 64, 69, 70, 70, 62, 67, 65]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:49.41748543Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1332,"":"downstairs","":"downstairs","session_id":"6d1d43f6-aa2f-46a8-a328-0066842e56cf"} {"msg":"[1]R generation[0..12]: [33, 36, 36, 33, 36, 34, 36, 37, 37, 32, 36, 34]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:49.417498899Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1332,"":"downstairs","":"downstairs","session_id":"6d1d43f6-aa2f-46a8-a328-0066842e56cf"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:49.417515648Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1332,"":"downstairs","":"downstairs","session_id":"6d1d43f6-aa2f-46a8-a328-0066842e56cf"} {"msg":"[2]R flush_numbers[0..12]: [63, 69, 69, 63, 68, 64, 69, 70, 70, 62, 67, 65]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:49.417529448Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1332,"":"downstairs","":"downstairs","session_id":"6d1d43f6-aa2f-46a8-a328-0066842e56cf"} {"msg":"[2]R generation[0..12]: [33, 36, 36, 33, 36, 34, 36, 37, 37, 32, 36, 34]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:49.417543047Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1332,"":"downstairs","":"downstairs","session_id":"6d1d43f6-aa2f-46a8-a328-0066842e56cf"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:49.417556846Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1332,"":"downstairs","":"downstairs","session_id":"6d1d43f6-aa2f-46a8-a328-0066842e56cf"} {"msg":"Max found gen is 38","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:49.417569766Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1332,"":"downstairs","":"downstairs","session_id":"6d1d43f6-aa2f-46a8-a328-0066842e56cf"} {"msg":"Generation requested: 38 >= found:38","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:49.417583055Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1332,"":"downstairs","":"downstairs","session_id":"6d1d43f6-aa2f-46a8-a328-0066842e56cf"} {"msg":"Next flush: 71","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:49.417596015Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1332,"":"downstairs","":"downstairs","session_id":"6d1d43f6-aa2f-46a8-a328-0066842e56cf"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-08T20:23:49.417609294Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1332,"":"downstairs","":"downstairs","session_id":"6d1d43f6-aa2f-46a8-a328-0066842e56cf"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:49.417630713Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1332,"session_id":"6d1d43f6-aa2f-46a8-a328-0066842e56cf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:49.417642992Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1332,"client":"0","":"downstairs","session_id":"6d1d43f6-aa2f-46a8-a328-0066842e56cf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:49.417656301Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1332,"client":"1","":"downstairs","session_id":"6d1d43f6-aa2f-46a8-a328-0066842e56cf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:49.417669501Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1332,"client":"2","":"downstairs","session_id":"6d1d43f6-aa2f-46a8-a328-0066842e56cf"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:49.41768264Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1332,"session_id":"6d1d43f6-aa2f-46a8-a328-0066842e56cf"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:49.41769505Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1332,"session_id":"6d1d43f6-aa2f-46a8-a328-0066842e56cf"} {"msg":"6690b422-d324-4e06-98fa-508fdf824df8 is now active with session: 6d1d43f6-aa2f-46a8-a328-0066842e56cf","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:49.417708059Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1332,"session_id":"6d1d43f6-aa2f-46a8-a328-0066842e56cf"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:49.417722308Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1332,"session_id":"6d1d43f6-aa2f-46a8-a328-0066842e56cf"} {"msg":"The guest has finished waiting for activation with:38","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:49.417734918Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1332} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 266 len 28672 data: 5 3 4 4 4 5 5 02/30 Read block 311 len 20480 03/30 Write block 221 len 28672 data: 5 4 4 4 5 4 6 04/30 Write block 392 len 40960 data: 3 3 3 2 2 3 3 3 3 3 05/30 Write block 518 len 8192 data: 2 3 06/30 Flush 07/30 Write block 376 len 20480 data: 2 2 3 4 3 08/30 Write block 503 len 36864 data: 2 3 2 2 2 4 4 4 4 09/30 Write block 408 len 32768 data: 5 5 5 7 6 5 6 5 10/30 Write block 440 len 8192 data: 3 3 11/30 Write block 319 len 8192 data: 4 3 12/30 Write block 216 len 24576 data: 3 3 5 4 3 6 13/30 Write block 456 len 8192 data: 3 3 14/30 Read block 387 len 32768 15/30 Read block 256 len 40960 16/30 Flush 17/30 Write block 120 len 28672 data: 5 4 5 6 6 3 3 18/30 Read block 377 len 8192 19/30 Write block 441 len 24576 data: 4 3 2 2 3 4 20/30 Write block 545 len 24576 data: 4 4 3 5 5 4 21/30 Read block 581 len 8192 22/30 Write block 540 len 8192 data: 4 4 23/30 Read block 124 len 20480 24/30 Read block 539 len 28672 25/30 Write block 379 len 20480 data: 5 4 3 4 4 26/30 Flush 27/30 Write block 499 len 32768 data: 3 3 3 3 3 4 3 3 28/30 Read block 126 len 32768 29/30 Write block 257 len 28672 data: 4 5 4 4 3 4 4 30/30 Read block 325 len 12288 ---------------------------------------------------------------- Crucible gen:38 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 8 Done Done Done false 1027 Acked Read 8 Done Done Done false 1028 Acked Write 7 Done Done Sent false 1029 Acked Read 3 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 2 2 Done 4 4 2 10 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:49.445489061Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1332} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 33 33 33 63 63 63 F F F 1 020-039 36 36 36 69 69 69 F F F 2 040-059 36 36 36 69 69 69 F F F 3 060-079 33 33 33 63 63 63 F F F 4 080-099 36 36 36 68 68 68 F F F 5 100-119 34 34 34 64 64 64 F F F 6 120-139 38 38 38 73 73 73 F F F 7 140-159 37 37 37 70 70 70 F F F 8 160-179 37 37 37 70 70 70 F F F 9 180-199 32 32 32 62 62 62 F F F 10 200-219 38 38 38 72 72 72 F F F 11 220-239 38 38 38 72 72 72 F F F 12 240-259 37 37 37 70 70 70 T T T 13 260-279 38 38 38 71 71 71 T T T 14 280-299 30 30 30 53 53 53 F F F 15 300-319 38 38 38 72 72 72 F F F 16 320-339 38 38 38 72 72 72 F F F 17 340-359 37 37 37 70 70 70 F F F 18 360-379 38 38 38 73 73 73 F F F 19 380-399 38 38 38 73 73 73 F F F 20 400-419 38 38 38 72 72 72 F F F 21 420-439 30 30 30 55 55 55 F F F 22 440-459 38 38 38 73 73 73 F F F 23 460-479 37 37 37 70 70 70 F F F 24 480-499 29 29 29 52 52 52 T T T 25 500-519 38 38 38 72 72 72 T T T 26 520-539 33 33 33 63 63 63 F F F 27 540-559 38 38 38 73 73 73 F F F 28 560-579 36 36 36 68 68 68 F F F 29 580-599 36 36 36 68 68 68 F F F Max gen: 38, Max flush: 73 A Difference in extent metadata is expected here On loop 19 Verifying data now No change after verify Loop: 19 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 33 33 33 63 63 63 F F F 1 020-039 36 36 36 69 69 69 F F F 2 040-059 36 36 36 69 69 69 F F F 3 060-079 33 33 33 63 63 63 F F F 4 080-099 36 36 36 68 68 68 F F F 5 100-119 34 34 34 64 64 64 F F F 6 120-139 38 38 38 73 73 73 F F F 7 140-159 37 37 37 70 70 70 F F F 8 160-179 37 37 37 70 70 70 F F F 9 180-199 32 32 32 62 62 62 F F F 10 200-219 38 38 38 72 72 72 F F F 11 220-239 38 38 38 72 72 72 F F F 12 240-259 39 39 39 74 74 74 F F F 13 260-279 39 39 39 74 74 74 F F F 14 280-299 30 30 30 53 53 53 F F F 15 300-319 38 38 38 72 72 72 F F F 16 320-339 38 38 38 72 72 72 F F F 17 340-359 37 37 37 70 70 70 F F F 18 360-379 38 38 38 73 73 73 F F F 19 380-399 38 38 38 73 73 73 F F F 20 400-419 38 38 38 72 72 72 F F F 21 420-439 30 30 30 55 55 55 F F F 22 440-459 38 38 38 73 73 73 F F F 23 460-479 37 37 37 70 70 70 F F F 24 480-499 39 39 39 74 74 74 F F F 25 500-519 39 39 39 74 74 74 F F F 26 520-539 33 33 33 63 63 63 F F F 27 540-559 38 38 38 73 73 73 F F F 28 560-579 36 36 36 68 68 68 F F F 29 580-599 36 36 36 68 68 68 F F F Max gen: 39, Max flush: 74 Begin loop 20 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:53.786153354Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1340,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:53.828911424Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1340,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:53.829302796Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1340,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:53.861139929Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1340,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:53.861693982Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1340} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"0fd98d0d5d909611f8b408ccda2559d35fe117bb\",\n git_commit_timestamp: \"2024-11-08T20:05:44.000000000Z\",\n git_branch: \"mkeeter/drop-skipped-backpressure-guard\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:53.8617474Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1340} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:53.861772918Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1340} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:53.861792048Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1340} {"msg":"Crucible 47ef9d0d-5ca2-4919-a155-088c42fcbe3d has session id: ddf36eef-49f9-4a09-99ad-aeb3fc475d71","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:53.862186778Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1340,"session_id":"ddf36eef-49f9-4a09-99ad-aeb3fc475d71"} {"msg":"Upstairs opts: Upstairs UUID: 47ef9d0d-5ca2-4919-a155-088c42fcbe3d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:53.862233326Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1340,"session_id":"ddf36eef-49f9-4a09-99ad-aeb3fc475d71"} {"msg":"Crucible stats registered with UUID: 47ef9d0d-5ca2-4919-a155-088c42fcbe3d","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:53.862253455Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1340,"session_id":"ddf36eef-49f9-4a09-99ad-aeb3fc475d71"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:53.893813492Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1340,"task":"crutest"} {"msg":"The guest has requested activation with gen:40","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:53.893848251Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1340} {"msg":"47ef9d0d-5ca2-4919-a155-088c42fcbe3d active request set","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:53.89387327Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1340,"session_id":"ddf36eef-49f9-4a09-99ad-aeb3fc475d71"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:53.893890079Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1340,"client":"0","":"downstairs","session_id":"ddf36eef-49f9-4a09-99ad-aeb3fc475d71"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:53.893917627Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1340,"client":"0","":"downstairs","session_id":"ddf36eef-49f9-4a09-99ad-aeb3fc475d71"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:53.893935607Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1340,"client":"1","":"downstairs","session_id":"ddf36eef-49f9-4a09-99ad-aeb3fc475d71"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:53.893952816Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1340,"client":"1","":"downstairs","session_id":"ddf36eef-49f9-4a09-99ad-aeb3fc475d71"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:53.893969225Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1340,"client":"2","":"downstairs","session_id":"ddf36eef-49f9-4a09-99ad-aeb3fc475d71"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:53.893985864Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1340,"client":"2","":"downstairs","session_id":"ddf36eef-49f9-4a09-99ad-aeb3fc475d71"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:53.894006063Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1340,"":"io task","client":"2","":"downstairs","session_id":"ddf36eef-49f9-4a09-99ad-aeb3fc475d71"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:53.894023672Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1340,"":"io task","client":"0","":"downstairs","session_id":"ddf36eef-49f9-4a09-99ad-aeb3fc475d71"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:53.894052971Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1340,"":"io task","client":"1","":"downstairs","session_id":"ddf36eef-49f9-4a09-99ad-aeb3fc475d71"} {"msg":"ds_connection connected from Ok(127.0.0.1:62157)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:53.894181745Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1340,"":"io task","client":"2","":"downstairs","session_id":"ddf36eef-49f9-4a09-99ad-aeb3fc475d71"} {"msg":"ds_connection connected from Ok(127.0.0.1:46969)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:53.894329787Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1340,"":"io task","client":"1","":"downstairs","session_id":"ddf36eef-49f9-4a09-99ad-aeb3fc475d71"} {"msg":"ds_connection connected from Ok(127.0.0.1:62700)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:53.894412263Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1340,"":"io task","client":"0","":"downstairs","session_id":"ddf36eef-49f9-4a09-99ad-aeb3fc475d71"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:53.894952197Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1340,"client":"2","":"downstairs","session_id":"ddf36eef-49f9-4a09-99ad-aeb3fc475d71"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:53.894981015Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1340,"client":"2","":"downstairs","session_id":"ddf36eef-49f9-4a09-99ad-aeb3fc475d71"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:53.894996405Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1340,"client":"1","":"downstairs","session_id":"ddf36eef-49f9-4a09-99ad-aeb3fc475d71"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:53.895010754Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1340,"client":"1","":"downstairs","session_id":"ddf36eef-49f9-4a09-99ad-aeb3fc475d71"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:53.895024193Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1340,"client":"0","":"downstairs","session_id":"ddf36eef-49f9-4a09-99ad-aeb3fc475d71"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:53.895037613Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1340,"client":"0","":"downstairs","session_id":"ddf36eef-49f9-4a09-99ad-aeb3fc475d71"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:53.895732819Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1340,"client":"2","":"downstairs","session_id":"ddf36eef-49f9-4a09-99ad-aeb3fc475d71"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:53.895774497Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1340,"client":"1","":"downstairs","session_id":"ddf36eef-49f9-4a09-99ad-aeb3fc475d71"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:53.895808195Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1340,"client":"0","":"downstairs","session_id":"ddf36eef-49f9-4a09-99ad-aeb3fc475d71"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:53.89631249Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1340,"client":"2","":"downstairs","session_id":"ddf36eef-49f9-4a09-99ad-aeb3fc475d71"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:53.896340609Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1340,"client":"2","":"downstairs","session_id":"ddf36eef-49f9-4a09-99ad-aeb3fc475d71"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:53.896355418Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1340,"session_id":"ddf36eef-49f9-4a09-99ad-aeb3fc475d71"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:53.896450444Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1340,"client":"1","":"downstairs","session_id":"ddf36eef-49f9-4a09-99ad-aeb3fc475d71"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:53.896470873Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1340,"client":"1","":"downstairs","session_id":"ddf36eef-49f9-4a09-99ad-aeb3fc475d71"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:53.896484542Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1340,"session_id":"ddf36eef-49f9-4a09-99ad-aeb3fc475d71"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:53.896496701Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1340,"client":"0","":"downstairs","session_id":"ddf36eef-49f9-4a09-99ad-aeb3fc475d71"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:53.896509991Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1340,"client":"0","":"downstairs","session_id":"ddf36eef-49f9-4a09-99ad-aeb3fc475d71"} {"msg":"[0]R flush_numbers[0..12]: [63, 69, 69, 63, 68, 64, 73, 70, 70, 62, 72, 72]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:53.89652364Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1340,"":"downstairs","":"downstairs","session_id":"ddf36eef-49f9-4a09-99ad-aeb3fc475d71"} {"msg":"[0]R generation[0..12]: [33, 36, 36, 33, 36, 34, 38, 37, 37, 32, 38, 38]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:53.896538219Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1340,"":"downstairs","":"downstairs","session_id":"ddf36eef-49f9-4a09-99ad-aeb3fc475d71"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:53.896552559Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1340,"":"downstairs","":"downstairs","session_id":"ddf36eef-49f9-4a09-99ad-aeb3fc475d71"} {"msg":"[1]R flush_numbers[0..12]: [63, 69, 69, 63, 68, 64, 73, 70, 70, 62, 72, 72]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:53.896566258Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1340,"":"downstairs","":"downstairs","session_id":"ddf36eef-49f9-4a09-99ad-aeb3fc475d71"} {"msg":"[1]R generation[0..12]: [33, 36, 36, 33, 36, 34, 38, 37, 37, 32, 38, 38]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:53.896579867Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1340,"":"downstairs","":"downstairs","session_id":"ddf36eef-49f9-4a09-99ad-aeb3fc475d71"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:53.896593797Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1340,"":"downstairs","":"downstairs","session_id":"ddf36eef-49f9-4a09-99ad-aeb3fc475d71"} {"msg":"[2]R flush_numbers[0..12]: [63, 69, 69, 63, 68, 64, 73, 70, 70, 62, 72, 72]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:53.896607506Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1340,"":"downstairs","":"downstairs","session_id":"ddf36eef-49f9-4a09-99ad-aeb3fc475d71"} {"msg":"[2]R generation[0..12]: [33, 36, 36, 33, 36, 34, 38, 37, 37, 32, 38, 38]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:53.896624405Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1340,"":"downstairs","":"downstairs","session_id":"ddf36eef-49f9-4a09-99ad-aeb3fc475d71"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:53.896638645Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1340,"":"downstairs","":"downstairs","session_id":"ddf36eef-49f9-4a09-99ad-aeb3fc475d71"} {"msg":"Max found gen is 40","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:53.896651844Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1340,"":"downstairs","":"downstairs","session_id":"ddf36eef-49f9-4a09-99ad-aeb3fc475d71"} {"msg":"Generation requested: 40 >= found:40","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:53.896665043Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1340,"":"downstairs","":"downstairs","session_id":"ddf36eef-49f9-4a09-99ad-aeb3fc475d71"} {"msg":"Next flush: 75","v":0,Wait for a query_work_queue command to finish before sending IO "name":"crucible","level":30,"time":"2024-11-08T20:23:53.89673207Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1340,"":"downstairs","":"downstairs","session_id":"ddf36eef-49f9-4a09-99ad-aeb3fc475d71"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:53.896752669Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1340,"":"downstairs","":"downstairs","session_id":"ddf36eef-49f9-4a09-99ad-aeb3fc475d71"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:53.896766688Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1340,"session_id":"ddf36eef-49f9-4a09-99ad-aeb3fc475d71"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:53.896778978Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1340,"client":"0","":"downstairs","session_id":"ddf36eef-49f9-4a09-99ad-aeb3fc475d71"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:53.896792387Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1340,"client":"1","":"downstairs","session_id":"ddf36eef-49f9-4a09-99ad-aeb3fc475d71"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:53.896805656Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1340,"client":"2","":"downstairs","session_id":"ddf36eef-49f9-4a09-99ad-aeb3fc475d71"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:53.896819116Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1340,"session_id":"ddf36eef-49f9-4a09-99ad-aeb3fc475d71"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:53.896831405Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1340,"session_id":"ddf36eef-49f9-4a09-99ad-aeb3fc475d71"} {"msg":"47ef9d0d-5ca2-4919-a155-088c42fcbe3d is now active with session: ddf36eef-49f9-4a09-99ad-aeb3fc475d71","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:53.896844304Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1340,"session_id":"ddf36eef-49f9-4a09-99ad-aeb3fc475d71"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:53.896856214Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1340,"session_id":"ddf36eef-49f9-4a09-99ad-aeb3fc475d71"} {"msg":"The guest has finished waiting for activation with:40","v":0,"name":"crucible","level":30Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2024-11-08T20:23:53.896868443Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1340} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 364 len 4096 data: 5 02/30 Write block 293 len 4096 data: 5 03/30 Read block 266 len 4096 04/30 Write block 207 len 24576 data: 5 4 5 4 4 3 05/30 Read block 575 len 24576 06/30 Read block 3 len 40960 07/30 Write block 360 len 24576 data: 9 9 7 6 6 4 08/30 Read block 501 len 12288 09/30 Write block 192 len 28672 data: 4 4 4 3 3 3 3 10/30 Flush 11/30 Write block 357 len 24576 data: 8 10 10 10 10 8 12/30 Read block 263 len 28672 13/30 Write block 586 len 40960 data: 7 7 6 5 5 4 4 4 2 2 14/30 Read block 415 len 24576 15/30 Write block 168 len 32768 data: 5 5 5 4 2 2 2 2 16/30 Write block 43 len 12288 data: 4 4 3 17/30 Read block 1 len 4096 18/30 Read block 423 len 12288 19/30 Write block 410 len 16384 data: 6 8 7 6 20/30 Write block 347 len 12288 data: 6 4 5 21/30 Flush 22/30 Write block 2 len 4096 data: 3 23/30 Read block 481 len 8192 24/30 Read block 97 len 28672 25/30 Write block 268 len 40960 data: 5 5 5 6 6 5 7 7 7 7 26/30 Read block 170 len 16384 27/30 Write block 408 len 4096 data: 6 28/30 Read block 545 len 28672 29/30 Write block 263 len 20480 data: 5 5 6 6 4 30/30 Read block 135 len 8192 ---------------------------------------------------------------- Crucible gen:40 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 1 Done Done Done false 1022 Acked Read 2 Done Done Done false 1023 Acked Read 7 Done Done Done false 1024 Acked Write 10 Done Done Done false 1025 Acked Read 4 Done Done Done false 1026 Acked Write 1 Done Done Done false 1027 Acked Read 7 Done Done Done false 1028 Acked Write 5 Done Done Done false 1029 Acked Read 2 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 9 9 9 27 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1020 1020 1020 Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:53.921198414Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1340} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 33 33 33 63 63 63 T T T 1 020-039 36 36 36 69 69 69 F F F 2 040-059 40 40 40 76 76 76 F F F 3 060-079 33 33 33 63 63 63 F F F 4 080-099 36 36 36 68 68 68 F F F 5 100-119 34 34 34 64 64 64 F F F 6 120-139 38 38 38 73 73 73 F F F 7 140-159 37 37 37 70 70 70 F F F 8 160-179 40 40 40 76 76 76 F F F 9 180-199 40 40 40 75 75 75 F F F 10 200-219 40 40 40 75 75 75 F F F 11 220-239 38 38 38 72 72 72 F F F 12 240-259 39 39 39 74 74 74 F F F 13 260-279 39 39 39 74 74 74 T T T 14 280-299 40 40 40 75 75 75 F F F 15 300-319 38 38 38 72 72 72 F F F 16 320-339 38 38 38 72 72 72 F F F 17 340-359 40 40 40 76 76 76 F F F 18 360-379 40 40 40 76 76 76 F F F 19 380-399 38 38 38 73 73 73 F F F 20 400-419 40 40 40 76 76 76 T T T 21 420-439 30 30 30 55 55 55 F F F 22 440-459 38 38 38 73 73 73 F F F 23 460-479 37 37 37 70 70 70 F F F 24 480-499 39 39 39 74 74 74 F F F 25 500-519 39 39 39 74 74 74 F F F 26 520-539 33 33 33 63 63 63 F F F 27 540-559 38 38 38 73 73 73 F F F 28 560-579 36 36 36 68 68 68 F F F 29 580-599 40 40 40 76 76 76 F F F Max gen: 40, Max flush: 76 A Difference in extent metadata is expected here On loop 20 Verifying data now No change after verify Loop: 20 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 41 41 41 77 77 77 F F F 1 020-039 36 36 36 69 69 69 F F F 2 040-059 40 40 40 76 76 76 F F F 3 060-079 33 33 33 63 63 63 F F F 4 080-099 36 36 36 68 68 68 F F F 5 100-119 34 34 34 64 64 64 F F F 6 120-139 38 38 38 73 73 73 F F F 7 140-159 37 37 37 70 70 70 F F F 8 160-179 40 40 40 76 76 76 F F F 9 180-199 40 40 40 75 75 75 F F F 10 200-219 40 40 40 75 75 75 F F F 11 220-239 38 38 38 72 72 72 F F F 12 240-259 39 39 39 74 74 74 F F F 13 260-279 41 41 41 77 77 77 F F F 14 280-299 40 40 40 75 75 75 F F F 15 300-319 38 38 38 72 72 72 F F F 16 320-339 38 38 38 72 72 72 F F F 17 340-359 40 40 40 76 76 76 F F F 18 360-379 40 40 40 76 76 76 F F F 19 380-399 38 38 38 73 73 73 F F F 20 400-419 41 41 41 77 77 77 F F F 21 420-439 30 30 30 55 55 55 F F F 22 440-459 38 38 38 73 73 73 F F F 23 460-479 37 37 37 70 70 70 F F F 24 480-499 39 39 39 74 74 74 F F F 25 500-519 39 39 39 74 74 74 F F F 26 520-539 33 33 33 63 63 63 F F F 27 540-559 38 38 38 73 73 73 F F F 28 560-579 36 36 36 68 68 68 F F F 29 580-599 40 40 40 76 76 76 F F F Max gen: 41, Max flush: 77 Begin loop 21 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:58.240710781Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1348,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:58.285701692Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1348,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:58.286448416Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1348} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"0fd98d0d5d909611f8b408ccda2559d35fe117bb\",\n git_commit_timestamp: \"2024-11-08T20:05:44.000000000Z\",\n git_branch: \"mkeeter/drop-skipped-backpressure-guard\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:58.286486334Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1348} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:58.286500453Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1348} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:58.286514582Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1348} {"msg":"Crucible 92866326-5140-4bfb-9b48-f1e59ad1b17e has session id: 23eba02f-11e1-457a-ae2d-c510663e5c6b","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:58.286951771Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1348,"session_id":"23eba02f-11e1-457a-ae2d-c510663e5c6b"} {"msg":"Upstairs opts: Upstairs UUID: 92866326-5140-4bfb-9b48-f1e59ad1b17e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:58.287000418Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1348,"session_id":"23eba02f-11e1-457a-ae2d-c510663e5c6b"} {"msg":"Crucible stats registered with UUID: 92866326-5140-4bfb-9b48-f1e59ad1b17e","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:58.287018138Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1348,"session_id":"23eba02f-11e1-457a-ae2d-c510663e5c6b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:58.318628723Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1348,"task":"crutest"} {"msg":"The guest has requested activation with gen:42","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:58.318661001Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1348} {"msg":"92866326-5140-4bfb-9b48-f1e59ad1b17e active request set","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:58.31869588Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1348,"session_id":"23eba02f-11e1-457a-ae2d-c510663e5c6b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:58.318712279Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1348,"client":"0","":"downstairs","session_id":"23eba02f-11e1-457a-ae2d-c510663e5c6b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:58.318740287Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1348,"client":"0","":"downstairs","session_id":"23eba02f-11e1-457a-ae2d-c510663e5c6b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:58.318769436Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1348,"client":"1","":"downstairs","session_id":"23eba02f-11e1-457a-ae2d-c510663e5c6b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:58.318803994Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1348,"client":"1","":"downstairs","session_id":"23eba02f-11e1-457a-ae2d-c510663e5c6b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:58.318839433Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1348,"client":"2","":"downstairs","session_id":"23eba02f-11e1-457a-ae2d-c510663e5c6b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:58.318874731Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1348,"client":"2","":"downstairs","session_id":"23eba02f-11e1-457a-ae2d-c510663e5c6b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:58.318905859Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1348,"":"io task","client":"2","":"downstairs","session_id":"23eba02f-11e1-457a-ae2d-c510663e5c6b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:58.318939868Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1348,"":"io task","client":"0","":"downstairs","session_id":"23eba02f-11e1-457a-ae2d-c510663e5c6b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:58.318975046Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1348,"":"io task","client":"1","":"downstairs","session_id":"23eba02f-11e1-457a-ae2d-c510663e5c6b"} {"msg":"ds_connection connected from Ok(127.0.0.1:43695)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:58.319142838Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1348,"":"io task","client":"0","":"downstairs","session_id":"23eba02f-11e1-457a-ae2d-c510663e5c6b"} {"msg":"ds_connection connected from Ok(127.0.0.1:56235)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:58.319187755Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1348,"":"io task","client":"2","":"downstairs","session_id":"23eba02f-11e1-457a-ae2d-c510663e5c6b"} {"msg":"ds_connection connected from Ok(127.0.0.1:46162)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:58.319216964Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1348,"":"io task","client":"1","":"downstairs","session_id":"23eba02f-11e1-457a-ae2d-c510663e5c6b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:58.319727159Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1348,"client":"2","":"downstairs","session_id":"23eba02f-11e1-457a-ae2d-c510663e5c6b"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:58.319767917Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1348,"client":"2","":"downstairs","session_id":"23eba02f-11e1-457a-ae2d-c510663e5c6b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:58.3199195Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1348,"client":"1","":"downstairs","session_id":"23eba02f-11e1-457a-ae2d-c510663e5c6b"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:58.319961307Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1348,"client":"1","":"downstairs","session_id":"23eba02f-11e1-457a-ae2d-c510663e5c6b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:58.319991106Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1348,"client":"0","":"downstairs","session_id":"23eba02f-11e1-457a-ae2d-c510663e5c6b"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:58.320033394Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1348,"client":"0","":"downstairs","session_id":"23eba02f-11e1-457a-ae2d-c510663e5c6b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:58.320335969Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1348,"client":"2","":"downstairs","session_id":"23eba02f-11e1-457a-ae2d-c510663e5c6b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:58.320820026Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1348,"client":"1","":"downstairs","session_id":"23eba02f-11e1-457a-ae2d-c510663e5c6b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:58.320862483Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1348,"client":"0","":"downstairs","session_id":"23eba02f-11e1-457a-ae2d-c510663e5c6b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:58.320896022Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1348,"client":"2","":"downstairs","session_id":"23eba02f-11e1-457a-ae2d-c510663e5c6b"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:58.32092502Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1348,"client":"2","":"downstairs","session_id":"23eba02f-11e1-457a-ae2d-c510663e5c6b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:58.320952039Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1348,"session_id":"23eba02f-11e1-457a-ae2d-c510663e5c6b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:58.321385928Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1348,"client":"1","":"downstairs","session_id":"23eba02f-11e1-457a-ae2d-c510663e5c6b"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:58.321417776Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1348,"client":"1","":"downstairs","session_id":"23eba02f-11e1-457a-ae2d-c510663e5c6b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:58.321434186Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1348,"session_id":"23eba02f-11e1-457a-ae2d-c510663e5c6b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:58.321447135Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1348,"client":"0","":"downstairs","session_id":"23eba02f-11e1-457a-ae2d-c510663e5c6b"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:58.321460764Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1348,"client":"0","":"downstairs","session_id":"23eba02f-11e1-457a-ae2d-c510663e5c6b"} {"msg":"[0]R flush_numbers[0..12]: [77, 69, 76, 63, 68, 64, 73, 70, 76, 75, 75, 72]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:58.321474834Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1348,"":"downstairs","":"downstairs","session_id":"23eba02f-11e1-457a-ae2d-c510663e5c6b"} {"msg":"[0]R generation[0..12]: [41, 36, 40, 33, 36, 34, 38, 37, 40, 40, 40, 38]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:58.321488663Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1348,"":"downstairs","":"downstairs","session_id":"23eba02f-11e1-457a-ae2d-c510663e5c6b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:58.321502842Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1348,"":"downstairs","":"downstairs","session_id":"23eba02f-11e1-457a-ae2d-c510663e5c6b"} {"msg":"[1]R flush_numbers[0..12]: [77, 69, 76, 63, 68, 64, 73, 70, 76, 75, 75, 72]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:58.321516662Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1348,"":"downstairs","":"downstairs","session_id":"23eba02f-11e1-457a-ae2d-c510663e5c6b"} {"msg":"[1]R generation[0..12]: [41, 36, 40, 33, 36, 34, 38, 37, 40, 40, 40, 38]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:58.321530191Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1348,"":"downstairs","":"downstairs","session_id":"23eba02f-11e1-457a-ae2d-c510663e5c6b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:58.32154426Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1348,"":"downstairs","":"downstairs","session_id":"23eba02f-11e1-457a-ae2d-c510663e5c6b"} {"msg":"[2]R flush_numbers[0..12]: [77, 69, 76, 63, 68, 64, 73, 70, 76, 75, 75, 72]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:58.32155806Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1348,"":"downstairs","":"downstairs","session_id":"23eba02f-11e1-457a-ae2d-c510663e5c6b"} {"msg":"[2]R generation[0..12]: [41, 36, 40, 33, 36, 34, 38, 37, 40, 40, 40, 38]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:58.321571569Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1348,"":"downstairs","":"downstairs","session_id":"23eba02f-11e1-457a-ae2d-c510663e5c6b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:58.321585508Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1348,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO 23eba02f-11e1-457a-ae2d-c510663e5c6b"} {"msg":"Max found gen is 42","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:58.321606127Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1348,"":"downstairs","":"downstairs","session_id":"23eba02f-11e1-457a-ae2d-c510663e5c6b"} {"msg":"Generation requested: 42 >= found:42","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:58.321621257Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1348,"":"downstairs","":"downstairs","session_id":"23eba02f-11e1-457a-ae2d-c510663e5c6b"} {"msg":"Next flush: 78","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:58.321634606Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1348,"":"downstairs","":"downstairs","session_id":"23eba02f-11e1-457a-ae2d-c510663e5c6b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:58.321647645Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1348,"":"downstairs","":"downstairs","session_id":"23eba02f-11e1-457a-ae2d-c510663e5c6b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:58.321660744Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1348,"session_id":"23eba02f-11e1-457a-ae2d-c510663e5c6b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:58.321672614Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1348,"client":"0","":"downstairs","session_id":"23eba02f-11e1-457a-ae2d-c510663e5c6b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:58.321685873Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1348,"client":"1","":"downstairs","session_id":"23eba02f-11e1-457a-ae2d-c510663e5c6b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:58.321698903Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1348,"client":"2","":"downstairs","session_id":"23eba02f-11e1-457a-ae2d-c510663e5c6b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:58.321712062Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1348,"session_id":"23eba02f-11e1-457a-ae2d-c510663e5c6b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:58.321724272Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1348,"session_id":"23eba02f-11e1-457a-ae2d-c510663e5c6b"} {"msg":"92866326-5140-4bfb-9b48-f1e59ad1b17e is now active with session: 23eba02f-11e1-457a-ae2d-c510663e5c6b","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:58.321736901Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1348,"session_id":"23eba02f-11e1-457a-ae2d-c510663e5c6b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:58.32174895Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1348,"session_id":"23eba02f-11e1-457a-ae2d-c510663e5c6b"} {"msg":"The guest has finished waiting for activation with:42","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:23:58.32176143Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1348} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 70 len 24576 data: 4 3 4 4 7 7 02/30 Write block 407 len 32768 data: 6 7 6 7 9 8 7 7 03/30 Read block 595 len 16384 04/30 Read block 125 len 24576 05/30 Flush 06/30 Write block 190 len 12288 data: 6 4 5 07/30 Read block 539 len 36864 08/30 Flush 09/30 Write block 244 len 36864 data: 5 4 4 6 7 8 9 8 9 10/30 Write block 564 len 40960 data: 2 2 2 2 3 3 3 5 5 5 11/30 Read block 430 len 28672 12/30 Read block 319 len 36864 13/30 Read block 399 len 16384 14/30 Write block 26 len 40960 data: 8 6 4 3 4 6 6 5 5 3 15/30 Read block 319 len 8192 16/30 Flush 17/30 Write block 377 len 8192 data: 3 4 18/30 Flush 19/30 Write block 599 len 4096 data: 2 20/30 Read block 534 len 24576 21/30 Write block 413 len 40960 data: 8 8 6 5 4 5 5 5 5 4 22/30 Read block 570 len 8192 23/30 Flush 24/30 Write block 232 len 36864 data: 4 3 2 3 3 5 5 5 6 25/30 Read block 415 len 40960 26/30 Write block 355 len 12288 data: 7 7 9 27/30 Read block 509 len 20480 28/30 Write block 472 len 12288 data: 4 5 5 29/30 Read block 100 len 4096 30/30 Write block 363 len 40960 data: 7 7 5 5 6 8 7 6 6 7 ---------------------------------------------------------------- Crucible gen:42 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 9 Done Done Done false 1024 Acked Read 10 Done Done Done false 1025 Acked Write 3 Done Done Done false 1026 Acked Read 5 Done Done Done false 1027 Acked Write 3 Done Done Done false 1028 Acked Read 1 Done Done Done false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 6 6 6 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:23:58.350760582Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1348} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 41 41 41 77 77 77 F F F 1 020-039 42 42 42 80 80 80 F F F 2 040-059 40 40 40 76 76 76 F F F 3 060-079 42 42 42 78 78 78 F F F 4 080-099 36 36 36 68 68 68 F F F 5 100-119 34 34 34 64 64 64 F F F 6 120-139 38 38 38 73 73 73 F F F 7 140-159 37 37 37 70 70 70 F F F 8 160-179 40 40 40 76 76 76 F F F 9 180-199 42 42 42 79 79 79 F F F 10 200-219 40 40 40 75 75 75 F F F 11 220-239 38 38 38 72 72 72 T T T 12 240-259 42 42 42 80 80 80 T T T 13 260-279 41 41 41 77 77 77 F F F 14 280-299 40 40 40 75 75 75 F F F 15 300-319 38 38 38 72 72 72 F F F 16 320-339 38 38 38 72 72 72 F F F 17 340-359 40 40 40 76 76 76 T T T 18 360-379 42 42 42 81 81 81 T T T 19 380-399 38 38 38 73 73 73 F F F 20 400-419 42 42 42 82 82 82 F F F 21 420-439 42 42 42 82 82 82 F F F 22 440-459 38 38 38 73 73 73 F F F 23 460-479 37 37 37 70 70 70 T T T 24 480-499 39 39 39 74 74 74 F F F 25 500-519 39 39 39 74 74 74 F F F 26 520-539 33 33 33 63 63 63 F F F 27 540-559 38 38 38 73 73 73 F F F 28 560-579 42 42 42 80 80 80 F F F 29 580-599 42 42 42 82 82 82 F F F Max gen: 42, Max flush: 82 A Difference in extent metadata is expected here On loop 21 Verifying data now No change after verify Loop: 21 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 41 41 41 77 77 77 F F F 1 020-039 42 42 42 80 80 80 F F F 2 040-059 40 40 40 76 76 76 F F F 3 060-079 42 42 42 78 78 78 F F F 4 080-099 36 36 36 68 68 68 F F F 5 100-119 34 34 34 64 64 64 F F F 6 120-139 38 38 38 73 73 73 F F F 7 140-159 37 37 37 70 70 70 F F F 8 160-179 40 40 40 76 76 76 F F F 9 180-199 42 42 42 79 79 79 F F F 10 200-219 40 40 40 75 75 75 F F F 11 220-239 43 43 43 83 83 83 F F F 12 240-259 43 43 43 83 83 83 F F F 13 260-279 41 41 41 77 77 77 F F F 14 280-299 40 40 40 75 75 75 F F F 15 300-319 38 38 38 72 72 72 F F F 16 320-339 38 38 38 72 72 72 F F F 17 340-359 43 43 43 83 83 83 F F F 18 360-379 43 43 43 83 83 83 F F F 19 380-399 38 38 38 73 73 73 F F F 20 400-419 42 42 42 82 82 82 F F F 21 420-439 42 42 42 82 82 82 F F F 22 440-459 38 38 38 73 73 73 F F F 23 460-479 43 43 43 83 83 83 F F F 24 480-499 39 39 39 74 74 74 F F F 25 500-519 39 39 39 74 74 74 F F F 26 520-539 33 33 33 63 63 63 F F F 27 540-559 38 38 38 73 73 73 F F F 28 560-579 42 42 42 80 80 80 F F F 29 580-599 42 42 42 82 82 82 F F F Max gen: 43, Max flush: 83 Begin loop 22 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:02.673003907Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1356,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:02.717885503Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1356,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:02.71857688Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1356} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"0fd98d0d5d909611f8b408ccda2559d35fe117bb\",\n git_commit_timestamp: \"2024-11-08T20:05:44.000000000Z\",\n git_branch: \"mkeeter/drop-skipped-backpressure-guard\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:02.718631127Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1356} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:02.718657356Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1356} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:02.718681884Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1356} {"msg":"Crucible f14b69fd-6e02-47e9-87e5-df2a54c1aa35 has session id: 91ffb351-cc0f-48b0-87ed-896077a3912a","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:02.719027398Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1356,"session_id":"91ffb351-cc0f-48b0-87ed-896077a3912a"} {"msg":"Upstairs opts: Upstairs UUID: f14b69fd-6e02-47e9-87e5-df2a54c1aa35, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:02.719072815Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1356,"session_id":"91ffb351-cc0f-48b0-87ed-896077a3912a"} {"msg":"Crucible stats registered with UUID: f14b69fd-6e02-47e9-87e5-df2a54c1aa35","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:02.719098714Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1356,"session_id":"91ffb351-cc0f-48b0-87ed-896077a3912a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:02.750327148Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1356,"task":"crutest"} {"msg":"The guest has requested activation with gen:44","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:02.750373735Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1356} {"msg":"f14b69fd-6e02-47e9-87e5-df2a54c1aa35 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:02.750412234Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1356,"session_id":"91ffb351-cc0f-48b0-87ed-896077a3912a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:02.750444562Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1356,"client":"0","":"downstairs","session_id":"91ffb351-cc0f-48b0-87ed-896077a3912a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:02.75049348Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1356,"client":"0","":"downstairs","session_id":"91ffb351-cc0f-48b0-87ed-896077a3912a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:02.750529008Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1356,"client":"1","":"downstairs","session_id":"91ffb351-cc0f-48b0-87ed-896077a3912a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:02.750562906Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1356,"client":"1","":"downstairs","session_id":"91ffb351-cc0f-48b0-87ed-896077a3912a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:02.750584045Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1356,"client":"2","":"downstairs","session_id":"91ffb351-cc0f-48b0-87ed-896077a3912a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:02.750606454Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1356,"client":"2","":"downstairs","session_id":"91ffb351-cc0f-48b0-87ed-896077a3912a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:02.750637813Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1356,"":"io task","client":"0","":"downstairs","session_id":"91ffb351-cc0f-48b0-87ed-896077a3912a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:02.750671271Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1356,"":"io task","client":"2","":"downstairs","session_id":"91ffb351-cc0f-48b0-87ed-896077a3912a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:02.750707589Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1356,"":"io task","client":"1","":"downstairs","session_id":"91ffb351-cc0f-48b0-87ed-896077a3912a"} {"msg":"ds_connection connected from Ok(127.0.0.1:46560)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:02.750741917Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1356,"":"io task","client":"1","":"downstairs","session_id":"91ffb351-cc0f-48b0-87ed-896077a3912a"} {"msg":"ds_connection connected from Ok(127.0.0.1:53216)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:02.750778646Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1356,"":"io task","client":"2","":"downstairs","session_id":"91ffb351-cc0f-48b0-87ed-896077a3912a"} {"msg":"ds_connection connected from Ok(127.0.0.1:49194)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:02.750967357Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1356,"":"io task","client":"0","":"downstairs","session_id":"91ffb351-cc0f-48b0-87ed-896077a3912a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:02.751494371Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1356,"client":"0","":"downstairs","session_id":"91ffb351-cc0f-48b0-87ed-896077a3912a"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:02.751524189Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1356,"client":"0","":"downstairs","session_id":"91ffb351-cc0f-48b0-87ed-896077a3912a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:02.751546598Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1356,"client":"1","":"downstairs","session_id":"91ffb351-cc0f-48b0-87ed-896077a3912a"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:02.751573097Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1356,"client":"1","":"downstairs","session_id":"91ffb351-cc0f-48b0-87ed-896077a3912a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:02.751599286Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1356,"client":"2","":"downstairs","session_id":"91ffb351-cc0f-48b0-87ed-896077a3912a"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:02.751626554Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1356,"client":"2","":"downstairs","session_id":"91ffb351-cc0f-48b0-87ed-896077a3912a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:02.752194386Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1356,"client":"0","":"downstairs","session_id":"91ffb351-cc0f-48b0-87ed-896077a3912a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:02.752239684Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1356,"client":"1","":"downstairs","session_id":"91ffb351-cc0f-48b0-87ed-896077a3912a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:02.752298991Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1356,"client":"2","":"downstairs","session_id":"91ffb351-cc0f-48b0-87ed-896077a3912a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:02.752767069Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1356,"client":"1","":"downstairs","session_id":"91ffb351-cc0f-48b0-87ed-896077a3912a"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:02.752806747Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1356,"client":"1","":"downstairs","session_id":"91ffb351-cc0f-48b0-87ed-896077a3912a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:02.752834745Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1356,"session_id":"91ffb351-cc0f-48b0-87ed-896077a3912a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:02.752862114Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1356,"client":"0","":"downstairs","session_id":"91ffb351-cc0f-48b0-87ed-896077a3912a"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:02.752887423Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1356,"client":"0","":"downstairs","session_id":"91ffb351-cc0f-48b0-87ed-896077a3912a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:02.752910111Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1356,"session_id":"91ffb351-cc0f-48b0-87ed-896077a3912a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:02.75293251Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1356,"client":"2","":"downstairs","session_id":"91ffb351-cc0f-48b0-87ed-896077a3912a"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:02.752956349Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1356,"client":"2","":"downstairs","session_id":"91ffb351-cc0f-48b0-87ed-896077a3912a"} {"msg":"[0]R flush_numbers[0..12]: [77, 80, 76, 78, 68, 64, 73, 70, 76, 79, 75, 83]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:02.752981378Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1356,"":"downstairs","":"downstairs","session_id":"91ffb351-cc0f-48b0-87ed-896077a3912a"} {"msg":"[0]R generation[0..12]: [41, 42, 40, 42, 36, 34, 38, 37, 40, 42, 40, 43]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:02.753010647Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1356,"":"downstairs","":"downstairs","session_id":"91ffb351-cc0f-48b0-87ed-896077a3912a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:02.753037925Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1356,"":"downstairs","":"downstairs","session_id":"91ffb351-cc0f-48b0-87ed-896077a3912a"} {"msg":"[1]R flush_numbers[0..12]: [77, 80, 76, 78, 68, 64, 73, 70, 76, 79, 75, 83]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:02.753061494Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1356,"":"downstairs","":"downstairs","session_id":"91ffb351-cc0f-48b0-87ed-896077a3912a"} {"msg":"[1]R generation[0..12]: [41, 42, 40, 42, 36, 34, 38, 37, 40, 42, 40, 43]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:02.753086643Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1356,"":"downstairs","":"downstairs","session_id":"91ffb351-cc0f-48b0-87ed-896077a3912a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:02.753113781Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1356,"":"downstairs","":"downstairs","session_id":"91ffb351-cc0f-48b0-87ed-896077a3912a"} {"msg":"[2]R flush_numbers[0..12]: [77, 80, 76, 78, 68, 64, 73, 70, 76, 79, 75, 83]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:02.75313907Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1356,"":"downstairs","":"downstairs","session_id":"91ffb351-cc0f-48b0-87ed-896077a3912a"} {"msg":"[2]R generation[0..12]: [41, 42, 40, 42, 36, 34, 38, 37, 40, 42, 40, 43]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:02.753164979Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1356,"":"downstairs","":"downstairs","session_id":"91ffb351-cc0f-48b0-87ed-896077a3912a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-08T20:24:02.753194508Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1356,"":"downstairs","":"downstairs","session_id":"91ffb351-cc0f-48b0-87ed-896077a3912a"} {"msg":"Max found gen is 44","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:02.753223116Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1356,"":"downstairs","":"downstairs","session_id":"91ffb351-cc0f-48b0-87ed-896077a3912a"} {"msg":"Generation requested: 44 >= found:44","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:02.753250795Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1356,"":"downstairs","":"downstairs","session_id":"91ffb351-cc0f-48b0-87ed-896077a3912a"} {"msg":"Next flush: 84","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:02.753278944Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1356,"":"downstairs","":"downstairs","session_id":"91ffb351-cc0f-48b0-87ed-896077a3912a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:02.753307252Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1356,"":"downstairs","":"downstairs","session_id":"91ffb351-cc0f-48b0-87ed-896077a3912a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:02.753335871Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1356,"session_id":"91ffb351-cc0f-48b0-87ed-896077a3912a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:02.753361529Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1356,"client":"0","":"downstairs","session_id":"91ffb351-cc0f-48b0-87ed-896077a3912a"} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:02.753395988Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1356,"client":"1","":"downstairs","session_id":"91ffb351-cc0f-48b0-87ed-896077a3912a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:02.753417287Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1356,"client":"2","":"downstairs","session_id":"91ffb351-cc0f-48b0-87ed-896077a3912a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:02.753432136Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1356,"session_id":"91ffb351-cc0f-48b0-87ed-896077a3912a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:02.753444975Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1356,"session_id":"91ffb351-cc0f-48b0-87ed-896077a3912a"} {"msg":"f14b69fd-6e02-47e9-87e5-df2a54c1aa35 is now active with session: 91ffb351-cc0f-48b0-87ed-896077a3912a","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:02.753457805Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1356,"session_id":"91ffb351-cc0f-48b0-87ed-896077a3912a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:02.753470094Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1356,"session_id":"91ffb351-cc0f-48b0-87ed-896077a3912a"} {"msg":"The guest has finished waiting for activation with:44","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:02.753482474Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1356} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 510 len 20480 data: 5 5 4 5 5 02/30 Write block 337 len 40960 data: 7 7 6 6 6 7 6 6 5 7 03/30 Read block 22 len 12288 04/30 Read block 27 len 12288 05/30 Write block 497 len 12288 data: 4 5 4 06/30 Flush 07/30 Write block 388 len 32768 data: 3 3 3 3 4 4 4 3 08/30 Write block 170 len 24576 data: 6 5 3 3 3 3 09/30 Write block 253 len 20480 data: 7 7 5 3 5 10/30 Read block 26 len 8192 11/30 Read block 280 len 12288 12/30 Write block 211 len 24576 data: 5 4 3 3 3 4 13/30 Write block 221 len 40960 data: 7 5 5 5 6 5 7 5 5 5 14/30 Read block 218 len 24576 15/30 Read block 299 len 28672 16/30 Read block 54 len 20480 17/30 Read block 131 len 20480 18/30 Write block 211 len 12288 data: 6 5 4 19/30 Write block 232 len 32768 data: 5 4 3 4 4 6 6 6 20/30 Read block 380 len 40960 21/30 Read block 114 len 4096 22/30 Read block 354 len 28672 23/30 Write block 434 len 36864 data: 4 5 5 5 5 5 4 5 4 24/30 Read block 244 len 8192 25/30 Flush 26/30 Write block 276 len 16384 data: 8 8 5 5 27/30 Write block 198 len 12288 data: 4 3 2 28/30 Write block 345 len 36864 data: 6 8 7 5 6 5 5 5 6 29/30 Read block 567 len 32768 30/30 Write block 298 len 16384 data: 3 4 4 5 ---------------------------------------------------------------- Crucible gen:44 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 4 Done Done Done false 1026 Acked Write 3 Done Done Done false 1027 Acked Write 9 Sent Sent Done false 1028 Acked Read 8 Sent Sent Done false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 3 1 7 Done 2 2 4 8 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:24:02.780301132Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1356} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 41 41 41 77 77 77 F F F 1 020-039 42 42 42 80 80 80 F F F 2 040-059 40 40 40 76 76 76 F F F 3 060-079 42 42 42 78 78 78 F F F 4 080-099 36 36 36 68 68 68 F F F 5 100-119 34 34 34 64 64 64 F F F 6 120-139 38 38 38 73 73 73 F F F 7 140-159 37 37 37 70 70 70 F F F 8 160-179 44 44 44 85 85 85 F F F 9 180-199 42 42 42 79 79 79 T T T 10 200-219 44 44 44 85 85 85 T T T 11 220-239 44 44 44 85 85 85 F F F 12 240-259 44 44 44 85 85 85 F F F 13 260-279 41 41 41 77 77 77 T T T 14 280-299 40 40 40 75 75 75 T T T 15 300-319 38 38 38 72 72 72 T T T 16 320-339 44 44 44 84 84 84 F F F 17 340-359 44 44 44 84 84 84 T T T 18 360-379 43 43 43 83 83 83 F F F 19 380-399 44 44 44 85 85 85 F F F 20 400-419 42 42 42 82 82 82 F F F 21 420-439 44 44 44 85 85 85 F F F 22 440-459 44 44 44 85 85 85 F F F 23 460-479 43 43 43 83 83 83 F F F 24 480-499 44 44 44 84 84 84 F F F 25 500-519 44 44 44 84 84 84 F F F 26 520-539 33 33 33 63 63 63 F F F 27 540-559 38 38 38 73 73 73 F F F 28 560-579 42 42 42 80 80 80 F F F 29 580-599 42 42 42 82 82 82 F F F Max gen: 44, Max flush: 85 A Difference in extent metadata is expected here On loop 22 Verifying data now No change after verify Loop: 22 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 41 41 41 77 77 77 F F F 1 020-039 42 42 42 80 80 80 F F F 2 040-059 40 40 40 76 76 76 F F F 3 060-079 42 42 42 78 78 78 F F F 4 080-099 36 36 36 68 68 68 F F F 5 100-119 34 34 34 64 64 64 F F F 6 120-139 38 38 38 73 73 73 F F F 7 140-159 37 37 37 70 70 70 F F F 8 160-179 44 44 44 85 85 85 F F F 9 180-199 45 45 45 86 86 86 F F F 10 200-219 45 45 45 86 86 86 F F F 11 220-239 44 44 44 85 85 85 F F F 12 240-259 44 44 44 85 85 85 F F F 13 260-279 45 45 45 86 86 86 F F F 14 280-299 45 45 45 86 86 86 F F F 15 300-319 45 45 45 86 86 86 F F F 16 320-339 44 44 44 84 84 84 F F F 17 340-359 45 45 45 86 86 86 F F F 18 360-379 43 43 43 83 83 83 F F F 19 380-399 44 44 44 85 85 85 F F F 20 400-419 42 42 42 82 82 82 F F F 21 420-439 44 44 44 85 85 85 F F F 22 440-459 44 44 44 85 85 85 F F F 23 460-479 43 43 43 83 83 83 F F F 24 480-499 44 44 44 84 84 84 F F F 25 500-519 44 44 44 84 84 84 F F F 26 520-539 33 33 33 63 63 63 F F F 27 540-559 38 38 38 73 73 73 F F F 28 560-579 42 42 42 80 80 80 F F F 29 580-599 42 42 42 82 82 82 F F F Max gen: 45, Max flush: 86 Begin loop 23 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:07.113567428Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1364,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:07.158014635Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1364,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:07.15873349Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1364} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"0fd98d0d5d909611f8b408ccda2559d35fe117bb\",\n git_commit_timestamp: \"2024-11-08T20:05:44.000000000Z\",\n git_branch: \"mkeeter/drop-skipped-backpressure-guard\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:07.158771449Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1364} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:07.158786238Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1364} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:07.158800757Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1364} {"msg":"Crucible 0eae2792-f603-49ac-9df8-f179cfe5fe38 has session id: 1e089e11-47bc-49bf-9ac1-e7b0fedd956a","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:07.159228086Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1364,"session_id":"1e089e11-47bc-49bf-9ac1-e7b0fedd956a"} {"msg":"Upstairs opts: Upstairs UUID: 0eae2792-f603-49ac-9df8-f179cfe5fe38, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:07.159273984Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1364,"session_id":"1e089e11-47bc-49bf-9ac1-e7b0fedd956a"} {"msg":"Crucible stats registered with UUID: 0eae2792-f603-49ac-9df8-f179cfe5fe38","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:07.159298433Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1364,"session_id":"1e089e11-47bc-49bf-9ac1-e7b0fedd956a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:07.19045697Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1364,"task":"crutest"} {"msg":"The guest has requested activation with gen:46","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:07.190489038Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1364} {"msg":"0eae2792-f603-49ac-9df8-f179cfe5fe38 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:07.190539406Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1364,"session_id":"1e089e11-47bc-49bf-9ac1-e7b0fedd956a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:07.190559685Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1364,"client":"0","":"downstairs","session_id":"1e089e11-47bc-49bf-9ac1-e7b0fedd956a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:07.190587294Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1364,"client":"0","":"downstairs","session_id":"1e089e11-47bc-49bf-9ac1-e7b0fedd956a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:07.190605193Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1364,"client":"1","":"downstairs","session_id":"1e089e11-47bc-49bf-9ac1-e7b0fedd956a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:07.190629741Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1364,"client":"1","":"downstairs","session_id":"1e089e11-47bc-49bf-9ac1-e7b0fedd956a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:07.19066214Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1364,"client":"2","":"downstairs","session_id":"1e089e11-47bc-49bf-9ac1-e7b0fedd956a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:07.190695548Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1364,"client":"2","":"downstairs","session_id":"1e089e11-47bc-49bf-9ac1-e7b0fedd956a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:07.190729236Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1364,"":"io task","client":"2","":"downstairs","session_id":"1e089e11-47bc-49bf-9ac1-e7b0fedd956a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:07.190763265Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1364,"":"io task","client":"0","":"downstairs","session_id":"1e089e11-47bc-49bf-9ac1-e7b0fedd956a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:07.190794843Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1364,"":"io task","client":"1","":"downstairs","session_id":"1e089e11-47bc-49bf-9ac1-e7b0fedd956a"} {"msg":"ds_connection connected from Ok(127.0.0.1:60416)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:07.190822812Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1364,"":"io task","client":"2","":"downstairs","session_id":"1e089e11-47bc-49bf-9ac1-e7b0fedd956a"} {"msg":"ds_connection connected from Ok(127.0.0.1:36271)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:07.190848821Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1364,"":"io task","client":"1","":"downstairs","session_id":"1e089e11-47bc-49bf-9ac1-e7b0fedd956a"} {"msg":"ds_connection connected from Ok(127.0.0.1:37148)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:07.190875839Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1364,"":"io task","client":"0","":"downstairs","session_id":"1e089e11-47bc-49bf-9ac1-e7b0fedd956a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:07.191581365Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1364,"client":"2","":"downstairs","session_id":"1e089e11-47bc-49bf-9ac1-e7b0fedd956a"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:07.191611324Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1364,"client":"2","":"downstairs","session_id":"1e089e11-47bc-49bf-9ac1-e7b0fedd956a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:07.191626433Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1364,"client":"0","":"downstairs","session_id":"1e089e11-47bc-49bf-9ac1-e7b0fedd956a"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:07.191640652Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1364,"client":"0","":"downstairs","session_id":"1e089e11-47bc-49bf-9ac1-e7b0fedd956a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:07.1916923Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1364,"client":"1","":"downstairs","session_id":"1e089e11-47bc-49bf-9ac1-e7b0fedd956a"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:07.191722378Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1364,"client":"1","":"downstairs","session_id":"1e089e11-47bc-49bf-9ac1-e7b0fedd956a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:07.192277201Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1364,"client":"1","":"downstairs","session_id":"1e089e11-47bc-49bf-9ac1-e7b0fedd956a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:07.192324479Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1364,"client":"2","":"downstairs","session_id":"1e089e11-47bc-49bf-9ac1-e7b0fedd956a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:07.192359257Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1364,"client":"0","":"downstairs","session_id":"1e089e11-47bc-49bf-9ac1-e7b0fedd956a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:07.192871622Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1364,"client":"1","":"downstairs","session_id":"1e089e11-47bc-49bf-9ac1-e7b0fedd956a"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:07.19289891Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1364,"client":"1","":"downstairs","session_id":"1e089e11-47bc-49bf-9ac1-e7b0fedd956a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:07.19291365Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1364,"session_id":"1e089e11-47bc-49bf-9ac1-e7b0fedd956a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:07.193002636Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1364,"client":"0","":"downstairs","session_id":"1e089e11-47bc-49bf-9ac1-e7b0fedd956a"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:07.193033704Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1364,"client":"0","":"downstairs","session_id":"1e089e11-47bc-49bf-9ac1-e7b0fedd956a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:07.193050023Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1364,"session_id":"1e089e11-47bc-49bf-9ac1-e7b0fedd956a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:07.193063353Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1364,"client":"2","":"downstairs","session_id":"1e089e11-47bc-49bf-9ac1-e7b0fedd956a"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:07.193077262Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1364,"client":"2","":"downstairs","session_id":"1e089e11-47bc-49bf-9ac1-e7b0fedd956a"} {"msg":"[0]R flush_numbers[0..12]: [77, 80, 76, 78, 68, 64, 73, 70, 85, 86, 86, 85]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:07.193091551Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1364,"":"downstairs","":"downstairs","session_id":"1e089e11-47bc-49bf-9ac1-e7b0fedd956a"} {"msg":"[0]R generation[0..12]: [41, 42, 40, 42, 36, 34, 38, 37, 44, 45, 45, 44]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:07.19310596Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1364,"":"downstairs","":"downstairs","session_id":"1e089e11-47bc-49bf-9ac1-e7b0fedd956a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:07.19312037Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1364,"":"downstairs","":"downstairs","session_id":"1e089e11-47bc-49bf-9ac1-e7b0fedd956a"} {"msg":"[1]R flush_numbers[0..12]: [77, 80, 76, 78, 68, 64, 73, 70, 85, 86, 86, 85]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:07.193134339Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1364,"":"downstairs","":"downstairs","session_id":"1e089e11-47bc-49bf-9ac1-e7b0fedd956a"} {"msg":"[1]R generation[0..12]: [41, 42, 40, 42, 36, 34, 38, 37, 44, 45, 45, 44]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:07.193148448Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1364,"":"downstairs","":"downstairs","session_id":"1e089e11-47bc-49bf-9ac1-e7b0fedd956a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:07.193166757Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1364,"":"downstairs","":"downstairs","session_id":"1e089e11-47bc-49bf-9ac1-e7b0fedd956a"} {"msg":"[2]R flush_numbers[0..12]: [77, 80, 76, 78, 68, 64, 73, 70, 85, 86, 86, 85]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:07.193181097Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1364,"":"downstairs","":"downstairs","session_id":"1e089e11-47bc-49bf-9ac1-e7b0fedd956a"} {"msg":"[2]R generation[0..12]: [41, 42, 40, 42, 36, 34, 38, 37, 44, 45, 45, 44]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:07.193204076Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1364,"":"downstairs","":"downstairs","session_id":"1e089e11-47bc-49bf-9ac1-e7b0fedd956a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:07.193232414Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1364,"":"downstairs","":"downstairs","session_id":"1e089e11-47bc-49bf-9ac1-e7b0fedd956a"} {"msg":"Max found gen is 46","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:07.193257683Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1364,"":"downstairs","":"downstairs","session_id":"1e089e11-47bc-49bf-9ac1-e7b0fedd956a"} {"msg":"Generation requested: 46 >= found:46","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:07.193272532Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1364,"":"downstairs","":"downstairs","session_id":"1e089e11-47bc-49bf-9ac1-e7b0fedd956a"} {"msg":"Next flush: 87","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-08T20:24:07.193295611Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1364,"":"downstairs","":"downstairs","session_id":"1e089e11-47bc-49bf-9ac1-e7b0fedd956a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:07.19332668Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1364,"":"downstairs","":"downstairs","session_id":"1e089e11-47bc-49bf-9ac1-e7b0fedd956a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:07.193354118Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1364,"session_id":"1e089e11-47bc-49bf-9ac1-e7b0fedd956a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:07.193368847Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1364,"client":"0","":"downstairs","session_id":"1e089e11-47bc-49bf-9ac1-e7b0fedd956a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:07.193382517Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1364,"client":"1","":"downstairs","session_id":"1e089e11-47bc-49bf-9ac1-e7b0fedd956a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:07.193405506Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1364,"client":"2","":"downstairs","session_id":"1e089e11-47bc-49bf-9ac1-e7b0fedd956a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:07.193432335Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1364,"session_id":"1e089e11-47bc-49bf-9ac1-e7b0fedd956a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:07.193456803Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1364,"session_id":"1e089e11-47bc-49bf-9ac1-e7b0fedd956a"} {"msg":"0eae2792-f603-49ac-9df8-f179cfe5fe38 is now active with session: 1e089e11-47bc-49bf-9ac1-e7b0fedd956a","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:07.193471882Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1364,"session_id":"1e089e11-47bc-49bf-9ac1-e7b0fedd956a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:07.193492621Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1364,"session_id":"1e089e11-47bc-49bf-9ac1-e7b0fedd956a"} {"msg":"The guest has finished waiting for activation with:46","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:07.19351643Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1364} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 265 len 8192 data: 7 7 02/30 Flush 03/30 Write block 452 len 8192 data: 5 4 04/30 Write block 445 len 20480 data: 4 5 4 4 4 05/30 Write block 107 len 36864 data: 6 5 5 4 3 3 4 5 3 06/30 Write block 245 len 24576 data: 5 5 7 8 9 10 07/30 Read block 469 len 4096 08/30 Read block 549 len 20480 09/30 Read block 560 len 4096 10/30 Read block 133 len 12288 11/30 Write block 318 len 8192 data: 4 5 12/30 Flush 13/30 Write block 355 len 24576 data: 8 8 10 11 11 11 14/30 Flush 15/30 Write block 555 len 32768 data: 5 5 5 5 5 4 3 2 16/30 Read block 105 len 36864 17/30 Read block 281 len 16384 18/30 Write block 521 len 12288 data: 3 4 4 19/30 Write block 322 len 40960 data: 3 3 2 3 3 2 3 3 3 3 20/30 Write block 238 len 4096 data: 7 21/30 Write block 583 len 28672 data: 5 6 9 8 8 7 6 22/30 Flush 23/30 Write block 377 len 20480 data: 4 5 6 5 4 24/30 Read block 233 len 4096 25/30 Write block 488 len 12288 data: 3 3 3 26/30 Read block 41 len 20480 27/30 Read block 349 len 32768 28/30 Read block 217 len 16384 29/30 Write block 154 len 32768 data: 4 3 2 2 2 2 2 2 30/30 Read block 205 len 36864 ---------------------------------------------------------------- Crucible gen:46 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 5 Done Done Done false 1023 Acked Read 1 Done Done Done false 1024 Acked Write 3 Done Done Done false 1025 Acked Read 5 Done Done Done false 1026 Acked Read 8 Done Done Done false 1027 Acked Read 4 Done Done Done false 1028 Acked Write 8 Done Done Sent false 1029 Acked Read 9 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 2 3 Done 8 7 6 21 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1021 1021 1021 Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:24:07.220622395Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1364} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 41 41 41 77 77 77 F F F 1 020-039 42 42 42 80 80 80 F F F 2 040-059 40 40 40 76 76 76 F F F 3 060-079 42 42 42 78 78 78 F F F 4 080-099 36 36 36 68 68 68 F F F 5 100-119 46 46 46 88 88 88 F F F 6 120-139 38 38 38 73 73 73 F F F 7 140-159 37 37 37 70 70 70 T T T 8 160-179 44 44 44 85 85 85 T T T 9 180-199 45 45 45 86 86 86 F F F 10 200-219 45 45 45 86 86 86 F F F 11 220-239 46 46 46 90 90 90 F F F 12 240-259 46 46 46 88 88 88 F F F 13 260-279 46 46 46 87 87 87 F F F 14 280-299 45 45 45 86 86 86 F F F 15 300-319 46 46 46 88 88 88 F F F 16 320-339 46 46 46 90 90 90 F F F 17 340-359 46 46 46 89 89 89 F F F 18 360-379 46 46 46 89 89 89 T T T 19 380-399 44 44 44 85 85 85 T T T 20 400-419 42 42 42 82 82 82 F F F 21 420-439 44 44 44 85 85 85 F F F 22 440-459 46 46 46 88 88 88 F F F 23 460-479 43 43 43 83 83 83 F F F 24 480-499 44 44 44 84 84 84 T T T 25 500-519 44 44 44 84 84 84 F F F 26 520-539 46 46 46 90 90 90 F F F 27 540-559 46 46 46 90 90 90 F F F 28 560-579 46 46 46 90 90 90 F F F 29 580-599 46 46 46 90 90 90 F F F Max gen: 46, Max flush: 90 A Difference in extent metadata is expected here On loop 23 Verifying data now No change after verify Loop: 23 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 41 41 41 77 77 77 F F F 1 020-039 42 42 42 80 80 80 F F F 2 040-059 40 40 40 76 76 76 F F F 3 060-079 42 42 42 78 78 78 F F F 4 080-099 36 36 36 68 68 68 F F F 5 100-119 46 46 46 88 88 88 F F F 6 120-139 38 38 38 73 73 73 F F F 7 140-159 47 47 47 91 91 91 F F F 8 160-179 47 47 47 91 91 91 F F F 9 180-199 45 45 45 86 86 86 F F F 10 200-219 45 45 45 86 86 86 F F F 11 220-239 46 46 46 90 90 90 F F F 12 240-259 46 46 46 88 88 88 F F F 13 260-279 46 46 46 87 87 87 F F F 14 280-299 45 45 45 86 86 86 F F F 15 300-319 46 46 46 88 88 88 F F F 16 320-339 46 46 46 90 90 90 F F F 17 340-359 46 46 46 89 89 89 F F F 18 360-379 47 47 47 91 91 91 F F F 19 380-399 47 47 47 91 91 91 F F F 20 400-419 42 42 42 82 82 82 F F F 21 420-439 44 44 44 85 85 85 F F F 22 440-459 46 46 46 88 88 88 F F F 23 460-479 43 43 43 83 83 83 F F F 24 480-499 47 47 47 91 91 91 F F F 25 500-519 44 44 44 84 84 84 F F F 26 520-539 46 46 46 90 90 90 F F F 27 540-559 46 46 46 90 90 90 F F F 28 560-579 46 46 46 90 90 90 F F F 29 580-599 46 46 46 90 90 90 F F F Max gen: 47, Max flush: 91 Begin loop 24 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:11.535431844Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1372,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:11.57990749Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1372,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:11.580510221Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1372} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"0fd98d0d5d909611f8b408ccda2559d35fe117bb\",\n git_commit_timestamp: \"2024-11-08T20:05:44.000000000Z\",\n git_branch: \"mkeeter/drop-skipped-backpressure-guard\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:11.580547719Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1372} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:11.580566128Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1372} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:11.580580887Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1372} {"msg":"Crucible 6a50e68c-4c61-4b56-b851-660843b2b577 has session id: 33309404-9d0b-4d4c-a793-2ad31dc19b3a","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:11.581042675Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1372,"session_id":"33309404-9d0b-4d4c-a793-2ad31dc19b3a"} {"msg":"Upstairs opts: Upstairs UUID: 6a50e68c-4c61-4b56-b851-660843b2b577, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:11.581092362Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1372,"session_id":"33309404-9d0b-4d4c-a793-2ad31dc19b3a"} {"msg":"Crucible stats registered with UUID: 6a50e68c-4c61-4b56-b851-660843b2b577","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:11.581117891Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1372,"session_id":"33309404-9d0b-4d4c-a793-2ad31dc19b3a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:11.613538476Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1372,"task":"crutest"} {"msg":"The guest has requested activation with gen:48","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:11.613570795Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1372} {"msg":"6a50e68c-4c61-4b56-b851-660843b2b577 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:11.613620152Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1372,"session_id":"33309404-9d0b-4d4c-a793-2ad31dc19b3a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:11.613640751Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1372,"client":"0","":"downstairs","session_id":"33309404-9d0b-4d4c-a793-2ad31dc19b3a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:11.61366815Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1372,"client":"0","":"downstairs","session_id":"33309404-9d0b-4d4c-a793-2ad31dc19b3a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:11.613685989Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1372,"client":"1","":"downstairs","session_id":"33309404-9d0b-4d4c-a793-2ad31dc19b3a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:11.613779644Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1372,"client":"1","":"downstairs","session_id":"33309404-9d0b-4d4c-a793-2ad31dc19b3a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:11.613826112Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1372,"client":"2","":"downstairs","session_id":"33309404-9d0b-4d4c-a793-2ad31dc19b3a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:11.613861171Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1372,"client":"2","":"downstairs","session_id":"33309404-9d0b-4d4c-a793-2ad31dc19b3a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:11.613889639Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1372,"":"io task","client":"2","":"downstairs","session_id":"33309404-9d0b-4d4c-a793-2ad31dc19b3a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:11.613924528Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1372,"":"io task","client":"0","":"downstairs","session_id":"33309404-9d0b-4d4c-a793-2ad31dc19b3a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:11.614024572Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1372,"":"io task","client":"1","":"downstairs","session_id":"33309404-9d0b-4d4c-a793-2ad31dc19b3a"} {"msg":"ds_connection connected from Ok(127.0.0.1:52076)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:11.61407295Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1372,"":"io task","client":"0","":"downstairs","session_id":"33309404-9d0b-4d4c-a793-2ad31dc19b3a"} {"msg":"ds_connection connected from Ok(127.0.0.1:33257)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:11.614109878Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1372,"":"io task","client":"2","":"downstairs","session_id":"33309404-9d0b-4d4c-a793-2ad31dc19b3a"} {"msg":"ds_connection connected from Ok(127.0.0.1:47424)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:11.614146447Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1372,"":"io task","client":"1","":"downstairs","session_id":"33309404-9d0b-4d4c-a793-2ad31dc19b3a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:11.614636923Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1372,"client":"2","":"downstairs","session_id":"33309404-9d0b-4d4c-a793-2ad31dc19b3a"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:11.614664941Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1372,"client":"2","":"downstairs","session_id":"33309404-9d0b-4d4c-a793-2ad31dc19b3a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:11.614827113Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1372,"client":"1","":"downstairs","session_id":"33309404-9d0b-4d4c-a793-2ad31dc19b3a"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:11.61490135Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1372,"client":"1","":"downstairs","session_id":"33309404-9d0b-4d4c-a793-2ad31dc19b3a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:11.614925608Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1372,"client":"0","":"downstairs","session_id":"33309404-9d0b-4d4c-a793-2ad31dc19b3a"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:11.614940798Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1372,"client":"0","":"downstairs","session_id":"33309404-9d0b-4d4c-a793-2ad31dc19b3a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:11.615323739Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1372,"client":"2","":"downstairs","session_id":"33309404-9d0b-4d4c-a793-2ad31dc19b3a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:11.615402545Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1372,"client":"0","":"downstairs","session_id":"33309404-9d0b-4d4c-a793-2ad31dc19b3a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:11.615439863Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1372,"client":"1","":"downstairs","session_id":"33309404-9d0b-4d4c-a793-2ad31dc19b3a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:11.615953058Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1372,"client":"0","":"downstairs","session_id":"33309404-9d0b-4d4c-a793-2ad31dc19b3a"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:11.615997766Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1372,"client":"0","":"downstairs","session_id":"33309404-9d0b-4d4c-a793-2ad31dc19b3a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:11.616024955Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1372,"session_id":"33309404-9d0b-4d4c-a793-2ad31dc19b3a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:11.616046154Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1372,"client":"2","":"downstairs","session_id":"33309404-9d0b-4d4c-a793-2ad31dc19b3a"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:11.616069543Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1372,"client":"2","":"downstairs","session_id":"33309404-9d0b-4d4c-a793-2ad31dc19b3a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:11.616092951Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1372,"session_id":"33309404-9d0b-4d4c-a793-2ad31dc19b3a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:11.61611539Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1372,"client":"1","":"downstairs","session_id":"33309404-9d0b-4d4c-a793-2ad31dc19b3a"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:11.616138689Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1372,"client":"1","":"downstairs","session_id":"33309404-9d0b-4d4c-a793-2ad31dc19b3a"} {"msg":"[0]R flush_numbers[0..12]: [77, 80, 76, 78, 68, 88, 73, 91, 91, 86, 86, 90]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:11.616161418Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1372,"":"downstairs","":"downstairs","session_id":"33309404-9d0b-4d4c-a793-2ad31dc19b3a"} {"msg":"[0]R generation[0..12]: [41, 42, 40, 42, 36, 46, 38, 47, 47, 45, 45, 46]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:11.616185467Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1372,"":"downstairs","":"downstairs","session_id":"33309404-9d0b-4d4c-a793-2ad31dc19b3a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:11.616212586Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1372,"":"downstairs","":"downstairs","session_id":"33309404-9d0b-4d4c-a793-2ad31dc19b3a"} {"msg":"[1]R flush_numbers[0..12]: [77, 80, 76, 78, 68, 88, 73, 91, 91, 86, 86, 90]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:11.616239704Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1372,"":"downstairs","":"downstairs","session_id":"33309404-9d0b-4d4c-a793-2ad31dc19b3a"} {"msg":"[1]R generation[0..12]: [41, 42, 40, 42, 36, 46, 38, 47, 47, 45, 45, 46]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:11.616265313Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1372,"":"downstairs","":"downstairs","session_id":"33309404-9d0b-4d4c-a793-2ad31dc19b3a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:11.616294272Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1372,"":"downstairs","":"downstairs","session_id":"33309404-9d0b-4d4c-a793-2ad31dc19b3a"} {"msg":"[2]R flush_numbers[0..12]: [77, 80, 76, 78, 68, 88, 73, 91, 91, 86, 86, 90]","v":0,"name":"crucibleWait for a query_work_queue command to finish before sending IO ","level":30,"time":"2024-11-08T20:24:11.61633252Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1372,"":"downstairs","":"downstairs","session_id":"33309404-9d0b-4d4c-a793-2ad31dc19b3a"} {"msg":"[2]R generation[0..12]: [41, 42, 40, 42, 36, 46, 38, 47, 47, 45, 45, 46]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:11.616363028Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1372,"":"downstairs","":"downstairs","session_id":"33309404-9d0b-4d4c-a793-2ad31dc19b3a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:11.616397087Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1372,"":"downstairs","":"downstairs","session_id":"33309404-9d0b-4d4c-a793-2ad31dc19b3a"} {"msg":"Max found gen is 48","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:11.616426975Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1372,"":"downstairs","":"downstairs","session_id":"33309404-9d0b-4d4c-a793-2ad31dc19b3a"} {"msg":"Generation requested: 48 >= found:48","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:11.616455444Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1372,"":"downstairs","":"downstairs","session_id":"33309404-9d0b-4d4c-a793-2ad31dc19b3a"} {"msg":"Next flush: 92","v":0,"name":"crucible","level":30Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2024-11-08T20:24:11.616483052Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1372,"":"downstairs","":"downstairs","session_id":"33309404-9d0b-4d4c-a793-2ad31dc19b3a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:11.616513181Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1372,"":"downstairs","":"downstairs","session_id":"33309404-9d0b-4d4c-a793-2ad31dc19b3a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:11.61654064Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1372,"session_id":"33309404-9d0b-4d4c-a793-2ad31dc19b3a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:11.616563608Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1372,"client":"0","":"downstairs","session_id":"33309404-9d0b-4d4c-a793-2ad31dc19b3a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:11.616578558Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1372,"client":"1","":"downstairs","session_id":"33309404-9d0b-4d4c-a793-2ad31dc19b3a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:11.616592277Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1372,"client":"2","":"downstairs","session_id":"33309404-9d0b-4d4c-a793-2ad31dc19b3a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:11.616605497Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1372,"session_id":"33309404-9d0b-4d4c-a793-2ad31dc19b3a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:11.616617986Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1372,"session_id":"33309404-9d0b-4d4c-a793-2ad31dc19b3a"} {"msg":"6a50e68c-4c61-4b56-b851-660843b2b577 is now active with session: 33309404-9d0b-4d4c-a793-2ad31dc19b3a","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:11.616630875Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1372,"session_id":"33309404-9d0b-4d4c-a793-2ad31dc19b3a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:11.616642964Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1372,"session_id":"33309404-9d0b-4d4c-a793-2ad31dc19b3a"} {"msg":"The guest has finished waiting for activation with:48","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:11.616655254Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1372} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 318 len 36864 data: 5 6 4 3 4 4 3 4 4 02/30 Write block 481 len 36864 data: 3 3 3 3 2 3 3 4 4 03/30 Write block 467 len 12288 data: 4 6 6 04/30 Write block 323 len 20480 data: 5 4 5 5 3 05/30 Write block 186 len 20480 data: 10 9 7 6 7 06/30 Write block 518 len 8192 data: 3 4 07/30 Read block 425 len 20480 08/30 Write block 580 len 40960 data: 3 3 3 6 7 10 9 9 8 7 09/30 Write block 280 len 16384 data: 5 4 5 4 10/30 Write block 6 len 32768 data: 4 4 4 4 4 4 5 4 11/30 Read block 288 len 16384 12/30 Write block 128 len 36864 data: 2 2 3 3 4 6 6 6 5 13/30 Read block 387 len 8192 14/30 Flush 15/30 Write block 575 len 24576 data: 4 3 3 4 3 4 16/30 Read block 111 len 32768 17/30 Read block 588 len 8192 18/30 Flush 19/30 Write block 541 len 16384 data: 5 5 6 6 20/30 Read block 430 len 24576 21/30 Write block 294 len 8192 data: 5 3 22/30 Read block 110 len 32768 23/30 Read block 260 len 12288 24/30 Write block 130 len 4096 data: 4 25/30 Flush 26/30 Flush 27/30 Write block 278 len 24576 data: 6 6 6 5 6 5 28/30 Write block 435 len 16384 data: 6 6 6 6 29/30 Read block 586 len 16384 30/30 Read block 381 len 20480 ---------------------------------------------------------------- Crucible gen:48 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 6 Done Done Done false 1027 Acked Write 4 Done Done Done false 1028 Acked Read 4 Done Done Done false 1029 Acked Read 5 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 4 4 3 11 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:24:11.64333986Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1372} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 48 48 48 92 92 92 F F F 1 020-039 42 42 42 80 80 80 F F F 2 040-059 40 40 40 76 76 76 F F F 3 060-079 42 42 42 78 78 78 F F F 4 080-099 36 36 36 68 68 68 F F F 5 100-119 46 46 46 88 88 88 F F F 6 120-139 48 48 48 94 94 94 F F F 7 140-159 47 47 47 91 91 91 F F F 8 160-179 47 47 47 91 91 91 F F F 9 180-199 48 48 48 92 92 92 F F F 10 200-219 45 45 45 86 86 86 F F F 11 220-239 46 46 46 90 90 90 F F F 12 240-259 46 46 46 88 88 88 F F F 13 260-279 46 46 46 87 87 87 T T T 14 280-299 48 48 48 94 94 94 T T T 15 300-319 48 48 48 92 92 92 F F F 16 320-339 48 48 48 92 92 92 F F F 17 340-359 46 46 46 89 89 89 F F F 18 360-379 47 47 47 91 91 91 F F F 19 380-399 47 47 47 91 91 91 F F F 20 400-419 42 42 42 82 82 82 F F F 21 420-439 44 44 44 85 85 85 T T T 22 440-459 46 46 46 88 88 88 F F F 23 460-479 48 48 48 92 92 92 F F F 24 480-499 48 48 48 92 92 92 F F F 25 500-519 48 48 48 92 92 92 F F F 26 520-539 46 46 46 90 90 90 F F F 27 540-559 48 48 48 94 94 94 F F F 28 560-579 48 48 48 93 93 93 F F F 29 580-599 48 48 48 93 93 93 F F F Max gen: 48, Max flush: 94 A Difference in extent metadata is expected here On loop 24 Verifying data now No change after verify Loop: 24 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 48 48 48 92 92 92 F F F 1 020-039 42 42 42 80 80 80 F F F 2 040-059 40 40 40 76 76 76 F F F 3 060-079 42 42 42 78 78 78 F F F 4 080-099 36 36 36 68 68 68 F F F 5 100-119 46 46 46 88 88 88 F F F 6 120-139 48 48 48 94 94 94 F F F 7 140-159 47 47 47 91 91 91 F F F 8 160-179 47 47 47 91 91 91 F F F 9 180-199 48 48 48 92 92 92 F F F 10 200-219 45 45 45 86 86 86 F F F 11 220-239 46 46 46 90 90 90 F F F 12 240-259 46 46 46 88 88 88 F F F 13 260-279 49 49 49 95 95 95 F F F 14 280-299 49 49 49 95 95 95 F F F 15 300-319 48 48 48 92 92 92 F F F 16 320-339 48 48 48 92 92 92 F F F 17 340-359 46 46 46 89 89 89 F F F 18 360-379 47 47 47 91 91 91 F F F 19 380-399 47 47 47 91 91 91 F F F 20 400-419 42 42 42 82 82 82 F F F 21 420-439 49 49 49 95 95 95 F F F 22 440-459 46 46 46 88 88 88 F F F 23 460-479 48 48 48 92 92 92 F F F 24 480-499 48 48 48 92 92 92 F F F 25 500-519 48 48 48 92 92 92 F F F 26 520-539 46 46 46 90 90 90 F F F 27 540-559 48 48 48 94 94 94 F F F 28 560-579 48 48 48 93 93 93 F F F 29 580-599 48 48 48 93 93 93 F F F Max gen: 49, Max flush: 95 Begin loop 25 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:15.932283981Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1380,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:15.976830804Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1380,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:15.977441214Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1380} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"0fd98d0d5d909611f8b408ccda2559d35fe117bb\",\n git_commit_timestamp: \"2024-11-08T20:05:44.000000000Z\",\n git_branch: \"mkeeter/drop-skipped-backpressure-guard\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:15.977479102Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1380} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:15.977494021Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1380} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:15.977508341Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1380} {"msg":"Crucible 8a98e85b-933c-4e1d-a69b-9cc9940b2727 has session id: e936178c-a8eb-4e3c-9813-f900e5f95e08","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:15.978061494Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1380,"session_id":"e936178c-a8eb-4e3c-9813-f900e5f95e08"} {"msg":"Upstairs opts: Upstairs UUID: 8a98e85b-933c-4e1d-a69b-9cc9940b2727, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:15.978106911Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1380,"session_id":"e936178c-a8eb-4e3c-9813-f900e5f95e08"} {"msg":"Crucible stats registered with UUID: 8a98e85b-933c-4e1d-a69b-9cc9940b2727","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:15.97813532Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1380,"session_id":"e936178c-a8eb-4e3c-9813-f900e5f95e08"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:16.009214821Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1380,"task":"crutest"} {"msg":"The guest has requested activation with gen:50","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:16.009248739Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1380} {"msg":"8a98e85b-933c-4e1d-a69b-9cc9940b2727 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:16.009284577Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1380,"session_id":"e936178c-a8eb-4e3c-9813-f900e5f95e08"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:16.009301397Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1380,"client":"0","":"downstairs","session_id":"e936178c-a8eb-4e3c-9813-f900e5f95e08"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:16.009329975Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1380,"client":"0","":"downstairs","session_id":"e936178c-a8eb-4e3c-9813-f900e5f95e08"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:16.009347914Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1380,"client":"1","":"downstairs","session_id":"e936178c-a8eb-4e3c-9813-f900e5f95e08"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:16.009364973Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1380,"client":"1","":"downstairs","session_id":"e936178c-a8eb-4e3c-9813-f900e5f95e08"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:16.009383503Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1380,"client":"2","":"downstairs","session_id":"e936178c-a8eb-4e3c-9813-f900e5f95e08"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:16.009414961Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1380,"client":"2","":"downstairs","session_id":"e936178c-a8eb-4e3c-9813-f900e5f95e08"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:16.00944686Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1380,"":"io task","client":"2","":"downstairs","session_id":"e936178c-a8eb-4e3c-9813-f900e5f95e08"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:16.009471558Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1380,"":"io task","client":"0","":"downstairs","session_id":"e936178c-a8eb-4e3c-9813-f900e5f95e08"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:16.009494687Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1380,"":"io task","client":"1","":"downstairs","session_id":"e936178c-a8eb-4e3c-9813-f900e5f95e08"} {"msg":"ds_connection connected from Ok(127.0.0.1:59241)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:16.009699607Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1380,"":"io task","client":"1","":"downstairs","session_id":"e936178c-a8eb-4e3c-9813-f900e5f95e08"} {"msg":"ds_connection connected from Ok(127.0.0.1:52440)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:16.009748985Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1380,"":"io task","client":"2","":"downstairs","session_id":"e936178c-a8eb-4e3c-9813-f900e5f95e08"} {"msg":"ds_connection connected from Ok(127.0.0.1:34614)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:16.009782283Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1380,"":"io task","client":"0","":"downstairs","session_id":"e936178c-a8eb-4e3c-9813-f900e5f95e08"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:16.010431481Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1380,"client":"2","":"downstairs","session_id":"e936178c-a8eb-4e3c-9813-f900e5f95e08"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:16.01045991Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1380,"client":"2","":"downstairs","session_id":"e936178c-a8eb-4e3c-9813-f900e5f95e08"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:16.010475489Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1380,"client":"0","":"downstairs","session_id":"e936178c-a8eb-4e3c-9813-f900e5f95e08"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:16.010489629Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1380,"client":"0","":"downstairs","session_id":"e936178c-a8eb-4e3c-9813-f900e5f95e08"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:16.010607543Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1380,"client":"1","":"downstairs","session_id":"e936178c-a8eb-4e3c-9813-f900e5f95e08"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:16.010632082Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1380,"client":"1","":"downstairs","session_id":"e936178c-a8eb-4e3c-9813-f900e5f95e08"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:16.01107385Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1380,"client":"2","":"downstairs","session_id":"e936178c-a8eb-4e3c-9813-f900e5f95e08"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:16.011254571Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1380,"client":"1","":"downstairs","session_id":"e936178c-a8eb-4e3c-9813-f900e5f95e08"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:16.011292459Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1380,"client":"0","":"downstairs","session_id":"e936178c-a8eb-4e3c-9813-f900e5f95e08"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:16.011559776Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1380,"client":"2","":"downstairs","session_id":"e936178c-a8eb-4e3c-9813-f900e5f95e08"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:16.011587155Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1380,"client":"2","":"downstairs","session_id":"e936178c-a8eb-4e3c-9813-f900e5f95e08"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:16.011601924Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1380,"session_id":"e936178c-a8eb-4e3c-9813-f900e5f95e08"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:16.01189977Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1380,"client":"1","":"downstairs","session_id":"e936178c-a8eb-4e3c-9813-f900e5f95e08"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:16.011928918Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1380,"client":"1","":"downstairs","session_id":"e936178c-a8eb-4e3c-9813-f900e5f95e08"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:16.011943987Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1380,"session_id":"e936178c-a8eb-4e3c-9813-f900e5f95e08"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:16.011957117Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1380,"client":"0","":"downstairs","session_id":"e936178c-a8eb-4e3c-9813-f900e5f95e08"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:16.011973856Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1380,"client":"0","":"downstairs","session_id":"e936178c-a8eb-4e3c-9813-f900e5f95e08"} {"msg":"[0]R flush_numbers[0..12]: [92, 80, 76, 78, 68, 88, 94, 91, 91, 92, 86, 90]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:16.011988095Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1380,"":"downstairs","":"downstairs","session_id":"e936178c-a8eb-4e3c-9813-f900e5f95e08"} {"msg":"[0]R generation[0..12]: [48, 42, 40, 42, 36, 46, 48, 47, 47, 48, 45, 46]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:16.012018024Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1380,"":"downstairs","":"downstairs","session_id":"e936178c-a8eb-4e3c-9813-f900e5f95e08"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:16.012033933Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1380,"":"downstairs","":"downstairs","session_id":"e936178c-a8eb-4e3c-9813-f900e5f95e08"} {"msg":"[1]R flush_numbers[0..12]: [92, 80, 76, 78, 68, 88, 94, 91, 91, 92, 86, 90]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:16.012048182Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1380,"":"downstairs","":"downstairs","session_id":"e936178c-a8eb-4e3c-9813-f900e5f95e08"} {"msg":"[1]R generation[0..12]: [48, 42, 40, 42, 36, 46, 48, 47, 47, 48, 45, 46]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:16.012062002Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1380,"":"downstairs","":"downstairs","session_id":"e936178c-a8eb-4e3c-9813-f900e5f95e08"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:16.012076331Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1380,"":"downstairs","":"downstairs","session_id":"e936178c-a8eb-4e3c-9813-f900e5f95e08"} {"msg":"[2]R flush_numbers[0..12]: [92, 80, 76, 78, 68, 88, 94, 91, 91, 92, 86, 90]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:16.01209Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1380,"":"downstairs","":"downstairs","session_id":"e936178c-a8eb-4e3c-9813-f900e5f95e08"} {"msg":"[2]R generation[0..12]: [48, 42, 40, 42, 36, 46, 48, 47, 47, 48, 45, 46]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:16.01210374Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1380,"":"downstairs","":"downstairs","session_id":"e936178c-a8eb-4e3c-9813-f900e5f95e08"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:16.012117839Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1380,"":"downstairs","":"downstairs","session_id":"e936178c-a8eb-4e3c-9813-f900e5f95e08"} {"msg":"Max found gen is 50","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-08T20:24:16.012130918Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1380,"":"downstairs","":"downstairs","session_id":"e936178c-a8eb-4e3c-9813-f900e5f95e08"} {"msg":"Generation requested: 50 >= found:50","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:16.012152817Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1380,"":"downstairs","":"downstairs","session_id":"e936178c-a8eb-4e3c-9813-f900e5f95e08"} {"msg":"Next flush: 96","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:16.012166047Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1380,"":"downstairs","":"downstairs","session_id":"e936178c-a8eb-4e3c-9813-f900e5f95e08"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:16.012179206Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1380,"":"downstairs","":"downstairs","session_id":"e936178c-a8eb-4e3c-9813-f900e5f95e08"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:16.012194685Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1380,"session_id":"e936178c-a8eb-4e3c-9813-f900e5f95e08"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:16.012207035Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1380,"client":"0","":"downstairs","session_id":"e936178c-a8eb-4e3c-9813-f900e5f95e08"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:16.012220324Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1380,"client":"1","":"downstairs","session_id":"e936178c-a8eb-4e3c-9813-f900e5f95e08"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:16.012233503Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1380,"client":"2","":"downstairs","session_id":"e936178c-a8eb-4e3c-9813-f900e5f95e08"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:16.012246843Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1380,"session_id":"e936178c-a8eb-4e3c-9813-f900e5f95e08"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:16.012259322Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1380,"session_id":"e936178c-a8eb-4e3c-9813-f900e5f95e08"} {"msg":"8a98e85b-933c-4e1d-a69b-9cc9940b2727 is now active with session: e936178c-a8eb-4e3c-9813-f900e5f95e08","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:16.012272202Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1380,"session_id":"e936178c-a8eb-4e3c-9813-f900e5f95e08"} {"msg":"Set Active after no reconciliation","v":0,Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 "name":"crucible","level":30,"time":"2024-11-08T20:24:16.012288241Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1380,"session_id":"e936178c-a8eb-4e3c-9813-f900e5f95e08"} {"msg":"The guest has finished waiting for activation with:50","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:16.01230066Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1380} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 60 len 16384 data: 4 4 4 4 02/30 Write block 159 len 4096 data: 3 03/30 Write block 137 len 20480 data: 7 7 6 5 6 04/30 Read block 534 len 12288 05/30 Write block 175 len 8192 data: 4 2 06/30 Read block 141 len 20480 07/30 Read block 1 len 24576 08/30 Read block 53 len 36864 09/30 Read block 426 len 8192 10/30 Read block 408 len 24576 11/30 Flush 12/30 Flush 13/30 Write block 335 len 36864 data: 7 7 8 8 7 7 7 8 7 14/30 Flush 15/30 Write block 390 len 40960 data: 4 4 5 5 5 4 3 4 4 4 16/30 Flush 17/30 Write block 28 len 8192 data: 5 4 18/30 Read block 146 len 8192 19/30 Write block 358 len 32768 data: 12 12 12 11 9 8 8 6 20/30 Write block 205 len 8192 data: 6 6 21/30 Read block 76 len 16384 22/30 Read block 468 len 40960 23/30 Write block 592 len 24576 data: 5 5 3 3 2 2 24/30 Write block 591 len 28672 data: 5 6 6 4 4 3 3 25/30 Read block 571 len 4096 26/30 Write block 12 len 28672 data: 6 5 2 2 3 3 3 27/30 Write block 352 len 20480 data: 6 7 6 9 9 28/30 Write block 441 len 16384 data: 6 5 3 3 29/30 Read block 322 len 12288 30/30 Read block 100 len 20480 ---------------------------------------------------------------- Crucible gen:50 GIO:true work queues: Upstairs:0 downstairs:14 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1016 Acked Write 2 Done Done Done false 1017 Acked Read 2 Done Done Done false 1018 Acked Write 8 Done Done Done false 1019 Acked Write 2 Done Done Done false 1020 Acked Read 4 Done Done Done false 1021 Acked Read 10 Done Done Done false 1022 Acked Write 6 Done Done Done false 1023 Acked Write 7 Done Done Done false 1024 Acked Read 1 Done Done Done false 1025 Acked Write 7 Done Done Done false 1026 Acked Write 5 Sent Done Done false 1027 Acked Write 4 Sent Done Done false 1028 Acked Read 3 Sent Done Done false 1029 Acked Read 5 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 4 1 0 5 Done 10 13 14 37 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1015 1015 1015 Downstairs last five completed: 1015 1014 1013 1012 1011 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:24:16.038439023Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1380} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 48 48 48 92 92 92 T T T 1 020-039 42 42 42 80 80 80 T T T 2 040-059 40 40 40 76 76 76 F F F 3 060-079 50 50 50 96 96 96 F F F 4 080-099 36 36 36 68 68 68 F F F 5 100-119 46 46 46 88 88 88 F F F 6 120-139 50 50 50 96 96 96 F F F 7 140-159 50 50 50 96 96 96 F F F 8 160-179 50 50 50 96 96 96 F F F 9 180-199 48 48 48 92 92 92 F F F 10 200-219 45 45 45 86 86 86 T T T 11 220-239 46 46 46 90 90 90 F F F 12 240-259 46 46 46 88 88 88 F F F 13 260-279 49 49 49 95 95 95 F F F 14 280-299 49 49 49 95 95 95 F F F 15 300-319 48 48 48 92 92 92 F F F 16 320-339 50 50 50 98 98 98 F F F 17 340-359 50 50 50 98 98 98 T T T 18 360-379 47 47 47 91 91 91 T T T 19 380-399 50 50 50 99 99 99 F F F 20 400-419 42 42 42 82 82 82 F F F 21 420-439 49 49 49 95 95 95 F F F 22 440-459 46 46 46 88 88 88 T T T 23 460-479 48 48 48 92 92 92 F F F 24 480-499 48 48 48 92 92 92 F F F 25 500-519 48 48 48 92 92 92 F F F 26 520-539 46 46 46 90 90 90 F F F 27 540-559 48 48 48 94 94 94 F F F 28 560-579 48 48 48 93 93 93 F F F 29 580-599 48 48 48 93 93 93 T T T Max gen: 50, Max flush: 99 A Difference in extent metadata is expected here On loop 25 Verifying data now No change after verify Loop: 25 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 51 51 51 100 100 100 F F F 1 020-039 51 51 51 100 100 100 F F F 2 040-059 40 40 40 76 76 76 F F F 3 060-079 50 50 50 96 96 96 F F F 4 080-099 36 36 36 68 68 68 F F F 5 100-119 46 46 46 88 88 88 F F F 6 120-139 50 50 50 96 96 96 F F F 7 140-159 50 50 50 96 96 96 F F F 8 160-179 50 50 50 96 96 96 F F F 9 180-199 48 48 48 92 92 92 F F F 10 200-219 51 51 51 100 100 100 F F F 11 220-239 46 46 46 90 90 90 F F F 12 240-259 46 46 46 88 88 88 F F F 13 260-279 49 49 49 95 95 95 F F F 14 280-299 49 49 49 95 95 95 F F F 15 300-319 48 48 48 92 92 92 F F F 16 320-339 50 50 50 98 98 98 F F F 17 340-359 51 51 51 100 100 100 F F F 18 360-379 51 51 51 100 100 100 F F F 19 380-399 50 50 50 99 99 99 F F F 20 400-419 42 42 42 82 82 82 F F F 21 420-439 49 49 49 95 95 95 F F F 22 440-459 51 51 51 100 100 100 F F F 23 460-479 48 48 48 92 92 92 F F F 24 480-499 48 48 48 92 92 92 F F F 25 500-519 48 48 48 92 92 92 F F F 26 520-539 46 46 46 90 90 90 F F F 27 540-559 48 48 48 94 94 94 F F F 28 560-579 48 48 48 93 93 93 F F F 29 580-599 51 51 51 100 100 100 F F F Max gen: 51, Max flush: 100 Begin loop 26 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:20.389125537Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1388,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:24:20.43202904Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1388,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:20.432409341Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1388,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:20.464463375Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1388,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:20.465124513Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1388} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"0fd98d0d5d909611f8b408ccda2559d35fe117bb\",\n git_commit_timestamp: \"2024-11-08T20:05:44.000000000Z\",\n git_branch: \"mkeeter/drop-skipped-backpressure-guard\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:20.465161391Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1388} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:20.46517551Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1388} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:20.46518975Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1388} {"msg":"Crucible eac9bfd9-cce5-41ec-8001-191ec8cb5356 has session id: f5ca9a5d-95c4-4c2a-a059-35cb5f5ab326","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:20.465687825Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1388,"session_id":"f5ca9a5d-95c4-4c2a-a059-35cb5f5ab326"} {"msg":"Upstairs opts: Upstairs UUID: eac9bfd9-cce5-41ec-8001-191ec8cb5356, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:20.465734423Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1388,"session_id":"f5ca9a5d-95c4-4c2a-a059-35cb5f5ab326"} {"msg":"Crucible stats registered with UUID: eac9bfd9-cce5-41ec-8001-191ec8cb5356","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:20.465757292Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1388,"session_id":"f5ca9a5d-95c4-4c2a-a059-35cb5f5ab326"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:20.496869611Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1388,"task":"crutest"} {"msg":"The guest has requested activation with gen:52","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:20.49690125Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1388} {"msg":"eac9bfd9-cce5-41ec-8001-191ec8cb5356 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:20.496935778Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1388,"session_id":"f5ca9a5d-95c4-4c2a-a059-35cb5f5ab326"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:20.496952497Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1388,"client":"0","":"downstairs","session_id":"f5ca9a5d-95c4-4c2a-a059-35cb5f5ab326"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:20.496980586Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1388,"client":"0","":"downstairs","session_id":"f5ca9a5d-95c4-4c2a-a059-35cb5f5ab326"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:20.497016284Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1388,"client":"1","":"downstairs","session_id":"f5ca9a5d-95c4-4c2a-a059-35cb5f5ab326"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:20.497035403Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1388,"client":"1","":"downstairs","session_id":"f5ca9a5d-95c4-4c2a-a059-35cb5f5ab326"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:20.497062192Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1388,"client":"2","":"downstairs","session_id":"f5ca9a5d-95c4-4c2a-a059-35cb5f5ab326"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:20.49709479Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1388,"client":"2","":"downstairs","session_id":"f5ca9a5d-95c4-4c2a-a059-35cb5f5ab326"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:20.497124119Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1388,"":"io task","client":"2","":"downstairs","session_id":"f5ca9a5d-95c4-4c2a-a059-35cb5f5ab326"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:20.497151917Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1388,"":"io task","client":"0","":"downstairs","session_id":"f5ca9a5d-95c4-4c2a-a059-35cb5f5ab326"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:20.497186446Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1388,"":"io task","client":"1","":"downstairs","session_id":"f5ca9a5d-95c4-4c2a-a059-35cb5f5ab326"} {"msg":"ds_connection connected from Ok(127.0.0.1:55604)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:20.497230244Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1388,"":"io task","client":"0","":"downstairs","session_id":"f5ca9a5d-95c4-4c2a-a059-35cb5f5ab326"} {"msg":"ds_connection connected from Ok(127.0.0.1:56740)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:20.497265032Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1388,"":"io task","client":"1","":"downstairs","session_id":"f5ca9a5d-95c4-4c2a-a059-35cb5f5ab326"} {"msg":"ds_connection connected from Ok(127.0.0.1:55748)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:20.497446513Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1388,"":"io task","client":"2","":"downstairs","session_id":"f5ca9a5d-95c4-4c2a-a059-35cb5f5ab326"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:20.498026685Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1388,"client":"1","":"downstairs","session_id":"f5ca9a5d-95c4-4c2a-a059-35cb5f5ab326"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:20.498072612Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1388,"client":"1","":"downstairs","session_id":"f5ca9a5d-95c4-4c2a-a059-35cb5f5ab326"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:20.498104141Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1388,"client":"0","":"downstairs","session_id":"f5ca9a5d-95c4-4c2a-a059-35cb5f5ab326"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:20.498134009Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1388,"client":"0","":"downstairs","session_id":"f5ca9a5d-95c4-4c2a-a059-35cb5f5ab326"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:20.498163628Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1388,"client":"2","":"downstairs","session_id":"f5ca9a5d-95c4-4c2a-a059-35cb5f5ab326"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:20.498263433Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1388,"client":"2","":"downstairs","session_id":"f5ca9a5d-95c4-4c2a-a059-35cb5f5ab326"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:20.49873931Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1388,"client":"1","":"downstairs","session_id":"f5ca9a5d-95c4-4c2a-a059-35cb5f5ab326"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:20.498815716Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1388,"client":"0","":"downstairs","session_id":"f5ca9a5d-95c4-4c2a-a059-35cb5f5ab326"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:20.498862644Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1388,"client":"2","":"downstairs","session_id":"f5ca9a5d-95c4-4c2a-a059-35cb5f5ab326"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:20.499287833Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1388,"client":"1","":"downstairs","session_id":"f5ca9a5d-95c4-4c2a-a059-35cb5f5ab326"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:20.499323291Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1388,"client":"1","":"downstairs","session_id":"f5ca9a5d-95c4-4c2a-a059-35cb5f5ab326"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:20.49934394Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1388,"session_id":"f5ca9a5d-95c4-4c2a-a059-35cb5f5ab326"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:20.499416287Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1388,"client":"0","":"downstairs","session_id":"f5ca9a5d-95c4-4c2a-a059-35cb5f5ab326"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:20.499440836Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1388,"client":"0","":"downstairs","session_id":"f5ca9a5d-95c4-4c2a-a059-35cb5f5ab326"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:20.499457725Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1388,"session_id":"f5ca9a5d-95c4-4c2a-a059-35cb5f5ab326"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:20.499473194Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1388,"client":"2","":"downstairs","session_id":"f5ca9a5d-95c4-4c2a-a059-35cb5f5ab326"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:20.499493673Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1388,"client":"2","":"downstairs","session_id":"f5ca9a5d-95c4-4c2a-a059-35cb5f5ab326"} {"msg":"[0]R flush_numbers[0..12]: [100, 100, 76, 96, 68, 88, 96, 96, 96, 92, 100, 90]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:20.499511182Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1388,"":"downstairs","":"downstairs","session_id":"f5ca9a5d-95c4-4c2a-a059-35cb5f5ab326"} {"msg":"[0]R generation[0..12]: [51, 51, 40, 50, 36, 46, 50, 50, 50, 48, 51, 46]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:20.499528561Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1388,"":"downstairs","":"downstairs","session_id":"f5ca9a5d-95c4-4c2a-a059-35cb5f5ab326"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:20.49954633Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1388,"":"downstairs","":"downstairs","session_id":"f5ca9a5d-95c4-4c2a-a059-35cb5f5ab326"} {"msg":"[1]R flush_numbers[0..12]: [100, 100, 76, 96, 68, 88, 96, 96, 96, 92, 100, 90]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:20.499563439Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1388,"":"downstairs","":"downstairs","session_id":"f5ca9a5d-95c4-4c2a-a059-35cb5f5ab326"} {"msg":"[1]R generation[0..12]: [51, 51, 40, 50, 36, 46, 50, 50, 50, 48, 51, 46]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:20.499580349Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1388,"":"downstairs","":"downstairs","session_id":"f5ca9a5d-95c4-4c2a-a059-35cb5f5ab326"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:20.499597548Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1388,"":"downstairs","":"downstairs","session_id":"f5ca9a5d-95c4-4c2a-a059-35cb5f5ab326"} {"msg":"[2]R flush_numbers[0..12]: [100, 100, 76, 96, 68, 88, 96, 96, 96, 92, 100, 90]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:20.499614707Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1388,"":"downstairs","":"downstairs","session_id":"f5ca9a5d-95c4-4c2a-a059-35cb5f5ab326"} {"msg":"[2]R generation[0..12]: [51, 51, 40, 50, 36, 46, 50, 50, 50, 48, 51, 46]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:20.499631526Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1388,"":"downstairs","":"downstairs","session_id":"f5ca9a5d-95c4-4c2a-a059-35cb5f5ab326"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:20.499648645Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1388,"":"downstairs","":"downstairs","session_id":"f5ca9a5d-95c4-4c2a-a059-35cb5f5ab326"} {"msg":"Max found gen is 52","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:20.499664714Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1388,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO f5ca9a5d-95c4-4c2a-a059-35cb5f5ab326"} {"msg":"Generation requested: 52 >= found:52","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:20.499692033Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1388,"":"downstairs","":"downstairs","session_id":"f5ca9a5d-95c4-4c2a-a059-35cb5f5ab326"} {"msg":"Next flush: 101","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:20.499708463Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1388,"":"downstairs","":"downstairs","session_id":"f5ca9a5d-95c4-4c2a-a059-35cb5f5ab326"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:20.499724742Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1388,"":"downstairs","":"downstairs","session_id":"f5ca9a5d-95c4-4c2a-a059-35cb5f5ab326"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:20.499743701Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1388,"session_id":"f5ca9a5d-95c4-4c2a-a059-35cb5f5ab326"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:20.49975913Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1388,"client":"0","":"downstairs","session_id":"f5ca9a5d-95c4-4c2a-a059-35cb5f5ab326"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:20.499775729Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1388,"client":"1","":"downstairs","session_id":"f5ca9a5d-95c4-4c2a-a059-35cb5f5ab326"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:20.499792538Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1388,"client":"2","":"downstairs","session_id":"f5ca9a5d-95c4-4c2a-a059-35cb5f5ab326"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:20.499809138Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1388,"session_id":"f5ca9a5d-95c4-4c2a-a059-35cb5f5ab326"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:20.499824577Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1388,"session_id":"f5ca9a5d-95c4-4c2a-a059-35cb5f5ab326"} {"msg":"eac9bfd9-cce5-41ec-8001-191ec8cb5356 is now active with session: f5ca9a5d-95c4-4c2a-a059-35cb5f5ab326","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:20.499840506Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1388,"session_id":"f5ca9a5d-95c4-4c2a-a059-35cb5f5ab326"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:20.499855625Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1388,"session_id":"f5ca9a5d-95c4-4c2a-a059-35cb5f5ab326"} {"msg":"The guest has finished waiting for activation with:52","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:20.499871055Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1388} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 572 len 24576 data: 6 6 4 5 4 4 02/30 Read block 42 len 12288 03/30 Read block 546 len 36864 04/30 Write block 226 len 20480 data: 6 8 6 6 6 05/30 Write block 14 len 8192 data: 3 3 06/30 Write block 127 len 40960 data: 2 3 3 5 4 5 7 7 7 6 07/30 Write block 8 len 40960 data: 5 5 5 5 7 6 4 4 4 4 08/30 Read block 549 len 24576 09/30 Write block 454 len 20480 data: 4 3 4 4 3 10/30 Read block 273 len 4096 11/30 Read block 27 len 40960 12/30 Read block 322 len 8192 13/30 Read block 588 len 8192 14/30 Read block 110 len 32768 15/30 Write block 353 len 16384 data: 8 7 10 10 16/30 Read block 482 len 8192 17/30 Flush 18/30 Write block 479 len 24576 data: 5 3 4 4 4 4 19/30 Write block 349 len 12288 data: 7 6 6 20/30 Read block 183 len 12288 21/30 Write block 179 len 8192 data: 5 7 22/30 Write block 493 len 20480 data: 4 6 5 4 5 23/30 Read block 55 len 24576 24/30 Read block 548 len 4096 25/30 Write block 115 len 20480 data: 4 4 4 4 5 26/30 Read block 436 len 40960 27/30 Write block 54 len 8192 data: 4 4 28/30 Write block 200 len 32768 data: 3 2 2 3 6 7 7 6 29/30 Read block 194 len 40960 30/30 Read block 478 len 20480 ---------------------------------------------------------------- Crucible gen:52 GIO:true work queues: Upstairs:0 downstairs:13 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1017 Acked Write 6 Done Done Done false 1018 Acked Write 3 Done Done Done false 1019 Acked Read 3 Done Done Done false 1020 Acked Write 2 Done Done Done false 1021 Acked Write 5 Done Done Done false 1022 Acked Read 6 Done Done Done false 1023 Acked Read 1 Done Done Done false 1024 Acked Write 5 Done Done Done false 1025 Acked Read 10 Done Done Done false 1026 Acked Write 2 Done Done Done false 1027 Acked Write 8 Done Done Done false 1028 Acked Read 10 Done Done Done false 1029 Acked Read 5 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 0 1 Done 13 12 13 38 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1016 1016 1016 Downstairs last five completed: 1016 1015 1014 1013 1012 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:24:20.523558267Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1388} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 52 52 52 101 101 101 F F F 1 020-039 51 51 51 100 100 100 F F F 2 040-059 40 40 40 76 76 76 T T T 3 060-079 50 50 50 96 96 96 F F F 4 080-099 36 36 36 68 68 68 F F F 5 100-119 46 46 46 88 88 88 T T T 6 120-139 52 52 52 101 101 101 F F F 7 140-159 50 50 50 96 96 96 F F F 8 160-179 50 50 50 96 96 96 T T T 9 180-199 48 48 48 92 92 92 T T T 10 200-219 51 51 51 100 100 100 T T T 11 220-239 52 52 52 101 101 101 F F F 12 240-259 46 46 46 88 88 88 F F F 13 260-279 49 49 49 95 95 95 F F F 14 280-299 49 49 49 95 95 95 F F F 15 300-319 48 48 48 92 92 92 F F F 16 320-339 50 50 50 98 98 98 F F F 17 340-359 52 52 52 101 101 101 T T T 18 360-379 51 51 51 100 100 100 F F F 19 380-399 50 50 50 99 99 99 F F F 20 400-419 42 42 42 82 82 82 F F F 21 420-439 49 49 49 95 95 95 F F F 22 440-459 52 52 52 101 101 101 F F F 23 460-479 48 48 48 92 92 92 T T T 24 480-499 48 48 48 92 92 92 T T T 25 500-519 48 48 48 92 92 92 F F F 26 520-539 46 46 46 90 90 90 F F F 27 540-559 48 48 48 94 94 94 F F F 28 560-579 52 52 52 101 101 101 F F F 29 580-599 51 51 51 100 100 100 F F F Max gen: 52, Max flush: 101 A Difference in extent metadata is expected here On loop 26 Verifying data now No change after verify Loop: 26 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 52 52 52 101 101 101 F F F 1 020-039 51 51 51 100 100 100 F F F 2 040-059 53 53 53 102 102 102 F F F 3 060-079 50 50 50 96 96 96 F F F 4 080-099 36 36 36 68 68 68 F F F 5 100-119 53 53 53 102 102 102 F F F 6 120-139 52 52 52 101 101 101 F F F 7 140-159 50 50 50 96 96 96 F F F 8 160-179 53 53 53 102 102 102 F F F 9 180-199 53 53 53 102 102 102 F F F 10 200-219 53 53 53 102 102 102 F F F 11 220-239 52 52 52 101 101 101 F F F 12 240-259 46 46 46 88 88 88 F F F 13 260-279 49 49 49 95 95 95 F F F 14 280-299 49 49 49 95 95 95 F F F 15 300-319 48 48 48 92 92 92 F F F 16 320-339 50 50 50 98 98 98 F F F 17 340-359 53 53 53 102 102 102 F F F 18 360-379 51 51 51 100 100 100 F F F 19 380-399 50 50 50 99 99 99 F F F 20 400-419 42 42 42 82 82 82 F F F 21 420-439 49 49 49 95 95 95 F F F 22 440-459 52 52 52 101 101 101 F F F 23 460-479 53 53 53 102 102 102 F F F 24 480-499 53 53 53 102 102 102 F F F 25 500-519 48 48 48 92 92 92 F F F 26 520-539 46 46 46 90 90 90 F F F 27 540-559 48 48 48 94 94 94 F F F 28 560-579 52 52 52 101 101 101 F F F 29 580-599 51 51 51 100 100 100 F F F Max gen: 53, Max flush: 102 Begin loop 27 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:24.912711991Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1396,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:24:24.955367097Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1396,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:24.955745738Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1396,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:24.988007211Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1396,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:24.988549985Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1396} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"0fd98d0d5d909611f8b408ccda2559d35fe117bb\",\n git_commit_timestamp: \"2024-11-08T20:05:44.000000000Z\",\n git_branch: \"mkeeter/drop-skipped-backpressure-guard\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:24.988584913Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1396} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:24.988598722Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1396} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:24.988612972Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1396} {"msg":"Crucible d7213225-7c6c-4163-9f93-ad6719d570b1 has session id: e9a9eecd-464d-4c66-ada3-5743e23439b1","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:24.989013782Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1396,"session_id":"e9a9eecd-464d-4c66-ada3-5743e23439b1"} {"msg":"Upstairs opts: Upstairs UUID: d7213225-7c6c-4163-9f93-ad6719d570b1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:24.98906Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1396,"session_id":"e9a9eecd-464d-4c66-ada3-5743e23439b1"} {"msg":"Crucible stats registered with UUID: d7213225-7c6c-4163-9f93-ad6719d570b1","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:24.989084838Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1396,"session_id":"e9a9eecd-464d-4c66-ada3-5743e23439b1"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:25.020466605Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1396,"task":"crutest"} {"msg":"The guest has requested activation with gen:54","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:25.020498633Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1396} {"msg":"d7213225-7c6c-4163-9f93-ad6719d570b1 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:25.020549861Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1396,"session_id":"e9a9eecd-464d-4c66-ada3-5743e23439b1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:25.02056988Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1396,"client":"0","":"downstairs","session_id":"e9a9eecd-464d-4c66-ada3-5743e23439b1"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:25.020597798Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1396,"client":"0","":"downstairs","session_id":"e9a9eecd-464d-4c66-ada3-5743e23439b1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:25.020615688Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1396,"client":"1","":"downstairs","session_id":"e9a9eecd-464d-4c66-ada3-5743e23439b1"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:25.020632977Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1396,"client":"1","":"downstairs","session_id":"e9a9eecd-464d-4c66-ada3-5743e23439b1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:25.020649566Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1396,"client":"2","":"downstairs","session_id":"e9a9eecd-464d-4c66-ada3-5743e23439b1"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:25.020699613Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1396,"client":"2","":"downstairs","session_id":"e9a9eecd-464d-4c66-ada3-5743e23439b1"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:25.020728402Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1396,"":"io task","client":"2","":"downstairs","session_id":"e9a9eecd-464d-4c66-ada3-5743e23439b1"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:25.020744441Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1396,"":"io task","client":"0","":"downstairs","session_id":"e9a9eecd-464d-4c66-ada3-5743e23439b1"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:25.02075875Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1396,"":"io task","client":"1","":"downstairs","session_id":"e9a9eecd-464d-4c66-ada3-5743e23439b1"} {"msg":"ds_connection connected from Ok(127.0.0.1:47615)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:25.020848896Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1396,"":"io task","client":"2","":"downstairs","session_id":"e9a9eecd-464d-4c66-ada3-5743e23439b1"} {"msg":"ds_connection connected from Ok(127.0.0.1:36960)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:25.021040827Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1396,"":"io task","client":"0","":"downstairs","session_id":"e9a9eecd-464d-4c66-ada3-5743e23439b1"} {"msg":"ds_connection connected from Ok(127.0.0.1:40715)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:25.021080665Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1396,"":"io task","client":"1","":"downstairs","session_id":"e9a9eecd-464d-4c66-ada3-5743e23439b1"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:25.021631998Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1396,"client":"2","":"downstairs","session_id":"e9a9eecd-464d-4c66-ada3-5743e23439b1"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:25.021675056Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1396,"client":"2","":"downstairs","session_id":"e9a9eecd-464d-4c66-ada3-5743e23439b1"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:25.021707014Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1396,"client":"1","":"downstairs","session_id":"e9a9eecd-464d-4c66-ada3-5743e23439b1"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:25.021735103Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1396,"client":"1","":"downstairs","session_id":"e9a9eecd-464d-4c66-ada3-5743e23439b1"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:25.02180071Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1396,"client":"0","":"downstairs","session_id":"e9a9eecd-464d-4c66-ada3-5743e23439b1"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:25.021899215Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1396,"client":"0","":"downstairs","session_id":"e9a9eecd-464d-4c66-ada3-5743e23439b1"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:25.022183951Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1396,"client":"1","":"downstairs","session_id":"e9a9eecd-464d-4c66-ada3-5743e23439b1"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:25.022219169Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1396,"client":"2","":"downstairs","session_id":"e9a9eecd-464d-4c66-ada3-5743e23439b1"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:25.022554923Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1396,"client":"0","":"downstairs","session_id":"e9a9eecd-464d-4c66-ada3-5743e23439b1"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:25.022776372Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1396,"client":"2","":"downstairs","session_id":"e9a9eecd-464d-4c66-ada3-5743e23439b1"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:25.02280397Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1396,"client":"2","":"downstairs","session_id":"e9a9eecd-464d-4c66-ada3-5743e23439b1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:25.02281925Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1396,"session_id":"e9a9eecd-464d-4c66-ada3-5743e23439b1"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:25.022876497Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1396,"client":"1","":"downstairs","session_id":"e9a9eecd-464d-4c66-ada3-5743e23439b1"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:25.022896226Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1396,"client":"1","":"downstairs","session_id":"e9a9eecd-464d-4c66-ada3-5743e23439b1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:25.022910135Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1396,"session_id":"e9a9eecd-464d-4c66-ada3-5743e23439b1"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:25.023079447Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1396,"client":"0","":"downstairs","session_id":"e9a9eecd-464d-4c66-ada3-5743e23439b1"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:25.023101856Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1396,"client":"0","":"downstairs","session_id":"e9a9eecd-464d-4c66-ada3-5743e23439b1"} {"msg":"[0]R flush_numbers[0..12]: [101, 100, 102, 96, 68, 102, 101, 96, 102, 102, 102, 101]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:25.023119665Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1396,"":"downstairs","":"downstairs","session_id":"e9a9eecd-464d-4c66-ada3-5743e23439b1"} {"msg":"[0]R generation[0..12]: [52, 51, 53, 50, 36, 53, 52, 50, 53, 53, 53, 52]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:25.023133894Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1396,"":"downstairs","":"downstairs","session_id":"e9a9eecd-464d-4c66-ada3-5743e23439b1"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:25.023147844Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1396,"":"downstairs","":"downstairs","session_id":"e9a9eecd-464d-4c66-ada3-5743e23439b1"} {"msg":"[1]R flush_numbers[0..12]: [101, 100, 102, 96, 68, 102, 101, 96, 102, 102, 102, 101]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:25.023161763Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1396,"":"downstairs","":"downstairs","session_id":"e9a9eecd-464d-4c66-ada3-5743e23439b1"} {"msg":"[1]R generation[0..12]: [52, 51, 53, 50, 36, 53, 52, 50, 53, 53, 53, 52]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:25.023175632Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1396,"":"downstairs","":"downstairs","session_id":"e9a9eecd-464d-4c66-ada3-5743e23439b1"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:25.023189722Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1396,"":"downstairs","":"downstairs","session_id":"e9a9eecd-464d-4c66-ada3-5743e23439b1"} {"msg":"[2]R flush_numbers[0..12]: [101, 100, 102, 96, 68, 102, 101, 96, 102, 102, 102, 101]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:25.023203181Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1396,"":"downstairs","":"downstairs","session_id":"e9a9eecd-464d-4c66-ada3-5743e23439b1"} {"msg":"[2]R generation[0..12]: [52, 51, 53, 50, 36, 53, 52, 50, 53, 53, 53, 52]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:25.0232168Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1396,"":"downstairs","":"downstairs","session_id":"e9a9eecd-464d-4c66-ada3-5743e23439b1"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:25.02323072Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1396,"":"downstairs","":"downstairs","session_id":"e9a9eecd-464d-4c66-ada3-5743e23439b1"} {"msg":"Max found gen is 54","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:25.023243969Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1396,"":"downstairs","":"downstairs","session_id":"e9a9eecd-464d-4c66-ada3-5743e23439b1"} {"msg":"Generation requested: 54 >= found:54","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:25.023257098Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1396,"":"downstairs","":"downstairs","session_id":"e9a9eecd-464d-4c66-ada3-5743e23439b1"} {"msg":"Next flush: 103","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:25.023270108Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1396,"":"downstairs","":"downstairs","session_id":"e9a9eecd-464d-4c66-ada3-5743e23439b1"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:25.023283027Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1396Wait for a query_work_queue command to finish before sending IO ,"":"downstairs","":"downstairs","session_id":"e9a9eecd-464d-4c66-ada3-5743e23439b1"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:25.023304106Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1396,"session_id":"e9a9eecd-464d-4c66-ada3-5743e23439b1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:25.023317945Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1396,"client":"0","":"downstairs","session_id":"e9a9eecd-464d-4c66-ada3-5743e23439b1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:25.023331415Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1396,"client":"1","":"downstairs","session_id":"e9a9eecd-464d-4c66-ada3-5743e23439b1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:25.023344644Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1396,"client":"2","":"downstairs","session_id":"e9a9eecd-464d-4c66-ada3-5743e23439b1"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:25.023358074Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1396,"session_id":"e9a9eecd-464d-4c66-ada3-5743e23439b1"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:25.023370413Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1396,"session_id":"e9a9eecd-464d-4c66-ada3-5743e23439b1"} {"msg":"d7213225-7c6c-4163-9f93-ad6719d570b1 is now active with session: e9a9eecd-464d-4c66-ada3-5743e23439b1","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:25.023383052Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1396,"session_id":"e9a9eecd-464d-4c66-ada3-5743e23439b1"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:25.023395162Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1396,"session_id":"e9a9eecd-464d-4c66-ada3-5743e23439b1"} {"msg":"The guest has finished waiting for activation with:54","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:25.023407411Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1396} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 483 len 36864 data: 5 5 3 4 4 5 5 4 3 02/30 Write block 399 len 4096 data: 5 03/30 Flush 04/30 Write block 421 len 24576 data: 6 5 5 3 4 4 05/30 Flush 06/30 Write block 232 len 36864 data: 6 5 4 5 5 7 8 7 7 07/30 Read block 578 len 28672 08/30 Read block 90 len 4096 09/30 Read block 341 len 36864 10/30 Read block 371 len 8192 11/30 Read block 132 len 4096 12/30 Write block 85 len 32768 data: 5 6 6 5 4 3 3 3 13/30 Write block 12 len 32768 data: 8 7 5 5 5 5 4 5 14/30 Write block 125 len 12288 data: 4 4 3 15/30 Write block 309 len 4096 data: 4 16/30 Read block 339 len 32768 17/30 Write block 567 len 36864 data: 3 4 4 4 6 7 7 5 6 18/30 Read block 187 len 4096 19/30 Write block 412 len 8192 data: 9 9 20/30 Read block 178 len 12288 21/30 Write block 282 len 20480 data: 7 6 4 5 7 22/30 Read block 516 len 28672 23/30 Read block 562 len 40960 24/30 Write block 19 len 36864 data: 6 6 7 7 7 8 8 9 7 25/30 Write block 46 len 4096 data: 3 26/30 Write block 48 len 32768 data: 3 3 4 4 3 3 5 5 27/30 Read block 460 len 16384 28/30 Write block 581 len 40960 data: 4 4 7 8 11 10 10 9 8 6 29/30 Write block 439 len 12288 data: 6 5 7 30/30 Write block 63 len 24576 data: 5 5 4 3 3 4 ---------------------------------------------------------------- Crucible gen:54 GIO:true work queues: Upstairs:0 downstairs:25 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1005 Acked Write 9 Done Done Done false 1006 Acked Read 7 Done Done Done false 1007 Acked Read 1 Done Done Done false 1008 Acked Read 9 Done Done Done false 1009 Acked Read 2 Done Done Done false 1010 Acked Read 1 Done Done Done false 1011 Acked Write 8 Done Done Done false 1012 Acked Write 8 Done Done Done false 1013 Acked Write 3 Done Done Done false 1014 Acked Write 1 Done Done Done false 1015 Acked Read 8 Done Done Done false 1016 Acked Write 9 Done Done Done false 1017 Acked Read 1 Done Done Done false 1018 Acked Write 2 Done Done Done false 1019 Acked Read 3 Done Done Done false 1020 Acked Write 5 Done Done Done false 1021 Acked Read 7 Done Done Done false 1022 Acked Read 10 Done Done Done false 1023 Acked Write 9 Done Done Done false 1024 Acked Write 1 Done Done Done false 1025 Acked Write 8 Done Done Done false 1026 Acked Read 4 Done Done Done false 1027 Acked Write 10 Sent Sent Sent false 1028 Acked Write 3 Sent Sent Sent false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 3 3 9 Done 22 22 22 66 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1004 1004 1004 Downstairs last five completed: 1004 1003 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:24:25.046620417Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1396} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 52 52 52 101 101 101 T T T 1 020-039 51 51 51 100 100 100 T T T 2 040-059 53 53 53 102 102 102 T T T 3 060-079 50 50 50 96 96 96 T T T 4 080-099 36 36 36 68 68 68 T T T 5 100-119 53 53 53 102 102 102 F F F 6 120-139 52 52 52 101 101 101 T T T 7 140-159 50 50 50 96 96 96 F F F 8 160-179 53 53 53 102 102 102 F F F 9 180-199 53 53 53 102 102 102 F F F 10 200-219 53 53 53 102 102 102 F F F 11 220-239 52 52 52 101 101 101 T T T 12 240-259 46 46 46 88 88 88 T T T 13 260-279 49 49 49 95 95 95 F F F 14 280-299 49 49 49 95 95 95 T T T 15 300-319 48 48 48 92 92 92 T T T 16 320-339 50 50 50 98 98 98 F F F 17 340-359 53 53 53 102 102 102 F F F 18 360-379 51 51 51 100 100 100 F F F 19 380-399 54 54 54 103 103 103 F F F 20 400-419 42 42 42 82 82 82 T T T 21 420-439 54 54 54 104 104 104 T T T 22 440-459 52 52 52 101 101 101 T T T 23 460-479 53 53 53 102 102 102 F F F 24 480-499 54 54 54 103 103 103 F F F 25 500-519 48 48 48 92 92 92 F F F 26 520-539 46 46 46 90 90 90 F F F 27 540-559 48 48 48 94 94 94 F F F 28 560-579 52 52 52 101 101 101 T T T 29 580-599 51 51 51 100 100 100 T T T Max gen: 54, Max flush: 104 A Difference in extent metadata is expected here On loop 27 Verifying data now No change after verify Loop: 27 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 55 55 55 105 105 105 F F F 1 020-039 55 55 55 105 105 105 F F F 2 040-059 55 55 55 105 105 105 F F F 3 060-079 55 55 55 105 105 105 F F F 4 080-099 55 55 55 105 105 105 F F F 5 100-119 53 53 53 102 102 102 F F F 6 120-139 55 55 55 105 105 105 F F F 7 140-159 50 50 50 96 96 96 F F F 8 160-179 53 53 53 102 102 102 F F F 9 180-199 53 53 53 102 102 102 F F F 10 200-219 53 53 53 102 102 102 F F F 11 220-239 55 55 55 105 105 105 F F F 12 240-259 55 55 55 105 105 105 F F F 13 260-279 49 49 49 95 95 95 F F F 14 280-299 55 55 55 105 105 105 F F F 15 300-319 55 55 55 105 105 105 F F F 16 320-339 50 50 50 98 98 98 F F F 17 340-359 53 53 53 102 102 102 F F F 18 360-379 51 51 51 100 100 100 F F F 19 380-399 54 54 54 103 103 103 F F F 20 400-419 55 55 55 105 105 105 F F F 21 420-439 55 55 55 105 105 105 F F F 22 440-459 55 55 55 105 105 105 F F F 23 460-479 53 53 53 102 102 102 F F F 24 480-499 54 54 54 103 103 103 F F F 25 500-519 48 48 48 92 92 92 F F F 26 520-539 46 46 46 90 90 90 F F F 27 540-559 48 48 48 94 94 94 F F F 28 560-579 55 55 55 105 105 105 F F F 29 580-599 55 55 55 105 105 105 F F F Max gen: 55, Max flush: 105 Begin loop 28 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:29.543700416Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1404,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:29.588352354Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1404,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:29.589033181Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1404} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"0fd98d0d5d909611f8b408ccda2559d35fe117bb\",\n git_commit_timestamp: \"2024-11-08T20:05:44.000000000Z\",\n git_branch: \"mkeeter/drop-skipped-backpressure-guard\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:29.589071169Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1404} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:29.589085749Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1404} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:29.589100578Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1404} {"msg":"Crucible e0a11671-e217-4b10-a41a-d08a7859c057 has session id: a11b6101-752a-44c3-af58-2e9eb7b626da","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:29.589565705Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1404,"session_id":"a11b6101-752a-44c3-af58-2e9eb7b626da"} {"msg":"Upstairs opts: Upstairs UUID: e0a11671-e217-4b10-a41a-d08a7859c057, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:29.589612713Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1404,"session_id":"a11b6101-752a-44c3-af58-2e9eb7b626da"} {"msg":"Crucible stats registered with UUID: e0a11671-e217-4b10-a41a-d08a7859c057","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:29.589643311Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1404,"session_id":"a11b6101-752a-44c3-af58-2e9eb7b626da"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:29.621512164Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1404,"task":"crutest"} {"msg":"The guest has requested activation with gen:56","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:29.621548262Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1404} {"msg":"e0a11671-e217-4b10-a41a-d08a7859c057 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:29.62158241Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1404,"session_id":"a11b6101-752a-44c3-af58-2e9eb7b626da"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:29.621599639Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1404,"client":"0","":"downstairs","session_id":"a11b6101-752a-44c3-af58-2e9eb7b626da"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:29.621628648Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1404,"client":"0","":"downstairs","session_id":"a11b6101-752a-44c3-af58-2e9eb7b626da"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:29.621646887Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1404,"client":"1","":"downstairs","session_id":"a11b6101-752a-44c3-af58-2e9eb7b626da"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:29.621664026Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1404,"client":"1","":"downstairs","session_id":"a11b6101-752a-44c3-af58-2e9eb7b626da"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:29.621680565Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1404,"client":"2","":"downstairs","session_id":"a11b6101-752a-44c3-af58-2e9eb7b626da"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:29.621697544Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1404,"client":"2","":"downstairs","session_id":"a11b6101-752a-44c3-af58-2e9eb7b626da"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:29.621713854Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1404,"":"io task","client":"2","":"downstairs","session_id":"a11b6101-752a-44c3-af58-2e9eb7b626da"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:29.621735273Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1404,"":"io task","client":"0","":"downstairs","session_id":"a11b6101-752a-44c3-af58-2e9eb7b626da"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:29.621767801Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1404,"":"io task","client":"1","":"downstairs","session_id":"a11b6101-752a-44c3-af58-2e9eb7b626da"} {"msg":"ds_connection connected from Ok(127.0.0.1:48382)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:29.622097345Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1404,"":"io task","client":"1","":"downstairs","session_id":"a11b6101-752a-44c3-af58-2e9eb7b626da"} {"msg":"ds_connection connected from Ok(127.0.0.1:35389)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:29.622128114Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1404,"":"io task","client":"0","":"downstairs","session_id":"a11b6101-752a-44c3-af58-2e9eb7b626da"} {"msg":"ds_connection connected from Ok(127.0.0.1:40115)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:29.622146513Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1404,"":"io task","client":"2","":"downstairs","session_id":"a11b6101-752a-44c3-af58-2e9eb7b626da"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:29.622577861Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1404,"client":"0","":"downstairs","session_id":"a11b6101-752a-44c3-af58-2e9eb7b626da"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:29.62262037Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1404,"client":"0","":"downstairs","session_id":"a11b6101-752a-44c3-af58-2e9eb7b626da"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:29.622651278Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1404,"client":"1","":"downstairs","session_id":"a11b6101-752a-44c3-af58-2e9eb7b626da"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:29.622679267Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1404,"client":"1","":"downstairs","session_id":"a11b6101-752a-44c3-af58-2e9eb7b626da"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:29.622706075Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1404,"client":"2","":"downstairs","session_id":"a11b6101-752a-44c3-af58-2e9eb7b626da"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:29.622732274Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1404,"client":"2","":"downstairs","session_id":"a11b6101-752a-44c3-af58-2e9eb7b626da"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:29.623199791Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1404,"client":"1","":"downstairs","session_id":"a11b6101-752a-44c3-af58-2e9eb7b626da"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:29.623239919Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1404,"client":"0","":"downstairs","session_id":"a11b6101-752a-44c3-af58-2e9eb7b626da"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:29.623447169Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1404,"client":"2","":"downstairs","session_id":"a11b6101-752a-44c3-af58-2e9eb7b626da"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:29.623755794Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1404,"client":"1","":"downstairs","session_id":"a11b6101-752a-44c3-af58-2e9eb7b626da"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:29.623786713Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1404,"client":"1","":"downstairs","session_id":"a11b6101-752a-44c3-af58-2e9eb7b626da"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:29.623802382Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1404,"session_id":"a11b6101-752a-44c3-af58-2e9eb7b626da"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:29.623815341Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1404,"client":"0","":"downstairs","session_id":"a11b6101-752a-44c3-af58-2e9eb7b626da"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:29.6238289Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1404,"client":"0","":"downstairs","session_id":"a11b6101-752a-44c3-af58-2e9eb7b626da"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:29.62384235Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1404,"session_id":"a11b6101-752a-44c3-af58-2e9eb7b626da"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:29.623997822Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1404,"client":"2","":"downstairs","session_id":"a11b6101-752a-44c3-af58-2e9eb7b626da"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:29.624020941Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1404,"client":"2","":"downstairs","session_id":"a11b6101-752a-44c3-af58-2e9eb7b626da"} {"msg":"[0]R flush_numbers[0..12]: [105, 105, 105, 105, 105, 102, 105, 96, 102, 102, 102, 105]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:29.62403647Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1404,"":"downstairs","":"downstairs","session_id":"a11b6101-752a-44c3-af58-2e9eb7b626da"} {"msg":"[0]R generation[0..12]: [55, 55, 55, 55, 55, 53, 55, 50, 53, 53, 53, 55]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:29.62405038Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1404,"":"downstairs","":"downstairs","session_id":"a11b6101-752a-44c3-af58-2e9eb7b626da"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:29.624064649Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1404,"":"downstairs","":"downstairs","session_id":"a11b6101-752a-44c3-af58-2e9eb7b626da"} {"msg":"[1]R flush_numbers[0..12]: [105, 105, 105, 105, 105, 102, 105, 96, 102, 102, 102, 105]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:29.624078508Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1404,"":"downstairs","":"downstairs","session_id":"a11b6101-752a-44c3-af58-2e9eb7b626da"} {"msg":"[1]R generation[0..12]: [55, 55, 55, 55, 55, 53, 55, 50, 53, 53, 53, 55]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:29.624092307Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1404,"":"downstairs","":"downstairs","session_id":"a11b6101-752a-44c3-af58-2e9eb7b626da"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:29.624106127Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1404,"":"downstairs","":"downstairs","session_id":"a11b6101-752a-44c3-af58-2e9eb7b626da"} {"msg":"[2]R flush_numbers[0..12]: [105, 105, 105, 105, 105, 102, 105, 96, 102, 102, 102, 105]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:29.624120106Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1404,"":"downstairs","":"downstairs","session_id":"a11b6101-752a-44c3-af58-2e9eb7b626da"} {"msg":"[2]R generation[0..12]: [55, 55, 55, 55, 55, 53, 55, 50, 53, 53, 53, 55]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:29.624133676Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1404,"":"downstairs","":"downstairs","session_id":"a11b6101-752a-44c3-af58-2e9eb7b626da"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:29.624147625Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1404,"":"downstairs","":"downstairs","session_id":"a11b6101-752a-44c3-af58-2e9eb7b626da"} {"msg":"Max found gen is 56","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:29.624160624Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1404,"":"downstairs","":"downstairs","session_id":"a11b6101-752a-44c3-af58-2e9eb7b626da"} {"msg":"Generation requested: 56 >= found:56","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:29.624173963Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1404,"":"downstairs","":"downstairs","session_id":"a11b6101-752a-44c3-af58-2e9eb7b626da"} {"msg":"Next flush: 106","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:29.624186973Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1404,"":"downstairs","":"downstairs","session_id":"a11b6101-752a-44c3-af58-2e9eb7b626da"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:29.624199962Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1404,"":"downstairs","":"downstairs","session_id":"a11b6101-752a-44c3-af58-2e9eb7b626da"} Wait for a query_work_queue command to finish before sending IO {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:29.624223381Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1404,"session_id":"a11b6101-752a-44c3-af58-2e9eb7b626da"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:29.62423594Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1404,"client":"0","":"downstairs","session_id":"a11b6101-752a-44c3-af58-2e9eb7b626da"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:29.62424908Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1404,"client":"1","":"downstairs","session_id":"a11b6101-752a-44c3-af58-2e9eb7b626da"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:29.624262229Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1404,"client":"2","":"downstairs","session_id":"a11b6101-752a-44c3-af58-2e9eb7b626da"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:29.624275869Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1404,"session_id":"a11b6101-752a-44c3-af58-2e9eb7b626da"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:29.624288058Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1404,"session_id":"a11b6101-752a-44c3-af58-2e9eb7b626da"} {"msg":"e0a11671-e217-4b10-a41a-d08a7859c057 is now active with session: a11b6101-752a-44c3-af58-2e9eb7b626da","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:29.624300597Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1404,"session_id":"a11b6101-752a-44c3-af58-2e9eb7b626da"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:29.624312567Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1404,"session_id":"a11b6101-752a-44c3-af58-2e9eb7b626da"} {"msg":"The guest has finished waiting for activation with:56","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:29.624324596Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1404} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 286 len 36864 data: 8 8 9 8 7 6 7 6 6 02/30 Read block 332 len 20480 03/30 Read block 14 len 20480 04/30 Read block 262 len 4096 05/30 Read block 196 len 32768 06/30 Write block 37 len 40960 data: 2 3 4 4 3 4 5 5 4 4 07/30 Read block 384 len 28672 08/30 Write block 361 len 12288 data: 12 10 9 09/30 Read block 437 len 20480 10/30 Read block 516 len 32768 11/30 Write block 386 len 36864 data: 3 3 4 4 5 5 6 6 6 12/30 Write block 100 len 40960 data: 4 4 4 4 4 3 4 7 6 6 13/30 Flush 14/30 Write block 65 len 24576 data: 5 4 4 5 4 5 15/30 Write block 429 len 32768 data: 7 7 6 6 4 5 7 7 16/30 Read block 287 len 40960 17/30 Read block 324 len 36864 18/30 Write block 206 len 36864 data: 8 7 5 6 5 7 6 5 4 19/30 Write block 446 len 8192 data: 6 5 20/30 Flush 21/30 Write block 480 len 4096 data: 4 22/30 Read block 29 len 24576 23/30 Read block 167 len 28672 24/30 Write block 144 len 32768 data: 7 6 5 5 4 5 5 6 25/30 Write block 397 len 28672 data: 5 5 6 4 4 4 4 26/30 Write block 295 len 8192 data: 4 2 27/30 Write block 22 len 24576 data: 8 8 9 9 10 8 28/30 Read block 182 len 8192 29/30 Write block 393 len 32768 data: 7 7 5 4 6 6 7 5 30/30 Write block 365 len 16384 data: 7 6 7 9 ---------------------------------------------------------------- Crucible gen:56 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1020 Acked Write 1 Done Done Done false 1021 Acked Read 6 Done Done Done false 1022 Acked Read 7 Done Done Done false 1023 Acked Write 8 Done Done Done false 1024 Acked Write 7 Done Done Done false 1025 Acked Write 2 Done Done Done false 1026 Acked Write 6 Done Done Done false 1027 Acked Read 2 Done Done Done false 1028 Acked Write 8 Sent Sent Sent false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 8 8 8 24 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1019 1019 1019 Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:24:29.649094285Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1404} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 55 55 55 105 105 105 F F F 1 020-039 56 56 56 106 106 106 T T T 2 040-059 56 56 56 106 106 106 F F F 3 060-079 56 56 56 107 107 107 F F F 4 080-099 55 55 55 105 105 105 F F F 5 100-119 56 56 56 106 106 106 F F F 6 120-139 55 55 55 105 105 105 F F F 7 140-159 50 50 50 96 96 96 T T T 8 160-179 53 53 53 102 102 102 F F F 9 180-199 53 53 53 102 102 102 F F F 10 200-219 56 56 56 107 107 107 F F F 11 220-239 55 55 55 105 105 105 F F F 12 240-259 55 55 55 105 105 105 F F F 13 260-279 49 49 49 95 95 95 F F F 14 280-299 56 56 56 106 106 106 T T T 15 300-319 55 55 55 105 105 105 F F F 16 320-339 50 50 50 98 98 98 F F F 17 340-359 53 53 53 102 102 102 F F F 18 360-379 56 56 56 106 106 106 T T T 19 380-399 56 56 56 106 106 106 T T T 20 400-419 55 55 55 105 105 105 T T T 21 420-439 56 56 56 107 107 107 F F F 22 440-459 56 56 56 107 107 107 F F F 23 460-479 53 53 53 102 102 102 F F F 24 480-499 54 54 54 103 103 103 T T T 25 500-519 48 48 48 92 92 92 F F F 26 520-539 46 46 46 90 90 90 F F F 27 540-559 48 48 48 94 94 94 F F F 28 560-579 55 55 55 105 105 105 F F F 29 580-599 55 55 55 105 105 105 F F F Max gen: 56, Max flush: 107 A Difference in extent metadata is expected here On loop 28 Verifying data now No change after verify Loop: 28 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 55 55 55 105 105 105 F F F 1 020-039 57 57 57 108 108 108 F F F 2 040-059 56 56 56 106 106 106 F F F 3 060-079 56 56 56 107 107 107 F F F 4 080-099 55 55 55 105 105 105 F F F 5 100-119 56 56 56 106 106 106 F F F 6 120-139 55 55 55 105 105 105 F F F 7 140-159 57 57 57 108 108 108 F F F 8 160-179 53 53 53 102 102 102 F F F 9 180-199 53 53 53 102 102 102 F F F 10 200-219 56 56 56 107 107 107 F F F 11 220-239 55 55 55 105 105 105 F F F 12 240-259 55 55 55 105 105 105 F F F 13 260-279 49 49 49 95 95 95 F F F 14 280-299 57 57 57 108 108 108 F F F 15 300-319 55 55 55 105 105 105 F F F 16 320-339 50 50 50 98 98 98 F F F 17 340-359 53 53 53 102 102 102 F F F 18 360-379 57 57 57 108 108 108 F F F 19 380-399 57 57 57 108 108 108 F F F 20 400-419 57 57 57 108 108 108 F F F 21 420-439 56 56 56 107 107 107 F F F 22 440-459 56 56 56 107 107 107 F F F 23 460-479 53 53 53 102 102 102 F F F 24 480-499 57 57 57 108 108 108 F F F 25 500-519 48 48 48 92 92 92 F F F 26 520-539 46 46 46 90 90 90 F F F 27 540-559 48 48 48 94 94 94 F F F 28 560-579 55 55 55 105 105 105 F F F 29 580-599 55 55 55 105 105 105 F F F Max gen: 57, Max flush: 108 Begin loop 29 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:33.999581269Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1412,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:34.043891653Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1412,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:34.044491163Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1412} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"0fd98d0d5d909611f8b408ccda2559d35fe117bb\",\n git_commit_timestamp: \"2024-11-08T20:05:44.000000000Z\",\n git_branch: \"mkeeter/drop-skipped-backpressure-guard\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:34.044530432Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1412} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:34.044545001Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1412} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:34.04455944Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1412} {"msg":"Crucible 3f93d348-97e6-4ee4-8898-b40361c09e1f has session id: d760a1e0-ccf9-405c-9836-0eb2d7bd74ff","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:34.04497645Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1412,"session_id":"d760a1e0-ccf9-405c-9836-0eb2d7bd74ff"} {"msg":"Upstairs opts: Upstairs UUID: 3f93d348-97e6-4ee4-8898-b40361c09e1f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:34.045022168Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1412,"session_id":"d760a1e0-ccf9-405c-9836-0eb2d7bd74ff"} {"msg":"Crucible stats registered with UUID: 3f93d348-97e6-4ee4-8898-b40361c09e1f","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:34.045046016Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1412,"session_id":"d760a1e0-ccf9-405c-9836-0eb2d7bd74ff"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:34.07627632Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1412,"task":"crutest"} {"msg":"The guest has requested activation with gen:58","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:34.076310859Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1412} {"msg":"3f93d348-97e6-4ee4-8898-b40361c09e1f active request set","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:34.076346207Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1412,"session_id":"d760a1e0-ccf9-405c-9836-0eb2d7bd74ff"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:34.076363936Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1412,"client":"0","":"downstairs","session_id":"d760a1e0-ccf9-405c-9836-0eb2d7bd74ff"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:34.076391904Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1412,"client":"0","":"downstairs","session_id":"d760a1e0-ccf9-405c-9836-0eb2d7bd74ff"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:34.076419233Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1412,"client":"1","":"downstairs","session_id":"d760a1e0-ccf9-405c-9836-0eb2d7bd74ff"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:34.076452261Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1412,"client":"1","":"downstairs","session_id":"d760a1e0-ccf9-405c-9836-0eb2d7bd74ff"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:34.07648691Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1412,"client":"2","":"downstairs","session_id":"d760a1e0-ccf9-405c-9836-0eb2d7bd74ff"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:34.076522238Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1412,"client":"2","":"downstairs","session_id":"d760a1e0-ccf9-405c-9836-0eb2d7bd74ff"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:34.076557476Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1412,"":"io task","client":"2","":"downstairs","session_id":"d760a1e0-ccf9-405c-9836-0eb2d7bd74ff"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:34.076593325Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1412,"":"io task","client":"0","":"downstairs","session_id":"d760a1e0-ccf9-405c-9836-0eb2d7bd74ff"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:34.076629483Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1412,"":"io task","client":"1","":"downstairs","session_id":"d760a1e0-ccf9-405c-9836-0eb2d7bd74ff"} {"msg":"ds_connection connected from Ok(127.0.0.1:35979)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:34.076663821Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1412,"":"io task","client":"1","":"downstairs","session_id":"d760a1e0-ccf9-405c-9836-0eb2d7bd74ff"} {"msg":"ds_connection connected from Ok(127.0.0.1:35149)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:34.07669638Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1412,"":"io task","client":"0","":"downstairs","session_id":"d760a1e0-ccf9-405c-9836-0eb2d7bd74ff"} {"msg":"ds_connection connected from Ok(127.0.0.1:44390)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:34.076845192Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1412,"":"io task","client":"2","":"downstairs","session_id":"d760a1e0-ccf9-405c-9836-0eb2d7bd74ff"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:34.077370087Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1412,"client":"0","":"downstairs","session_id":"d760a1e0-ccf9-405c-9836-0eb2d7bd74ff"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:34.077397855Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1412,"client":"0","":"downstairs","session_id":"d760a1e0-ccf9-405c-9836-0eb2d7bd74ff"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:34.077537548Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1412,"client":"1","":"downstairs","session_id":"d760a1e0-ccf9-405c-9836-0eb2d7bd74ff"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:34.077565247Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1412,"client":"1","":"downstairs","session_id":"d760a1e0-ccf9-405c-9836-0eb2d7bd74ff"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:34.077594906Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1412,"client":"2","":"downstairs","session_id":"d760a1e0-ccf9-405c-9836-0eb2d7bd74ff"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:34.077611525Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1412,"client":"2","":"downstairs","session_id":"d760a1e0-ccf9-405c-9836-0eb2d7bd74ff"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:34.078105021Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1412,"client":"0","":"downstairs","session_id":"d760a1e0-ccf9-405c-9836-0eb2d7bd74ff"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:34.078147719Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1412,"client":"2","":"downstairs","session_id":"d760a1e0-ccf9-405c-9836-0eb2d7bd74ff"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:34.078220675Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1412,"client":"1","":"downstairs","session_id":"d760a1e0-ccf9-405c-9836-0eb2d7bd74ff"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:34.078689462Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1412,"client":"0","":"downstairs","session_id":"d760a1e0-ccf9-405c-9836-0eb2d7bd74ff"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:34.078718351Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1412,"client":"0","":"downstairs","session_id":"d760a1e0-ccf9-405c-9836-0eb2d7bd74ff"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:34.07873312Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1412,"session_id":"d760a1e0-ccf9-405c-9836-0eb2d7bd74ff"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:34.078746009Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1412,"client":"2","":"downstairs","session_id":"d760a1e0-ccf9-405c-9836-0eb2d7bd74ff"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:34.078759869Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1412,"client":"2","":"downstairs","session_id":"d760a1e0-ccf9-405c-9836-0eb2d7bd74ff"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:34.078773468Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1412,"session_id":"d760a1e0-ccf9-405c-9836-0eb2d7bd74ff"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:34.078785927Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1412,"client":"1","":"downstairs","session_id":"d760a1e0-ccf9-405c-9836-0eb2d7bd74ff"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:34.078799197Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1412,"client":"1","":"downstairs","session_id":"d760a1e0-ccf9-405c-9836-0eb2d7bd74ff"} {"msg":"[0]R flush_numbers[0..12]: [105, 108, 106, 107, 105, 106, 105, 108, 102, 102, 107, 105]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:34.078813216Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1412,"":"downstairs","":"downstairs","session_id":"d760a1e0-ccf9-405c-9836-0eb2d7bd74ff"} {"msg":"[0]R generation[0..12]: [55, 57, 56, 56, 55, 56, 55, 57, 53, 53, 56, 55]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:34.078830125Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1412,"":"downstairs","":"downstairs","session_id":"d760a1e0-ccf9-405c-9836-0eb2d7bd74ff"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:34.078844685Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1412,"":"downstairs","":"downstairs","session_id":"d760a1e0-ccf9-405c-9836-0eb2d7bd74ff"} {"msg":"[1]R flush_numbers[0..12]: [105, 108, 106, 107, 105, 106, 105, 108, 102, 102, 107, 105]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:34.078858814Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1412,"":"downstairs","":"downstairs","session_id":"d760a1e0-ccf9-405c-9836-0eb2d7bd74ff"} {"msg":"[1]R generation[0..12]: [55, 57, 56, 56, 55, 56, 55, 57, 53, 53, 56, 55]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:34.078872613Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1412,"":"downstairs","":"downstairs","session_id":"d760a1e0-ccf9-405c-9836-0eb2d7bd74ff"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:34.078886762Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1412,"":"downstairs","":"downstairs","session_id":"d760a1e0-ccf9-405c-9836-0eb2d7bd74ff"} {"msg":"[2]R flush_numbers[0..12]: [105, 108, 106, 107, 105, 106, 105, 108, 102, 102, 107, 105]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:34.078900542Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1412,"":"downstairs","":"downstairs","session_id":"d760a1e0-ccf9-405c-9836-0eb2d7bd74ff"} {"msg":"[2]R generation[0..12]: [55, 57, 56, 56, 55, 56, 55, 57, 53, 53, 56, 55]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:34.078914161Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1412,"":"downstairs","":"downstairs","session_id":"d760a1e0-ccf9-405c-9836-0eb2d7bd74ff"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:34.078928011Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1412,"":"downstairs","":"downstairs","session_id":"d760a1e0-ccf9-405c-9836-0eb2d7bd74ff"} {"msg":"Max found gen is 58","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:34.0789409Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1412,"":"downstairs","":"downstairs","session_id":"d760a1e0-ccf9-405c-9836-0eb2d7bd74ff"} {"msg":"Generation requested: 58 >= found:58","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:34.078954509Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1412,"":"downstairs","":"downstairs","session_id":"d760a1e0-ccf9-405c-9836-0eb2d7bd74ff"} {"msg":"Next flush: 109","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:34.078967469Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1412,"":"downstairs","":"downstairs","session_id":"d760a1e0-ccf9-405c-9836-0eb2d7bd74ff"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:34.078980698Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1412,"":"downstairs","":"downstairs","session_id":"d760a1e0-ccf9-405c-9836-0eb2d7bd74ff"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-08T20:24:34.078993937Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1412,"session_id":"d760a1e0-ccf9-405c-9836-0eb2d7bd74ff"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:34.079011176Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1412,"client":"0","":"downstairs","session_id":"d760a1e0-ccf9-405c-9836-0eb2d7bd74ff"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:34.079026166Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1412,"client":"1","":"downstairs","session_id":"d760a1e0-ccf9-405c-9836-0eb2d7bd74ff"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:34.079039385Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1412,"client":"2","":"downstairs","session_id":"d760a1e0-ccf9-405c-9836-0eb2d7bd74ff"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:34.079052674Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1412,"session_id":"d760a1e0-ccf9-405c-9836-0eb2d7bd74ff"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:34.079065124Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1412,"session_id":"d760a1e0-ccf9-405c-9836-0eb2d7bd74ff"} {"msg":"3f93d348-97e6-4ee4-8898-b40361c09e1f is now active with session: d760a1e0-ccf9-405c-9836-0eb2d7bd74ff","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:34.079077893Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1412,"session_id":"d760a1e0-ccf9-405c-9836-0eb2d7bd74ff"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:34.079090073Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1412,"session_id":"d760a1e0-ccf9-405c-9836-0eb2d7bd74ff"} {"msg":"The guest has finished waiting for activation with:58","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:34.079102342Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1412} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 544 len 12288 data: 7 5 5 02/30 Write block 201 len 8192 data: 3 3 03/30 Write block 155 len 32768 data: 4 3 3 3 4 3 3 2 04/30 Read block 103 len 16384 05/30 Read block 147 len 12288 06/30 Write block 15 len 8192 data: 6 6 07/30 Read block 528 len 4096 08/30 Flush 09/30 Write block 470 len 40960 data: 6 5 5 6 6 6 6 6 6 6 10/30 Read block 547 len 28672 11/30 Read block 540 len 20480 12/30 Write block 164 len 36864 data: 3 3 3 2 6 6 7 6 4 13/30 Write block 170 len 8192 data: 8 7 14/30 Write block 103 len 36864 data: 5 5 4 5 8 7 7 5 4 15/30 Read block 559 len 24576 16/30 Write block 247 len 36864 data: 8 9 10 11 9 10 8 8 6 17/30 Flush 18/30 Write block 311 len 24576 data: 6 6 7 4 5 5 19/30 Read block 150 len 36864 20/30 Read block 380 len 20480 21/30 Read block 527 len 20480 22/30 Read block 126 len 40960 23/30 Write block 76 len 40960 data: 9 8 6 5 4 3 4 5 4 6 24/30 Read block 316 len 32768 25/30 Write block 308 len 36864 data: 4 5 6 7 7 8 5 6 6 26/30 Read block 232 len 12288 27/30 Write block 469 len 16384 data: 7 7 6 6 28/30 Read block 315 len 8192 29/30 Read block 323 len 32768 30/30 Read block 28 len 8192 ---------------------------------------------------------------- Crucible gen:58 GIO:true work queues: Upstairs:0 downstairs:13 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1017 Acked Write 6 Done Done Done false 1018 Acked Read 9 Done Done Done false 1019 Acked Read 5 Done Done Done false 1020 Acked Read 5 Done Done Done false 1021 Acked Read 10 Done Done Done false 1022 Acked Write 10 Done Done Done false 1023 Acked Read 8 Done Done Done false 1024 Acked Write 9 Done Done Done false 1025 Acked Read 3 Done Done Done false 1026 Acked Write 4 Done Done Done false 1027 Acked Read 2 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Read 2 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 13 13 13 39 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1016 1016 1016 Downstairs last five completed: 1016 1015 1014 1013 1012 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:24:34.103859452Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1412} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 58 58 58 109 109 109 F F F 1 020-039 57 57 57 108 108 108 F F F 2 040-059 56 56 56 106 106 106 F F F 3 060-079 56 56 56 107 107 107 T T T 4 080-099 55 55 55 105 105 105 T T T 5 100-119 58 58 58 110 110 110 F F F 6 120-139 55 55 55 105 105 105 F F F 7 140-159 58 58 58 109 109 109 F F F 8 160-179 58 58 58 110 110 110 F F F 9 180-199 53 53 53 102 102 102 F F F 10 200-219 58 58 58 109 109 109 F F F 11 220-239 55 55 55 105 105 105 F F F 12 240-259 58 58 58 110 110 110 F F F 13 260-279 49 49 49 95 95 95 F F F 14 280-299 57 57 57 108 108 108 F F F 15 300-319 55 55 55 105 105 105 T T T 16 320-339 50 50 50 98 98 98 F F F 17 340-359 53 53 53 102 102 102 F F F 18 360-379 57 57 57 108 108 108 F F F 19 380-399 57 57 57 108 108 108 F F F 20 400-419 57 57 57 108 108 108 F F F 21 420-439 56 56 56 107 107 107 F F F 22 440-459 56 56 56 107 107 107 F F F 23 460-479 58 58 58 110 110 110 T T T 24 480-499 57 57 57 108 108 108 F F F 25 500-519 48 48 48 92 92 92 F F F 26 520-539 46 46 46 90 90 90 F F F 27 540-559 58 58 58 109 109 109 F F F 28 560-579 55 55 55 105 105 105 F F F 29 580-599 55 55 55 105 105 105 F F F Max gen: 58, Max flush: 110 A Difference in extent metadata is expected here On loop 29 Verifying data now No change after verify Loop: 29 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 58 58 58 109 109 109 F F F 1 020-039 57 57 57 108 108 108 F F F 2 040-059 56 56 56 106 106 106 F F F 3 060-079 59 59 59 111 111 111 F F F 4 080-099 59 59 59 111 111 111 F F F 5 100-119 58 58 58 110 110 110 F F F 6 120-139 55 55 55 105 105 105 F F F 7 140-159 58 58 58 109 109 109 F F F 8 160-179 58 58 58 110 110 110 F F F 9 180-199 53 53 53 102 102 102 F F F 10 200-219 58 58 58 109 109 109 F F F 11 220-239 55 55 55 105 105 105 F F F 12 240-259 58 58 58 110 110 110 F F F 13 260-279 49 49 49 95 95 95 F F F 14 280-299 57 57 57 108 108 108 F F F 15 300-319 59 59 59 111 111 111 F F F 16 320-339 50 50 50 98 98 98 F F F 17 340-359 53 53 53 102 102 102 F F F 18 360-379 57 57 57 108 108 108 F F F 19 380-399 57 57 57 108 108 108 F F F 20 400-419 57 57 57 108 108 108 F F F 21 420-439 56 56 56 107 107 107 F F F 22 440-459 56 56 56 107 107 107 F F F 23 460-479 59 59 59 111 111 111 F F F 24 480-499 57 57 57 108 108 108 F F F 25 500-519 48 48 48 92 92 92 F F F 26 520-539 46 46 46 90 90 90 F F F 27 540-559 58 58 58 109 109 109 F F F 28 560-579 55 55 55 105 105 105 F F F 29 580-599 55 55 55 105 105 105 F F F Max gen: 59, Max flush: 111 Begin loop 30 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:38.404793587Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1420,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:38.44993443Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1420,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:38.450653125Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1420} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"0fd98d0d5d909611f8b408ccda2559d35fe117bb\",\n git_commit_timestamp: \"2024-11-08T20:05:44.000000000Z\",\n git_branch: \"mkeeter/drop-skipped-backpressure-guard\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:38.450697093Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1420} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:38.450714552Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1420} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:38.450732421Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1420} {"msg":"Crucible 6ecce326-3d0c-4fa2-9ab6-1d9fc8774062 has session id: 5c205b9f-e1f9-4ae2-b6b4-e6224589017d","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:38.451199319Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1420,"session_id":"5c205b9f-e1f9-4ae2-b6b4-e6224589017d"} {"msg":"Upstairs opts: Upstairs UUID: 6ecce326-3d0c-4fa2-9ab6-1d9fc8774062, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:38.451254696Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1420,"session_id":"5c205b9f-e1f9-4ae2-b6b4-e6224589017d"} {"msg":"Crucible stats registered with UUID: 6ecce326-3d0c-4fa2-9ab6-1d9fc8774062","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:38.451284014Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1420,"session_id":"5c205b9f-e1f9-4ae2-b6b4-e6224589017d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:38.483026303Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1420,"task":"crutest"} {"msg":"The guest has requested activation with gen:60","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:38.483060381Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1420} {"msg":"6ecce326-3d0c-4fa2-9ab6-1d9fc8774062 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:38.483112399Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1420,"session_id":"5c205b9f-e1f9-4ae2-b6b4-e6224589017d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:38.483132558Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1420,"client":"0","":"downstairs","session_id":"5c205b9f-e1f9-4ae2-b6b4-e6224589017d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:38.483164296Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1420,"client":"0","":"downstairs","session_id":"5c205b9f-e1f9-4ae2-b6b4-e6224589017d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:38.483182725Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1420,"client":"1","":"downstairs","session_id":"5c205b9f-e1f9-4ae2-b6b4-e6224589017d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:38.483209854Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1420,"client":"1","":"downstairs","session_id":"5c205b9f-e1f9-4ae2-b6b4-e6224589017d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:38.483242412Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1420,"client":"2","":"downstairs","session_id":"5c205b9f-e1f9-4ae2-b6b4-e6224589017d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:38.483275091Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1420,"client":"2","":"downstairs","session_id":"5c205b9f-e1f9-4ae2-b6b4-e6224589017d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:38.48330177Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1420,"":"io task","client":"2","":"downstairs","session_id":"5c205b9f-e1f9-4ae2-b6b4-e6224589017d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:38.483333088Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1420,"":"io task","client":"0","":"downstairs","session_id":"5c205b9f-e1f9-4ae2-b6b4-e6224589017d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:38.483365816Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1420,"":"io task","client":"1","":"downstairs","session_id":"5c205b9f-e1f9-4ae2-b6b4-e6224589017d"} {"msg":"ds_connection connected from Ok(127.0.0.1:39931)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:38.483425724Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1420,"":"io task","client":"2","":"downstairs","session_id":"5c205b9f-e1f9-4ae2-b6b4-e6224589017d"} {"msg":"ds_connection connected from Ok(127.0.0.1:58644)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:38.483463082Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1420,"":"io task","client":"1","":"downstairs","session_id":"5c205b9f-e1f9-4ae2-b6b4-e6224589017d"} {"msg":"ds_connection connected from Ok(127.0.0.1:51053)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:38.4834976Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1420,"":"io task","client":"0","":"downstairs","session_id":"5c205b9f-e1f9-4ae2-b6b4-e6224589017d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:38.484238154Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1420,"client":"1","":"downstairs","session_id":"5c205b9f-e1f9-4ae2-b6b4-e6224589017d"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:38.484265982Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1420,"client":"1","":"downstairs","session_id":"5c205b9f-e1f9-4ae2-b6b4-e6224589017d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:38.484280572Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1420,"client":"2","":"downstairs","session_id":"5c205b9f-e1f9-4ae2-b6b4-e6224589017d"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:38.484294211Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1420,"client":"2","":"downstairs","session_id":"5c205b9f-e1f9-4ae2-b6b4-e6224589017d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:38.48430775Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1420,"client":"0","":"downstairs","session_id":"5c205b9f-e1f9-4ae2-b6b4-e6224589017d"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:38.48432388Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1420,"client":"0","":"downstairs","session_id":"5c205b9f-e1f9-4ae2-b6b4-e6224589017d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:38.484881982Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1420,"client":"0","":"downstairs","session_id":"5c205b9f-e1f9-4ae2-b6b4-e6224589017d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:38.485046474Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1420,"client":"1","":"downstairs","session_id":"5c205b9f-e1f9-4ae2-b6b4-e6224589017d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:38.485086962Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1420,"client":"2","":"downstairs","session_id":"5c205b9f-e1f9-4ae2-b6b4-e6224589017d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:38.485473963Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1420,"client":"0","":"downstairs","session_id":"5c205b9f-e1f9-4ae2-b6b4-e6224589017d"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:38.485510682Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1420,"client":"0","":"downstairs","session_id":"5c205b9f-e1f9-4ae2-b6b4-e6224589017d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:38.485528151Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1420,"session_id":"5c205b9f-e1f9-4ae2-b6b4-e6224589017d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:38.485604097Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1420,"client":"1","":"downstairs","session_id":"5c205b9f-e1f9-4ae2-b6b4-e6224589017d"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:38.485623586Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1420,"client":"1","":"downstairs","session_id":"5c205b9f-e1f9-4ae2-b6b4-e6224589017d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:38.485637455Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1420,"session_id":"5c205b9f-e1f9-4ae2-b6b4-e6224589017d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:38.485697093Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1420,"client":"2","":"downstairs","session_id":"5c205b9f-e1f9-4ae2-b6b4-e6224589017d"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:38.485715841Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1420,"client":"2","":"downstairs","session_id":"5c205b9f-e1f9-4ae2-b6b4-e6224589017d"} {"msg":"[0]R flush_numbers[0..12]: [109, 108, 106, 111, 111, 110, 105, 109, 110, 102, 109, 105]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:38.485730081Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1420,"":"downstairs","":"downstairs","session_id":"5c205b9f-e1f9-4ae2-b6b4-e6224589017d"} {"msg":"[0]R generation[0..12]: [58, 57, 56, 59, 59, 58, 55, 58, 58, 53, 58, 55]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:38.48574398Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1420,"":"downstairs","":"downstairs","session_id":"5c205b9f-e1f9-4ae2-b6b4-e6224589017d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:38.48575791Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1420,"":"downstairs","":"downstairs","session_id":"5c205b9f-e1f9-4ae2-b6b4-e6224589017d"} {"msg":"[1]R flush_numbers[0..12]: [109, 108, 106, 111, 111, 110, 105, 109, 110, 102, 109, 105]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:38.485774659Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1420,"":"downstairs","":"downstairs","session_id":"5c205b9f-e1f9-4ae2-b6b4-e6224589017d"} {"msg":"[1]R generation[0..12]: [58, 57, 56, 59, 59, 58, 55, 58, 58, 53, 58, 55]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:38.485788588Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1420,"":"downstairs","":"downstairs","session_id":"5c205b9f-e1f9-4ae2-b6b4-e6224589017d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:38.485802597Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1420,"":"downstairs","":"downstairs","session_id":"5c205b9f-e1f9-4ae2-b6b4-e6224589017d"} {"msg":"[2]R flush_numbers[0..12]: [109, 108, 106, 111, 111, 110, 105, 109, 110, 102, 109, 105]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:38.485816327Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1420,"":"downstairs","":"downstairs","session_id":"5c205b9f-e1f9-4ae2-b6b4-e6224589017d"} {"msg":"[2]R generation[0..12]: [58, 57, 56, 59, 59, 58, 55, 58, 58, 53, 58, 55]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:38.485829846Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1420,"":"downstairs","":"downstairs","session_id":"5c205b9f-e1f9-4ae2-b6b4-e6224589017d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:38.485843665Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1420,"":"downstairs","":"downstairs","session_id":"5c205b9f-e1f9-4ae2-b6b4-e6224589017d"} {"msg":"Max found gen is 60","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:38.485856595Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1420,"":"downstairs","":"downstairs","session_id":"5c205b9f-e1f9-4ae2-b6b4-e6224589017d"} {"msg":"Generation requested: 60 >= found:60","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:38.485869714Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1420,"":"downstairs","":"downstairs","session_id":"5c205b9f-e1f9-4ae2-b6b4-e6224589017d"} {"msg":"Next flush: 112","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:38.485882593Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1420,"":"downstairs","":"downstairs","session_id":"5c205b9f-e1f9-4ae2-b6b4-e6224589017d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:38.485895463Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1420,"":"downstairs","":"downstairs","session_id":"5c205b9f-e1f9-4ae2-b6b4-e6224589017d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:38.485908612Z","hostname":Wait for a query_work_queue command to finish before sending IO "w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1420,"session_id":"5c205b9f-e1f9-4ae2-b6b4-e6224589017d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:38.485929421Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1420,"client":"0","":"downstairs","session_id":"5c205b9f-e1f9-4ae2-b6b4-e6224589017d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:38.48594319Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1420,"client":"1","":"downstairs","session_id":"5c205b9f-e1f9-4ae2-b6b4-e6224589017d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:38.48595636Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1420,"client":"2","":"downstairs","session_id":"5c205b9f-e1f9-4ae2-b6b4-e6224589017d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:38.485969669Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1420,"session_id":"5c205b9f-e1f9-4ae2-b6b4-e6224589017d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:38.485983528Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1420,"session_id":"5c205b9f-e1f9-4ae2-b6b4-e6224589017d"} {"msg":"6ecce326-3d0c-4fa2-9ab6-1d9fc8774062 is now active with session: 5c205b9f-e1f9-4ae2-b6b4-e6224589017d","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:38.486011407Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1420,"session_id":"5c205b9f-e1f9-4ae2-b6b4-e6224589017d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:38.486025636Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1420,"session_id":"5c205b9f-e1f9-4ae2-b6b4-e6224589017d"} {"msg":"The guest has finished waiting for activation with:60","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:38.486038026Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1420} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 73 len 24576 data: 5 8 8 10 9 7 02/30 Write block 572 len 8192 data: 8 8 03/30 Read block 277 len 4096 04/30 Flush 05/30 Write block 377 len 28672 data: 5 6 7 6 5 5 5 06/30 Write block 109 len 16384 data: 8 6 5 4 07/30 Read block 374 len 32768 08/30 Flush 09/30 Write block 193 len 32768 data: 5 5 4 4 4 5 4 4 10/30 Write block 353 len 20480 data: 9 8 11 11 11 11/30 Read block 372 len 8192 12/30 Flush 13/30 Write block 481 len 32768 data: 5 5 6 6 4 5 5 6 14/30 Read block 453 len 32768 15/30 Write block 213 len 32768 data: 6 5 4 5 4 6 5 4 16/30 Write block 303 len 16384 data: 4 3 2 3 17/30 Write block 565 len 8192 data: 3 3 18/30 Read block 367 len 24576 19/30 Write block 269 len 8192 data: 6 6 20/30 Read block 493 len 12288 21/30 Write block 479 len 32768 data: 7 5 6 6 7 7 5 6 22/30 Write block 284 len 20480 data: 5 6 9 9 10 23/30 Read block 146 len 24576 24/30 Read block 426 len 24576 25/30 Write block 119 len 20480 data: 6 6 5 6 7 26/30 Read block 323 len 32768 27/30 Write block 190 len 32768 data: 8 5 6 6 6 5 5 5 28/30 Read block 225 len 28672 29/30 Write block 54 len 16384 data: 6 6 5 4 30/30 Write block 197 len 28672 data: 6 6 5 5 4 4 4 ---------------------------------------------------------------- Crucible gen:60 GIO:true work queues: Upstairs:0 downstairs:18 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1012 Acked Write 8 Done Done Done false 1013 Acked Read 8 Done Done Done false 1014 Acked Write 8 Done Done Done false 1015 Acked Write 4 Done Done Done false 1016 Acked Write 2 Done Done Done false 1017 Acked Read 6 Done Done Done false 1018 Acked Write 2 Done Done Done false 1019 Acked Read 3 Done Done Done false 1020 Acked Write 8 Done Done Done false 1021 Acked Write 5 Done Done Done false 1022 Acked Read 6 Done Done Done false 1023 Acked Read 6 Done Done Done false 1024 Acked Write 5 Done Done Done false 1025 Acked Read 8 Done Done Done false 1026 Acked Write 8 Done Done Done false 1027 Acked Read 7 Done Done Done false 1028 Acked Write 4 Done Done Done false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 17 17 17 51 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1011 1011 1011 Downstairs last five completed: 1011 1010 1009 1008 1007 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:24:38.510838244Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1420} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 58 58 58 109 109 109 F F F 1 020-039 57 57 57 108 108 108 F F F 2 040-059 56 56 56 106 106 106 T T T 3 060-079 60 60 60 112 112 112 F F F 4 080-099 59 59 59 111 111 111 F F F 5 100-119 60 60 60 113 113 113 T T T 6 120-139 55 55 55 105 105 105 T T T 7 140-159 58 58 58 109 109 109 F F F 8 160-179 58 58 58 110 110 110 F F F 9 180-199 60 60 60 114 114 114 T T T 10 200-219 60 60 60 114 114 114 T T T 11 220-239 55 55 55 105 105 105 T T T 12 240-259 58 58 58 110 110 110 F F F 13 260-279 49 49 49 95 95 95 T T T 14 280-299 57 57 57 108 108 108 T T T 15 300-319 59 59 59 111 111 111 T T T 16 320-339 50 50 50 98 98 98 F F F 17 340-359 60 60 60 114 114 114 F F F 18 360-379 60 60 60 113 113 113 F F F 19 380-399 60 60 60 113 113 113 F F F 20 400-419 57 57 57 108 108 108 F F F 21 420-439 56 56 56 107 107 107 F F F 22 440-459 56 56 56 107 107 107 F F F 23 460-479 59 59 59 111 111 111 T T T 24 480-499 57 57 57 108 108 108 T T T 25 500-519 48 48 48 92 92 92 F F F 26 520-539 46 46 46 90 90 90 F F F 27 540-559 58 58 58 109 109 109 F F F 28 560-579 60 60 60 112 112 112 T T T 29 580-599 55 55 55 105 105 105 F F F Max gen: 60, Max flush: 114 A Difference in extent metadata is expected here On loop 30 Verifying data now No change after verify Loop: 30 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 58 58 58 109 109 109 F F F 1 020-039 57 57 57 108 108 108 F F F 2 040-059 61 61 61 115 115 115 F F F 3 060-079 60 60 60 112 112 112 F F F 4 080-099 59 59 59 111 111 111 F F F 5 100-119 61 61 61 115 115 115 F F F 6 120-139 61 61 61 115 115 115 F F F 7 140-159 58 58 58 109 109 109 F F F 8 160-179 58 58 58 110 110 110 F F F 9 180-199 61 61 61 115 115 115 F F F 10 200-219 61 61 61 115 115 115 F F F 11 220-239 61 61 61 115 115 115 F F F 12 240-259 58 58 58 110 110 110 F F F 13 260-279 61 61 61 115 115 115 F F F 14 280-299 61 61 61 115 115 115 F F F 15 300-319 61 61 61 115 115 115 F F F 16 320-339 50 50 50 98 98 98 F F F 17 340-359 60 60 60 114 114 114 F F F 18 360-379 60 60 60 113 113 113 F F F 19 380-399 60 60 60 113 113 113 F F F 20 400-419 57 57 57 108 108 108 F F F 21 420-439 56 56 56 107 107 107 F F F 22 440-459 56 56 56 107 107 107 F F F 23 460-479 61 61 61 115 115 115 F F F 24 480-499 61 61 61 115 115 115 F F F 25 500-519 48 48 48 92 92 92 F F F 26 520-539 46 46 46 90 90 90 F F F 27 540-559 58 58 58 109 109 109 F F F 28 560-579 61 61 61 115 115 115 F F F 29 580-599 55 55 55 105 105 105 F F F Max gen: 61, Max flush: 115 Begin loop 31 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:42.927837806Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1428,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:24:42.970617995Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1428,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:42.971012936Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1428,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:43.002944555Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1428,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:43.003480718Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1428} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"0fd98d0d5d909611f8b408ccda2559d35fe117bb\",\n git_commit_timestamp: \"2024-11-08T20:05:44.000000000Z\",\n git_branch: \"mkeeter/drop-skipped-backpressure-guard\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:43.003524287Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1428} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:43.003542055Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1428} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:43.003560325Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1428} {"msg":"Crucible 909b4463-368d-4d1d-b3e0-5d1b7a30fc21 has session id: f784f0ad-e834-49d5-95f9-59df0f41db65","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:43.00406253Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1428,"session_id":"f784f0ad-e834-49d5-95f9-59df0f41db65"} {"msg":"Upstairs opts: Upstairs UUID: 909b4463-368d-4d1d-b3e0-5d1b7a30fc21, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:43.004104798Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1428,"session_id":"f784f0ad-e834-49d5-95f9-59df0f41db65"} {"msg":"Crucible stats registered with UUID: 909b4463-368d-4d1d-b3e0-5d1b7a30fc21","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:43.004124197Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1428,"session_id":"f784f0ad-e834-49d5-95f9-59df0f41db65"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:43.035474665Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1428,"task":"crutest"} {"msg":"The guest has requested activation with gen:62","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:43.035510223Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1428} {"msg":"909b4463-368d-4d1d-b3e0-5d1b7a30fc21 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:43.035544791Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1428,"session_id":"f784f0ad-e834-49d5-95f9-59df0f41db65"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:43.03557485Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1428,"client":"0","":"downstairs","session_id":"f784f0ad-e834-49d5-95f9-59df0f41db65"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:43.035620868Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1428,"client":"0","":"downstairs","session_id":"f784f0ad-e834-49d5-95f9-59df0f41db65"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:43.035657746Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1428,"client":"1","":"downstairs","session_id":"f784f0ad-e834-49d5-95f9-59df0f41db65"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:43.035694264Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1428,"client":"1","":"downstairs","session_id":"f784f0ad-e834-49d5-95f9-59df0f41db65"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:43.035727842Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1428,"client":"2","":"downstairs","session_id":"f784f0ad-e834-49d5-95f9-59df0f41db65"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:43.035762731Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1428,"client":"2","":"downstairs","session_id":"f784f0ad-e834-49d5-95f9-59df0f41db65"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:43.03579079Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1428,"":"io task","client":"2","":"downstairs","session_id":"f784f0ad-e834-49d5-95f9-59df0f41db65"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:43.035817748Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1428,"":"io task","client":"0","":"downstairs","session_id":"f784f0ad-e834-49d5-95f9-59df0f41db65"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:43.035851136Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1428,"":"io task","client":"1","":"downstairs","session_id":"f784f0ad-e834-49d5-95f9-59df0f41db65"} {"msg":"ds_connection connected from Ok(127.0.0.1:53088)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:43.035886925Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1428,"":"io task","client":"2","":"downstairs","session_id":"f784f0ad-e834-49d5-95f9-59df0f41db65"} {"msg":"ds_connection connected from Ok(127.0.0.1:39018)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:43.036020838Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1428,"":"io task","client":"0","":"downstairs","session_id":"f784f0ad-e834-49d5-95f9-59df0f41db65"} {"msg":"ds_connection connected from Ok(127.0.0.1:41639)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:43.036058026Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1428,"":"io task","client":"1","":"downstairs","session_id":"f784f0ad-e834-49d5-95f9-59df0f41db65"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:43.036551702Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1428,"client":"1","":"downstairs","session_id":"f784f0ad-e834-49d5-95f9-59df0f41db65"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:43.036590051Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1428,"client":"1","":"downstairs","session_id":"f784f0ad-e834-49d5-95f9-59df0f41db65"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:43.036620199Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1428,"client":"2","":"downstairs","session_id":"f784f0ad-e834-49d5-95f9-59df0f41db65"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:43.036651228Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1428,"client":"2","":"downstairs","session_id":"f784f0ad-e834-49d5-95f9-59df0f41db65"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:43.036680566Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1428,"client":"0","":"downstairs","session_id":"f784f0ad-e834-49d5-95f9-59df0f41db65"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:43.036695675Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1428,"client":"0","":"downstairs","session_id":"f784f0ad-e834-49d5-95f9-59df0f41db65"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:43.037221149Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1428,"client":"2","":"downstairs","session_id":"f784f0ad-e834-49d5-95f9-59df0f41db65"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:43.037254178Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1428,"client":"1","":"downstairs","session_id":"f784f0ad-e834-49d5-95f9-59df0f41db65"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:43.03742573Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1428,"client":"0","":"downstairs","session_id":"f784f0ad-e834-49d5-95f9-59df0f41db65"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:43.037726205Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1428,"client":"2","":"downstairs","session_id":"f784f0ad-e834-49d5-95f9-59df0f41db65"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:43.037754234Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1428,"client":"2","":"downstairs","session_id":"f784f0ad-e834-49d5-95f9-59df0f41db65"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:43.037769043Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1428,"session_id":"f784f0ad-e834-49d5-95f9-59df0f41db65"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:43.037796281Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1428,"client":"1","":"downstairs","session_id":"f784f0ad-e834-49d5-95f9-59df0f41db65"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:43.037810851Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1428,"client":"1","":"downstairs","session_id":"f784f0ad-e834-49d5-95f9-59df0f41db65"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:43.03782415Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1428,"session_id":"f784f0ad-e834-49d5-95f9-59df0f41db65"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:43.037977663Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1428,"client":"0","":"downstairs","session_id":"f784f0ad-e834-49d5-95f9-59df0f41db65"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:43.037997132Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1428,"client":"0","":"downstairs","session_id":"f784f0ad-e834-49d5-95f9-59df0f41db65"} {"msg":"[0]R flush_numbers[0..12]: [109, 108, 115, 112, 111, 115, 115, 109, 110, 115, 115, 115]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:43.03802463Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1428,"":"downstairs","":"downstairs","session_id":"f784f0ad-e834-49d5-95f9-59df0f41db65"} {"msg":"[0]R generation[0..12]: [58, 57, 61, 60, 59, 61, 61, 58, 58, 61, 61, 61]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:43.038039209Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1428,"":"downstairs","":"downstairs","session_id":"f784f0ad-e834-49d5-95f9-59df0f41db65"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:43.038056369Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1428,"":"downstairs","":"downstairs","session_id":"f784f0ad-e834-49d5-95f9-59df0f41db65"} {"msg":"[1]R flush_numbers[0..12]: [109, 108, 115, 112, 111, 115, 115, 109, 110, 115, 115, 115]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:43.038070588Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1428,"":"downstairs","":"downstairs","session_id":"f784f0ad-e834-49d5-95f9-59df0f41db65"} {"msg":"[1]R generation[0..12]: [58, 57, 61, 60, 59, 61, 61, 58, 58, 61, 61, 61]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:43.038084347Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1428,"":"downstairs","":"downstairs","session_id":"f784f0ad-e834-49d5-95f9-59df0f41db65"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:43.038098407Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1428,"":"downstairs","":"downstairs","session_id":"f784f0ad-e834-49d5-95f9-59df0f41db65"} {"msg":"[2]R flush_numbers[0..12]: [109, 108, 115, 112, 111, 115, 115, 109, 110, 115, 115, 115]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:43.038112196Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1428,"":"downstairs","":"downstairs","session_id":"f784f0ad-e834-49d5-95f9-59df0f41db65"} {"msg":"[2]R generation[0..12]: [58, 57, 61, 60, 59, 61, 61, 58, 58, 61, 61, 61]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:43.038125855Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1428,"":"downstairs","":"downstairs","session_id":"f784f0ad-e834-49d5-95f9-59df0f41db65"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:43.038139665Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1428,"":"downstairs","":"downstairs","session_id":"f784f0ad-e834-49d5-95f9-59df0f41db65"} {"msg":"Max found gen is 62","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:43.038152634Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1428,"":"downstairs","":"downstairs","session_id":"f784f0ad-e834-49d5-95f9-59df0f41db65"} {"msg":"Generation requested: 62 >= found:62","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:43.038165963Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1428,"":"downstairs","":"downstairs","session_id":"f784f0ad-e834-49d5-95f9-59df0f41db65"} {"msg":"Next flush: 116","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:43.038178993Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1428,"":"downstairs","":"downstairs","session_id":"f784f0ad-e834-49d5-95f9-59df0f41db65"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:43.038192102Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1428,"":"downstairs","":"downstairs","session_id":"f784f0ad-e834-49d5-95f9-59df0f41db65"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-08T20:24:43.038205172Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1428,"session_id":"f784f0ad-e834-49d5-95f9-59df0f41db65"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:43.038221461Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1428,"client":"0","":"downstairs","session_id":"f784f0ad-e834-49d5-95f9-59df0f41db65"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:43.03823459Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1428,"client":"1","":"downstairs","session_id":"f784f0ad-e834-49d5-95f9-59df0f41db65"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:43.038249249Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1428,"client":"2","":"downstairs","session_id":"f784f0ad-e834-49d5-95f9-59df0f41db65"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:43.038262949Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1428,"session_id":"f784f0ad-e834-49d5-95f9-59df0f41db65"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:43.038275368Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1428,"session_id":"f784f0ad-e834-49d5-95f9-59df0f41db65"} {"msg":"909b4463-368d-4d1d-b3e0-5d1b7a30fc21 is now active with session: f784f0ad-e834-49d5-95f9-59df0f41db65","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:43.038288288Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1428,"session_id":"f784f0ad-e834-49d5-95f9-59df0f41db65"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:43.038303917Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1428,"session_id":"f784f0ad-e834-49d5-95f9-59df0f41db65"} {"msg":"The guest has finished waiting for activation with:62","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:43.038326766Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1428} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 450 len 4096 data: 4 02/30 Flush 03/30 Flush 04/30 Write block 372 len 40960 data: 8 4 4 3 3 6 7 8 7 6 05/30 Write block 507 len 28672 data: 3 5 5 6 6 5 6 06/30 Write block 280 len 4096 data: 7 07/30 Read block 518 len 16384 08/30 Read block 489 len 24576 09/30 Write block 37 len 40960 data: 3 4 5 5 4 5 6 6 5 5 10/30 Read block 426 len 40960 11/30 Read block 506 len 8192 12/30 Read block 251 len 36864 13/30 Read block 170 len 24576 14/30 Flush 15/30 Write block 496 len 40960 data: 5 6 6 5 4 4 4 4 5 4 16/30 Read block 559 len 24576 17/30 Flush 18/30 Write block 371 len 32768 data: 7 9 5 5 4 4 7 8 19/30 Write block 196 len 28672 data: 6 7 7 6 6 5 5 20/30 Read block 381 len 16384 21/30 Write block 313 len 12288 data: 9 6 7 22/30 Write block 493 len 40960 data: 5 7 6 6 7 7 6 5 5 5 23/30 Write block 33 len 32768 data: 6 6 4 3 4 5 6 6 24/30 Write block 384 len 4096 data: 4 25/30 Read block 404 len 4096 26/30 Write block 235 len 8192 data: 6 6 27/30 Write block 563 len 28672 data: 2 3 4 4 4 5 5 28/30 Read block 517 len 32768 29/30 Read block 53 len 40960 30/30 Read block 359 len 28672 ---------------------------------------------------------------- Crucible gen:62 GIO:true work queues: Upstairs:0 downstairs:13 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1017 Acked Write 8 Done Done Done false 1018 Acked Write 7 Done Done Done false 1019 Acked Read 4 Done Done Done false 1020 Acked Write 3 Done Done Done false 1021 Acked Write 10 Done Done Done false 1022 Acked Write 8 Done Done Done false 1023 Acked Write 1 Done Done Done false 1024 Acked Read 1 Done Done Done false 1025 Acked Write 2 Done Done Done false 1026 Acked Write 7 Done Done Done false 1027 Acked Read 8 Done Done Done false 1028 Acked Read 10 Done Done Done false 1029 Acked Read 7 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 13 13 12 38 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1016 1016 1016 Downstairs last five completed: 1016 1015 1014 1013 1012 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:24:43.064176872Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1428} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 58 58 58 109 109 109 F F F 1 020-039 62 62 62 118 118 118 T T T 2 040-059 62 62 62 118 118 118 T T T 3 060-079 60 60 60 112 112 112 F F F 4 080-099 59 59 59 111 111 111 F F F 5 100-119 61 61 61 115 115 115 F F F 6 120-139 61 61 61 115 115 115 F F F 7 140-159 58 58 58 109 109 109 F F F 8 160-179 58 58 58 110 110 110 F F F 9 180-199 61 61 61 115 115 115 T T T 10 200-219 61 61 61 115 115 115 T T T 11 220-239 61 61 61 115 115 115 T T T 12 240-259 58 58 58 110 110 110 F F F 13 260-279 61 61 61 115 115 115 F F F 14 280-299 62 62 62 118 118 118 F F F 15 300-319 61 61 61 115 115 115 T T T 16 320-339 50 50 50 98 98 98 F F F 17 340-359 60 60 60 114 114 114 F F F 18 360-379 62 62 62 118 118 118 T T T 19 380-399 62 62 62 118 118 118 T T T 20 400-419 57 57 57 108 108 108 F F F 21 420-439 56 56 56 107 107 107 F F F 22 440-459 62 62 62 116 116 116 F F F 23 460-479 61 61 61 115 115 115 F F F 24 480-499 62 62 62 119 119 119 T T T 25 500-519 62 62 62 119 119 119 T T T 26 520-539 46 46 46 90 90 90 F F F 27 540-559 58 58 58 109 109 109 F F F 28 560-579 61 61 61 115 115 115 T T T 29 580-599 55 55 55 105 105 105 F F F Max gen: 62, Max flush: 119 A Difference in extent metadata is expected here On loop 31 Verifying data now No change after verify Loop: 31 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 58 58 58 109 109 109 F F F 1 020-039 63 63 63 120 120 120 F F F 2 040-059 63 63 63 120 120 120 F F F 3 060-079 60 60 60 112 112 112 F F F 4 080-099 59 59 59 111 111 111 F F F 5 100-119 61 61 61 115 115 115 F F F 6 120-139 61 61 61 115 115 115 F F F 7 140-159 58 58 58 109 109 109 F F F 8 160-179 58 58 58 110 110 110 F F F 9 180-199 63 63 63 120 120 120 F F F 10 200-219 63 63 63 120 120 120 F F F 11 220-239 63 63 63 120 120 120 F F F 12 240-259 58 58 58 110 110 110 F F F 13 260-279 61 61 61 115 115 115 F F F 14 280-299 62 62 62 118 118 118 F F F 15 300-319 63 63 63 120 120 120 F F F 16 320-339 50 50 50 98 98 98 F F F 17 340-359 60 60 60 114 114 114 F F F 18 360-379 63 63 63 120 120 120 F F F 19 380-399 63 63 63 120 120 120 F F F 20 400-419 57 57 57 108 108 108 F F F 21 420-439 56 56 56 107 107 107 F F F 22 440-459 62 62 62 116 116 116 F F F 23 460-479 61 61 61 115 115 115 F F F 24 480-499 63 63 63 120 120 120 F F F 25 500-519 63 63 63 120 120 120 F F F 26 520-539 46 46 46 90 90 90 F F F 27 540-559 58 58 58 109 109 109 F F F 28 560-579 63 63 63 120 120 120 F F F 29 580-599 55 55 55 105 105 105 F F F Max gen: 63, Max flush: 120 Begin loop 32 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:47.494849668Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1436,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:47.538960312Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1436,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:47.539593931Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1436} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"0fd98d0d5d909611f8b408ccda2559d35fe117bb\",\n git_commit_timestamp: \"2024-11-08T20:05:44.000000000Z\",\n git_branch: \"mkeeter/drop-skipped-backpressure-guard\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:47.539629939Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1436} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:47.539643989Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1436} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:47.539658148Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1436} {"msg":"Crucible 2af9ae5d-1a88-4547-ac11-3ceb02d32682 has session id: 4decb119-4b09-4c26-927f-b1073d97b19f","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:47.540072358Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1436,"session_id":"4decb119-4b09-4c26-927f-b1073d97b19f"} {"msg":"Upstairs opts: Upstairs UUID: 2af9ae5d-1a88-4547-ac11-3ceb02d32682, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:47.540118896Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1436,"session_id":"4decb119-4b09-4c26-927f-b1073d97b19f"} {"msg":"Crucible stats registered with UUID: 2af9ae5d-1a88-4547-ac11-3ceb02d32682","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:47.540144894Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1436,"session_id":"4decb119-4b09-4c26-927f-b1073d97b19f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:47.573070675Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1436,"task":"crutest"} {"msg":"The guest has requested activation with gen:64","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:47.573116983Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1436} {"msg":"2af9ae5d-1a88-4547-ac11-3ceb02d32682 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:47.573188369Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1436,"session_id":"4decb119-4b09-4c26-927f-b1073d97b19f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:47.573234257Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1436,"client":"0","":"downstairs","session_id":"4decb119-4b09-4c26-927f-b1073d97b19f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:47.573283635Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1436,"client":"0","":"downstairs","session_id":"4decb119-4b09-4c26-927f-b1073d97b19f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:47.573320583Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1436,"client":"1","":"downstairs","session_id":"4decb119-4b09-4c26-927f-b1073d97b19f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:47.573362581Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1436,"client":"1","":"downstairs","session_id":"4decb119-4b09-4c26-927f-b1073d97b19f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:47.573400579Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1436,"client":"2","":"downstairs","session_id":"4decb119-4b09-4c26-927f-b1073d97b19f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:47.573437867Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1436,"client":"2","":"downstairs","session_id":"4decb119-4b09-4c26-927f-b1073d97b19f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:47.573473545Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1436,"":"io task","client":"2","":"downstairs","session_id":"4decb119-4b09-4c26-927f-b1073d97b19f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:47.573509604Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1436,"":"io task","client":"0","":"downstairs","session_id":"4decb119-4b09-4c26-927f-b1073d97b19f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:47.573543622Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1436,"":"io task","client":"1","":"downstairs","session_id":"4decb119-4b09-4c26-927f-b1073d97b19f"} {"msg":"ds_connection connected from Ok(127.0.0.1:48759)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:47.573643647Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1436,"":"io task","client":"2","":"downstairs","session_id":"4decb119-4b09-4c26-927f-b1073d97b19f"} {"msg":"ds_connection connected from Ok(127.0.0.1:37501)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:47.573682195Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1436,"":"io task","client":"0","":"downstairs","session_id":"4decb119-4b09-4c26-927f-b1073d97b19f"} {"msg":"ds_connection connected from Ok(127.0.0.1:57853)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:47.573717664Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1436,"":"io task","client":"1","":"downstairs","session_id":"4decb119-4b09-4c26-927f-b1073d97b19f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:47.574234588Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1436,"client":"0","":"downstairs","session_id":"4decb119-4b09-4c26-927f-b1073d97b19f"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:47.574269157Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1436,"client":"0","":"downstairs","session_id":"4decb119-4b09-4c26-927f-b1073d97b19f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:47.574287746Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1436,"client":"1","":"downstairs","session_id":"4decb119-4b09-4c26-927f-b1073d97b19f"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:47.574304625Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1436,"client":"1","":"downstairs","session_id":"4decb119-4b09-4c26-927f-b1073d97b19f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:47.57440488Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1436,"client":"2","":"downstairs","session_id":"4decb119-4b09-4c26-927f-b1073d97b19f"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:47.574472857Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1436,"client":"2","":"downstairs","session_id":"4decb119-4b09-4c26-927f-b1073d97b19f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:47.574880817Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1436,"client":"1","":"downstairs","session_id":"4decb119-4b09-4c26-927f-b1073d97b19f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:47.574912925Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1436,"client":"0","":"downstairs","session_id":"4decb119-4b09-4c26-927f-b1073d97b19f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:47.5750091Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1436,"client":"2","":"downstairs","session_id":"4decb119-4b09-4c26-927f-b1073d97b19f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:47.575456999Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1436,"client":"0","":"downstairs","session_id":"4decb119-4b09-4c26-927f-b1073d97b19f"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:47.575488427Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1436,"client":"0","":"downstairs","session_id":"4decb119-4b09-4c26-927f-b1073d97b19f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:47.575503896Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1436,"session_id":"4decb119-4b09-4c26-927f-b1073d97b19f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:47.575560203Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1436,"client":"1","":"downstairs","session_id":"4decb119-4b09-4c26-927f-b1073d97b19f"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:47.575577933Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1436,"client":"1","":"downstairs","session_id":"4decb119-4b09-4c26-927f-b1073d97b19f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:47.575591892Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1436,"session_id":"4decb119-4b09-4c26-927f-b1073d97b19f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:47.575604151Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1436,"client":"2","":"downstairs","session_id":"4decb119-4b09-4c26-927f-b1073d97b19f"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:47.575617561Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1436,"client":"2","":"downstairs","session_id":"4decb119-4b09-4c26-927f-b1073d97b19f"} {"msg":"[0]R flush_numbers[0..12]: [109, 120, 120, 112, 111, 115, 115, 109, 110, 120, 120, 120]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:47.575655649Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1436,"":"downstairs","":"downstairs","session_id":"4decb119-4b09-4c26-927f-b1073d97b19f"} {"msg":"[0]R generation[0..12]: [58, 63, 63, 60, 59, 61, 61, 58, 58, 63, 63, 63]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:47.575672128Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1436,"":"downstairs","":"downstairs","session_id":"4decb119-4b09-4c26-927f-b1073d97b19f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:47.575686117Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1436,"":"downstairs","":"downstairs","session_id":"4decb119-4b09-4c26-927f-b1073d97b19f"} {"msg":"[1]R flush_numbers[0..12]: [109, 120, 120, 112, 111, 115, 115, 109, 110, 120, 120, 120]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:47.575699867Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1436,"":"downstairs","":"downstairs","session_id":"4decb119-4b09-4c26-927f-b1073d97b19f"} {"msg":"[1]R generation[0..12]: [58, 63, 63, 60, 59, 61, 61, 58, 58, 63, 63, 63]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:47.575716546Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1436,"":"downstairs","":"downstairs","session_id":"4decb119-4b09-4c26-927f-b1073d97b19f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:47.575730845Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1436,"":"downstairs","":"downstairs","session_id":"4decb119-4b09-4c26-927f-b1073d97b19f"} {"msg":"[2]R flush_numbers[0..12]: [109, 120, 120, 112, 111, 115, 115, 109, 110, 120, 120, 120]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:47.575744514Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1436,"":"downstairs","":"downstairs","session_id":"4decb119-4b09-4c26-927f-b1073d97b19f"} {"msg":"[2]R generation[0..12]: [58, 63, 63, 60, 59, 61, 61, 58, 58, 63, 63, 63]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:47.575758094Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1436,"":"downstairs","":"downstairs","session_id":"4decb119-4b09-4c26-927f-b1073d97b19f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:47.575771993Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1436,"":"downstairs","":"downstairs","session_id":"4decb119-4b09-4c26-927f-b1073d97b19f"} {"msg":"Max found gen is 64","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:47.575785052Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1436,"":"downstairs","":"downstairs","session_id":"4decb119-4b09-4c26-927f-b1073d97b19f"} {"msg":"Generation requested: 64 >= found:64","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:47.575799682Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1436,"":"downstairs","":"downstairs","session_id":"4decb119-4b09-4c26-927f-b1073d97b19f"} {"msg":"Next flush: 121","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:47.575812881Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1436,"":"downstairs","":"downstairs","session_id":"4decb119-4b09-4c26-927f-b1073d97b19f"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-08T20:24:47.575825901Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1436,"":"downstairs","":"downstairs","session_id":"4decb119-4b09-4c26-927f-b1073d97b19f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:47.575909426Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1436,"session_id":"4decb119-4b09-4c26-927f-b1073d97b19f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:47.575924986Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1436,"client":"0","":"downstairs","session_id":"4decb119-4b09-4c26-927f-b1073d97b19f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:47.575938355Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1436,"client":"1","":"downstairs","session_id":"4decb119-4b09-4c26-927f-b1073d97b19f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:47.575951634Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1436,"client":"2","":"downstairs","session_id":"4decb119-4b09-4c26-927f-b1073d97b19f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:47.575964744Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1436,"session_id":"4decb119-4b09-4c26-927f-b1073d97b19f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:47.575977133Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1436,"session_id":"4decb119-4b09-4c26-927f-b1073d97b19f"} {"msg":"2af9ae5d-1a88-4547-ac11-3ceb02d32682 is now active with session: 4decb119-4b09-4c26-927f-b1073d97b19f","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:47.575993752Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1436,"session_id":"4decb119-4b09-4c26-927f-b1073d97b19f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:47.57603786Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1436,"session_id":"4decb119-4b09-4c26-927f-b1073d97b19f"} {"msg":"The guest has finished waiting for activation with:64","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:47.576052859Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1436} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 490 len 12288 data: 5 4 3 02/30 Read block 368 len 28672 03/30 Read block 445 len 32768 04/30 Read block 443 len 4096 05/30 Read block 511 len 24576 06/30 Flush 07/30 Write block 293 len 8192 data: 7 7 08/30 Flush 09/30 Flush 10/30 Flush 11/30 Write block 120 len 16384 data: 7 6 7 8 12/30 Read block 507 len 40960 13/30 Read block 556 len 16384 14/30 Read block 137 len 24576 15/30 Read block 390 len 20480 16/30 Write block 346 len 4096 data: 9 17/30 Write block 396 len 24576 data: 5 7 7 8 6 5 18/30 Write block 88 len 32768 data: 6 5 4 4 4 4 2 3 19/30 Read block 206 len 28672 20/30 Flush 21/30 Write block 0 len 40960 data: 2 2 4 3 4 5 5 5 6 6 22/30 Read block 272 len 8192 23/30 Flush 24/30 Write block 190 len 8192 data: 9 6 25/30 Read block 493 len 24576 26/30 Read block 275 len 16384 27/30 Read block 216 len 20480 28/30 Read block 217 len 20480 29/30 Write block 259 len 4096 data: 5 30/30 Read block 17 len 40960 ---------------------------------------------------------------- Crucible gen:64 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 2 Done Done Done false 1024 Acked Read 6 Done Done Done false 1025 Acked Read 4 Done Done Done false 1026 Acked Read 5 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Write 1 Done Done Sent false 1029 Acked Read 10 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 2 3 Done 7 6 5 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:24:47.602322726Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1436} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 64 64 64 126 126 126 F F F 1 020-039 63 63 63 120 120 120 F F F 2 040-059 63 63 63 120 120 120 F F F 3 060-079 60 60 60 112 112 112 F F F 4 080-099 64 64 64 125 125 125 F F F 5 100-119 61 61 61 115 115 115 F F F 6 120-139 64 64 64 125 125 125 F F F 7 140-159 58 58 58 109 109 109 F F F 8 160-179 58 58 58 110 110 110 F F F 9 180-199 63 63 63 120 120 120 T T T 10 200-219 63 63 63 120 120 120 F F F 11 220-239 63 63 63 120 120 120 F F F 12 240-259 58 58 58 110 110 110 T T T 13 260-279 61 61 61 115 115 115 F F F 14 280-299 64 64 64 122 122 122 F F F 15 300-319 63 63 63 120 120 120 F F F 16 320-339 50 50 50 98 98 98 F F F 17 340-359 64 64 64 125 125 125 F F F 18 360-379 63 63 63 120 120 120 F F F 19 380-399 64 64 64 125 125 125 F F F 20 400-419 64 64 64 125 125 125 F F F 21 420-439 56 56 56 107 107 107 F F F 22 440-459 62 62 62 116 116 116 F F F 23 460-479 61 61 61 115 115 115 F F F 24 480-499 64 64 64 121 121 121 F F F 25 500-519 63 63 63 120 120 120 F F F 26 520-539 46 46 46 90 90 90 F F F 27 540-559 58 58 58 109 109 109 F F F 28 560-579 63 63 63 120 120 120 F F F 29 580-599 55 55 55 105 105 105 F F F Max gen: 64, Max flush: 126 A Difference in extent metadata is expected here On loop 32 Verifying data now No change after verify Loop: 32 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 64 64 64 126 126 126 F F F 1 020-039 63 63 63 120 120 120 F F F 2 040-059 63 63 63 120 120 120 F F F 3 060-079 60 60 60 112 112 112 F F F 4 080-099 64 64 64 125 125 125 F F F 5 100-119 61 61 61 115 115 115 F F F 6 120-139 64 64 64 125 125 125 F F F 7 140-159 58 58 58 109 109 109 F F F 8 160-179 58 58 58 110 110 110 F F F 9 180-199 65 65 65 127 127 127 F F F 10 200-219 63 63 63 120 120 120 F F F 11 220-239 63 63 63 120 120 120 F F F 12 240-259 65 65 65 127 127 127 F F F 13 260-279 61 61 61 115 115 115 F F F 14 280-299 64 64 64 122 122 122 F F F 15 300-319 63 63 63 120 120 120 F F F 16 320-339 50 50 50 98 98 98 F F F 17 340-359 64 64 64 125 125 125 F F F 18 360-379 63 63 63 120 120 120 F F F 19 380-399 64 64 64 125 125 125 F F F 20 400-419 64 64 64 125 125 125 F F F 21 420-439 56 56 56 107 107 107 F F F 22 440-459 62 62 62 116 116 116 F F F 23 460-479 61 61 61 115 115 115 F F F 24 480-499 64 64 64 121 121 121 F F F 25 500-519 63 63 63 120 120 120 F F F 26 520-539 46 46 46 90 90 90 F F F 27 540-559 58 58 58 109 109 109 F F F 28 560-579 63 63 63 120 120 120 F F F 29 580-599 55 55 55 105 105 105 F F F Max gen: 65, Max flush: 127 Begin loop 33 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:51.876442731Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1444,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:51.921771145Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1444,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:51.922367856Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1444} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"0fd98d0d5d909611f8b408ccda2559d35fe117bb\",\n git_commit_timestamp: \"2024-11-08T20:05:44.000000000Z\",\n git_branch: \"mkeeter/drop-skipped-backpressure-guard\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:51.922406354Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1444} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:51.922434963Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1444} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:51.922464142Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1444} {"msg":"Crucible a42653e1-c5f1-4043-b334-87201a3d1134 has session id: d0b385c1-65c3-4fba-984e-f9878b7da0ca","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:51.922870362Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1444,"session_id":"d0b385c1-65c3-4fba-984e-f9878b7da0ca"} {"msg":"Upstairs opts: Upstairs UUID: a42653e1-c5f1-4043-b334-87201a3d1134, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:51.922917059Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1444,"session_id":"d0b385c1-65c3-4fba-984e-f9878b7da0ca"} {"msg":"Crucible stats registered with UUID: a42653e1-c5f1-4043-b334-87201a3d1134","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:51.922940838Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1444,"session_id":"d0b385c1-65c3-4fba-984e-f9878b7da0ca"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:51.953865757Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1444,"task":"crutest"} {"msg":"The guest has requested activation with gen:66","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:51.953900745Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1444} {"msg":"a42653e1-c5f1-4043-b334-87201a3d1134 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:51.953980551Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1444,"session_id":"d0b385c1-65c3-4fba-984e-f9878b7da0ca"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:51.954020979Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1444,"client":"0","":"downstairs","session_id":"d0b385c1-65c3-4fba-984e-f9878b7da0ca"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:51.954048988Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1444,"client":"0","":"downstairs","session_id":"d0b385c1-65c3-4fba-984e-f9878b7da0ca"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:51.954063267Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1444,"client":"1","":"downstairs","session_id":"d0b385c1-65c3-4fba-984e-f9878b7da0ca"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:51.954077177Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1444,"client":"1","":"downstairs","session_id":"d0b385c1-65c3-4fba-984e-f9878b7da0ca"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:51.954090566Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1444,"client":"2","":"downstairs","session_id":"d0b385c1-65c3-4fba-984e-f9878b7da0ca"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:51.954103935Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1444,"client":"2","":"downstairs","session_id":"d0b385c1-65c3-4fba-984e-f9878b7da0ca"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:51.954116964Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1444,"":"io task","client":"2","":"downstairs","session_id":"d0b385c1-65c3-4fba-984e-f9878b7da0ca"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:51.954133954Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1444,"":"io task","client":"0","":"downstairs","session_id":"d0b385c1-65c3-4fba-984e-f9878b7da0ca"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:51.954147853Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1444,"":"io task","client":"1","":"downstairs","session_id":"d0b385c1-65c3-4fba-984e-f9878b7da0ca"} {"msg":"ds_connection connected from Ok(127.0.0.1:50298)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:51.954250138Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1444,"":"io task","client":"0","":"downstairs","session_id":"d0b385c1-65c3-4fba-984e-f9878b7da0ca"} {"msg":"ds_connection connected from Ok(127.0.0.1:42323)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:51.954285666Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1444,"":"io task","client":"1","":"downstairs","session_id":"d0b385c1-65c3-4fba-984e-f9878b7da0ca"} {"msg":"ds_connection connected from Ok(127.0.0.1:64831)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:51.954332464Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1444,"":"io task","client":"2","":"downstairs","session_id":"d0b385c1-65c3-4fba-984e-f9878b7da0ca"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:51.955122895Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1444,"client":"0","":"downstairs","session_id":"d0b385c1-65c3-4fba-984e-f9878b7da0ca"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:51.955151854Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1444,"client":"0","":"downstairs","session_id":"d0b385c1-65c3-4fba-984e-f9878b7da0ca"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:51.955166723Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1444,"client":"1","":"downstairs","session_id":"d0b385c1-65c3-4fba-984e-f9878b7da0ca"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:51.955180543Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1444,"client":"1","":"downstairs","session_id":"d0b385c1-65c3-4fba-984e-f9878b7da0ca"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:51.955193902Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1444,"client":"2","":"downstairs","session_id":"d0b385c1-65c3-4fba-984e-f9878b7da0ca"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:51.955247969Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1444,"client":"2","":"downstairs","session_id":"d0b385c1-65c3-4fba-984e-f9878b7da0ca"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:51.955866099Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1444,"client":"2","":"downstairs","session_id":"d0b385c1-65c3-4fba-984e-f9878b7da0ca"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:51.955937746Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1444,"client":"1","":"downstairs","session_id":"d0b385c1-65c3-4fba-984e-f9878b7da0ca"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:51.955959434Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1444,"client":"0","":"downstairs","session_id":"d0b385c1-65c3-4fba-984e-f9878b7da0ca"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:51.956584714Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1444,"client":"0","":"downstairs","session_id":"d0b385c1-65c3-4fba-984e-f9878b7da0ca"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:51.956618362Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1444,"client":"0","":"downstairs","session_id":"d0b385c1-65c3-4fba-984e-f9878b7da0ca"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:51.956634801Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1444,"session_id":"d0b385c1-65c3-4fba-984e-f9878b7da0ca"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:51.956647771Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1444,"client":"2","":"downstairs","session_id":"d0b385c1-65c3-4fba-984e-f9878b7da0ca"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:51.95666155Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1444,"client":"2","":"downstairs","session_id":"d0b385c1-65c3-4fba-984e-f9878b7da0ca"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:51.95667511Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1444,"session_id":"d0b385c1-65c3-4fba-984e-f9878b7da0ca"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:51.956687469Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1444,"client":"1","":"downstairs","session_id":"d0b385c1-65c3-4fba-984e-f9878b7da0ca"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:51.956700748Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1444,"client":"1","":"downstairs","session_id":"d0b385c1-65c3-4fba-984e-f9878b7da0ca"} {"msg":"[0]R flush_numbers[0..12]: [126, 120, 120, 112, 125, 115, 125, 109, 110, 127, 120, 120]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:51.956714648Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1444,"":"downstairs","":"downstairs","session_id":"d0b385c1-65c3-4fba-984e-f9878b7da0ca"} {"msg":"[0]R generation[0..12]: [64, 63, 63, 60, 64, 61, 64, 58, 58, 65, 63, 63]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:51.956728647Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1444,"":"downstairs","":"downstairs","session_id":"d0b385c1-65c3-4fba-984e-f9878b7da0ca"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:51.956742696Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1444,"":"downstairs","":"downstairs","session_id":"d0b385c1-65c3-4fba-984e-f9878b7da0ca"} {"msg":"[1]R flush_numbers[0..12]: [126, 120, 120, 112, 125, 115, 125, 109, 110, 127, 120, 120]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:51.956756686Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1444,"":"downstairs","":"downstairs","session_id":"d0b385c1-65c3-4fba-984e-f9878b7da0ca"} {"msg":"[1]R generation[0..12]: [64, 63, 63, 60, 64, 61, 64, 58, 58, 65, 63, 63]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:51.956770035Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1444,"":"downstairs","":"downstairs","session_id":"d0b385c1-65c3-4fba-984e-f9878b7da0ca"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:51.956784474Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1444,"":"downstairs","":"downstairs","session_id":"d0b385c1-65c3-4fba-984e-f9878b7da0ca"} {"msg":"[2]R flush_numbers[0..12]: [126, 120, 120, 112, 125, 115, 125, 109, 110, 127, 120, 120]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:51.956798513Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1444,"":"downstairs","":"downstairs","session_id":"d0b385c1-65c3-4fba-984e-f9878b7da0ca"} {"msg":"[2]R generation[0..12]: [64, 63, 63, 60, 64, 61, 64, 58, 58, 65, 63, 63]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:51.956812203Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1444,"":"downstairs","":"downstairs","session_id":"d0b385c1-65c3-4fba-984e-f9878b7da0ca"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-08T20:24:51.956828462Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1444,"":"downstairs","":"downstairs","session_id":"d0b385c1-65c3-4fba-984e-f9878b7da0ca"} {"msg":"Max found gen is 66","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:51.956847601Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1444,"":"downstairs","":"downstairs","session_id":"d0b385c1-65c3-4fba-984e-f9878b7da0ca"} {"msg":"Generation requested: 66 >= found:66","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:51.95686105Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1444,"":"downstairs","":"downstairs","session_id":"d0b385c1-65c3-4fba-984e-f9878b7da0ca"} {"msg":"Next flush: 128","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:51.95687436Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1444,"":"downstairs","":"downstairs","session_id":"d0b385c1-65c3-4fba-984e-f9878b7da0ca"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:51.956887579Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1444,"":"downstairs","":"downstairs","session_id":"d0b385c1-65c3-4fba-984e-f9878b7da0ca"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:51.956901139Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1444,"session_id":"d0b385c1-65c3-4fba-984e-f9878b7da0ca"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:51.956913428Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1444,"client":"0","":"downstairs","session_id":"d0b385c1-65c3-4fba-984e-f9878b7da0ca"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:51.956926967Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1444,"client":"1","":"downstairs","session_id":"d0b385c1-65c3-4fba-984e-f9878b7da0ca"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:51.956940437Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1444,"client":"2","":"downstairs","session_id":"d0b385c1-65c3-4fba-984e-f9878b7da0ca"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:51.956954146Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1444,"session_id":"d0b385c1-65c3-4fba-984e-f9878b7da0ca"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:51.956966765Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1444,"session_id":"d0b385c1-65c3-4fba-984e-f9878b7da0ca"} {"msg":"a42653e1-c5f1-4043-b334-87201a3d1134 is now active with session: d0b385c1-65c3-4fba-984e-f9878b7da0ca","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:51.956979745Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1444,"session_id":"d0b385c1-65c3-4fba-984e-f9878b7da0ca"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:51.956991914Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1444,"session_id":"d0b385c1-65c3-4fba-984e-f9878b7da0ca"} {"msg":"The guest has finished waiting for activation with:66","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:51.957004253Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1444} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 261 len 20480 data: 4 5 6 6 8 02/30 Write block 20 len 28672 data: 7 8 9 9 10 10 11 03/30 Write block 537 len 32768 data: 6 5 5 5 6 6 7 8 04/30 Write block 536 len 36864 data: 6 7 6 6 6 7 7 8 9 05/30 Read block 23 len 20480 06/30 Read block 319 len 4096 07/30 Read block 262 len 4096 08/30 Read block 162 len 20480 09/30 Write block 284 len 36864 data: 6 7 10 10 11 9 8 7 8 10/30 Read block 407 len 20480 11/30 Read block 426 len 28672 12/30 Write block 478 len 16384 data: 7 8 6 7 13/30 Flush 14/30 Write block 154 len 4096 data: 5 15/30 Read block 169 len 16384 16/30 Write block 538 len 12288 data: 7 7 7 17/30 Read block 341 len 28672 18/30 Read block 371 len 4096 19/30 Write block 124 len 4096 data: 7 20/30 Read block 196 len 20480 21/30 Write block 536 len 40960 data: 7 8 8 8 8 8 8 9 10 6 22/30 Read block 482 len 16384 23/30 Write block 298 len 36864 data: 4 5 5 6 5 5 4 3 4 24/30 Flush 25/30 Write block 252 len 16384 data: 11 9 9 7 26/30 Write block 471 len 20480 data: 7 7 7 7 7 27/30 Read block 358 len 32768 28/30 Write block 436 len 36864 data: 8 7 7 7 6 8 6 4 4 29/30 Read block 174 len 28672 30/30 Read block 312 len 36864 ---------------------------------------------------------------- Crucible gen:66 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 4 Done Done Done false 1025 Acked Write 5 Done Done Done false 1026 Acked Read 8 Done Done Done false 1027 Acked Write 9 Done Done Done false 1028 Acked Read 7 Done Done Done false 1029 Acked Read 9 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 6 6 6 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:24:51.984887311Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1444} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 64 64 64 126 126 126 F F F 1 020-039 66 66 66 128 128 128 F F F 2 040-059 63 63 63 120 120 120 F F F 3 060-079 60 60 60 112 112 112 F F F 4 080-099 64 64 64 125 125 125 F F F 5 100-119 61 61 61 115 115 115 F F F 6 120-139 66 66 66 129 129 129 F F F 7 140-159 66 66 66 129 129 129 F F F 8 160-179 58 58 58 110 110 110 F F F 9 180-199 65 65 65 127 127 127 F F F 10 200-219 63 63 63 120 120 120 F F F 11 220-239 63 63 63 120 120 120 F F F 12 240-259 65 65 65 127 127 127 T T T 13 260-279 66 66 66 128 128 128 F F F 14 280-299 66 66 66 129 129 129 F F F 15 300-319 66 66 66 129 129 129 F F F 16 320-339 50 50 50 98 98 98 F F F 17 340-359 64 64 64 125 125 125 F F F 18 360-379 63 63 63 120 120 120 F F F 19 380-399 64 64 64 125 125 125 F F F 20 400-419 64 64 64 125 125 125 F F F 21 420-439 56 56 56 107 107 107 T T T 22 440-459 62 62 62 116 116 116 T T T 23 460-479 66 66 66 128 128 128 T T T 24 480-499 66 66 66 128 128 128 F F F 25 500-519 63 63 63 120 120 120 F F F 26 520-539 66 66 66 129 129 129 F F F 27 540-559 66 66 66 129 129 129 F F F 28 560-579 63 63 63 120 120 120 F F F 29 580-599 55 55 55 105 105 105 F F F Max gen: 66, Max flush: 129 A Difference in extent metadata is expected here On loop 33 Verifying data now No change after verify Loop: 33 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 64 64 64 126 126 126 F F F 1 020-039 66 66 66 128 128 128 F F F 2 040-059 63 63 63 120 120 120 F F F 3 060-079 60 60 60 112 112 112 F F F 4 080-099 64 64 64 125 125 125 F F F 5 100-119 61 61 61 115 115 115 F F F 6 120-139 66 66 66 129 129 129 F F F 7 140-159 66 66 66 129 129 129 F F F 8 160-179 58 58 58 110 110 110 F F F 9 180-199 65 65 65 127 127 127 F F F 10 200-219 63 63 63 120 120 120 F F F 11 220-239 63 63 63 120 120 120 F F F 12 240-259 67 67 67 130 130 130 F F F 13 260-279 66 66 66 128 128 128 F F F 14 280-299 66 66 66 129 129 129 F F F 15 300-319 66 66 66 129 129 129 F F F 16 320-339 50 50 50 98 98 98 F F F 17 340-359 64 64 64 125 125 125 F F F 18 360-379 63 63 63 120 120 120 F F F 19 380-399 64 64 64 125 125 125 F F F 20 400-419 64 64 64 125 125 125 F F F 21 420-439 67 67 67 130 130 130 F F F 22 440-459 67 67 67 130 130 130 F F F 23 460-479 67 67 67 130 130 130 F F F 24 480-499 66 66 66 128 128 128 F F F 25 500-519 63 63 63 120 120 120 F F F 26 520-539 66 66 66 129 129 129 F F F 27 540-559 66 66 66 129 129 129 F F F 28 560-579 63 63 63 120 120 120 F F F 29 580-599 55 55 55 105 105 105 F F F Max gen: 67, Max flush: 130 Begin loop 34 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:56.287561462Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1452,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:56.333242129Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1452,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:56.333861699Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1452} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"0fd98d0d5d909611f8b408ccda2559d35fe117bb\",\n git_commit_timestamp: \"2024-11-08T20:05:44.000000000Z\",\n git_branch: \"mkeeter/drop-skipped-backpressure-guard\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:56.333900927Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1452} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:56.333915846Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1452} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:56.333930226Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1452} {"msg":"Crucible 8309b2e4-f6a3-4ba7-a291-b7a6be544ea2 has session id: 9f59ccae-0735-48b8-a87f-99e5e9f981c5","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:56.334389253Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1452,"session_id":"9f59ccae-0735-48b8-a87f-99e5e9f981c5"} {"msg":"Upstairs opts: Upstairs UUID: 8309b2e4-f6a3-4ba7-a291-b7a6be544ea2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:56.334426172Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1452,"session_id":"9f59ccae-0735-48b8-a87f-99e5e9f981c5"} {"msg":"Crucible stats registered with UUID: 8309b2e4-f6a3-4ba7-a291-b7a6be544ea2","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:56.334441561Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1452,"session_id":"9f59ccae-0735-48b8-a87f-99e5e9f981c5"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:56.365495543Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1452,"task":"crutest"} {"msg":"The guest has requested activation with gen:68","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:56.365528751Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1452} {"msg":"8309b2e4-f6a3-4ba7-a291-b7a6be544ea2 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:56.365578549Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1452,"session_id":"9f59ccae-0735-48b8-a87f-99e5e9f981c5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:56.365598658Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1452,"client":"0","":"downstairs","session_id":"9f59ccae-0735-48b8-a87f-99e5e9f981c5"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:56.365626387Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1452,"client":"0","":"downstairs","session_id":"9f59ccae-0735-48b8-a87f-99e5e9f981c5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:56.365643856Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1452,"client":"1","":"downstairs","session_id":"9f59ccae-0735-48b8-a87f-99e5e9f981c5"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:56.365660555Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1452,"client":"1","":"downstairs","session_id":"9f59ccae-0735-48b8-a87f-99e5e9f981c5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:56.365677094Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1452,"client":"2","":"downstairs","session_id":"9f59ccae-0735-48b8-a87f-99e5e9f981c5"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:56.365703663Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1452,"client":"2","":"downstairs","session_id":"9f59ccae-0735-48b8-a87f-99e5e9f981c5"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:56.365735811Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1452,"":"io task","client":"2","":"downstairs","session_id":"9f59ccae-0735-48b8-a87f-99e5e9f981c5"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:56.365771229Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1452,"":"io task","client":"0","":"downstairs","session_id":"9f59ccae-0735-48b8-a87f-99e5e9f981c5"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:56.365804658Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1452,"":"io task","client":"1","":"downstairs","session_id":"9f59ccae-0735-48b8-a87f-99e5e9f981c5"} {"msg":"ds_connection connected from Ok(127.0.0.1:38944)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:56.365838536Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1452,"":"io task","client":"2","":"downstairs","session_id":"9f59ccae-0735-48b8-a87f-99e5e9f981c5"} {"msg":"ds_connection connected from Ok(127.0.0.1:37717)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:56.366033957Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1452,"":"io task","client":"0","":"downstairs","session_id":"9f59ccae-0735-48b8-a87f-99e5e9f981c5"} {"msg":"ds_connection connected from Ok(127.0.0.1:44238)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:56.366099484Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1452,"":"io task","client":"1","":"downstairs","session_id":"9f59ccae-0735-48b8-a87f-99e5e9f981c5"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:56.366816658Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1452,"client":"2","":"downstairs","session_id":"9f59ccae-0735-48b8-a87f-99e5e9f981c5"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:56.366857436Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1452,"client":"2","":"downstairs","session_id":"9f59ccae-0735-48b8-a87f-99e5e9f981c5"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:56.366888535Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1452,"client":"0","":"downstairs","session_id":"9f59ccae-0735-48b8-a87f-99e5e9f981c5"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:56.366917753Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1452,"client":"0","":"downstairs","session_id":"9f59ccae-0735-48b8-a87f-99e5e9f981c5"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:56.366949032Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1452,"client":"1","":"downstairs","session_id":"9f59ccae-0735-48b8-a87f-99e5e9f981c5"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:56.367006649Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1452,"client":"1","":"downstairs","session_id":"9f59ccae-0735-48b8-a87f-99e5e9f981c5"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:56.367570772Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1452,"client":"1","":"downstairs","session_id":"9f59ccae-0735-48b8-a87f-99e5e9f981c5"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:56.36761171Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1452,"client":"2","":"downstairs","session_id":"9f59ccae-0735-48b8-a87f-99e5e9f981c5"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:56.367641918Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1452,"client":"0","":"downstairs","session_id":"9f59ccae-0735-48b8-a87f-99e5e9f981c5"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:56.368112165Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1452,"client":"1","":"downstairs","session_id":"9f59ccae-0735-48b8-a87f-99e5e9f981c5"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:56.368137324Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1452,"client":"1","":"downstairs","session_id":"9f59ccae-0735-48b8-a87f-99e5e9f981c5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:56.368151443Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1452,"session_id":"9f59ccae-0735-48b8-a87f-99e5e9f981c5"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:56.368243299Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1452,"client":"0","":"downstairs","session_id":"9f59ccae-0735-48b8-a87f-99e5e9f981c5"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:56.368264858Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1452,"client":"0","":"downstairs","session_id":"9f59ccae-0735-48b8-a87f-99e5e9f981c5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:56.368283267Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1452,"session_id":"9f59ccae-0735-48b8-a87f-99e5e9f981c5"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:56.368296086Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1452,"client":"2","":"downstairs","session_id":"9f59ccae-0735-48b8-a87f-99e5e9f981c5"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:56.368309836Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1452,"client":"2","":"downstairs","session_id":"9f59ccae-0735-48b8-a87f-99e5e9f981c5"} {"msg":"[0]R flush_numbers[0..12]: [126, 128, 120, 112, 125, 115, 129, 129, 110, 127, 120, 120]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:56.368323685Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1452,"":"downstairs","":"downstairs","session_id":"9f59ccae-0735-48b8-a87f-99e5e9f981c5"} {"msg":"[0]R generation[0..12]: [64, 66, 63, 60, 64, 61, 66, 66, 58, 65, 63, 63]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:56.368337404Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1452,"":"downstairs","":"downstairs","session_id":"9f59ccae-0735-48b8-a87f-99e5e9f981c5"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:56.368351423Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1452,"":"downstairs","":"downstairs","session_id":"9f59ccae-0735-48b8-a87f-99e5e9f981c5"} {"msg":"[1]R flush_numbers[0..12]: [126, 128, 120, 112, 125, 115, 129, 129, 110, 127, 120, 120]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:56.368365243Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1452,"":"downstairs","":"downstairs","session_id":"9f59ccae-0735-48b8-a87f-99e5e9f981c5"} {"msg":"[1]R generation[0..12]: [64, 66, 63, 60, 64, 61, 66, 66, 58, 65, 63, 63]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:56.368378822Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1452,"":"downstairs","":"downstairs","session_id":"9f59ccae-0735-48b8-a87f-99e5e9f981c5"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:56.368392821Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1452,"":"downstairs","":"downstairs","session_id":"9f59ccae-0735-48b8-a87f-99e5e9f981c5"} {"msg":"[2]R flush_numbers[0..12]: [126, 128, 120, 112, 125, 115, 129, 129, 110, 127, 120, 120]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:56.368406491Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1452,"":"downstairs","":"downstairs","session_id":"9f59ccae-0735-48b8-a87f-99e5e9f981c5"} {"msg":"[2]R generation[0..12]: [64, 66, 63, 60, 64, 61, 66, 66, 58, 65, 63, 63]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:56.36842006Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1452,"":"downstairs","":"downstairs","session_id":"9f59ccae-0735-48b8-a87f-99e5e9f981c5"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:56.368434079Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1452,"":"downstairs","":"downstairs","session_id":"9f59ccae-0735-48b8-a87f-99e5e9f981c5"} {"msg":"Max found gen is 68","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:56.368447019Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1452,"":"downstairs","":"downstairs","session_id":"9f59ccae-0735-48b8-a87f-99e5e9f981c5"} {"msg":"Generation requested: 68 >= found:68","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:56.368460228Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1452,"":"downstairs","":"downstairs","session_id":"9f59ccae-0735-48b8-a87f-99e5e9f981c5"} {"msg":"Next flush: 131","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:56.368475037Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1452,"":"downstairs","":"downstairs","session_id":"9f59ccae-0735-48b8-a87f-99e5e9f981c5"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:56.368488157Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1452,"":"downstairs","":"downstairs","session_id":"9f59ccae-0735-48b8-a87f-99e5e9f981c5"} {"msg":"No downstairs reconciliation required","v":0,"Wait for a query_work_queue command to finish before sending IO name":"crucible","level":30,"time":"2024-11-08T20:24:56.368522115Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1452,"session_id":"9f59ccae-0735-48b8-a87f-99e5e9f981c5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:56.368539434Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1452,"client":"0","":"downstairs","session_id":"9f59ccae-0735-48b8-a87f-99e5e9f981c5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:56.368553293Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1452,"client":"1","":"downstairs","session_id":"9f59ccae-0735-48b8-a87f-99e5e9f981c5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:56.368566653Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1452,"client":"2","":"downstairs","session_id":"9f59ccae-0735-48b8-a87f-99e5e9f981c5"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:56.368579722Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1452,"session_id":"9f59ccae-0735-48b8-a87f-99e5e9f981c5"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:56.368592082Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1452,"session_id":"9f59ccae-0735-48b8-a87f-99e5e9f981c5"} {"msg":"8309b2e4-f6a3-4ba7-a291-b7a6be544ea2 is now active with session: 9f59ccae-0735-48b8-a87f-99e5e9f981c5","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:56.368604781Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1452,"session_id":"9f59ccae-0735-48b8-a87f-99e5e9f981c5"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:56.36861683Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1452,"session_id":"9f59ccae-0735-48b8-a87f-99e5e9f981c5"} {"msg":"The guest has finished waiting for activation with:68","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:56.36862904Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1452} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 390 len 36864 data: 6 6 7 8 8 6 6 8 8 03/30 Read block 445 len 36864 04/30 Read block 418 len 36864 05/30 Read block 472 len 4096 06/30 Write block 121 len 20480 data: 7 8 9 8 5 07/30 Write block 375 len 32768 data: 5 5 8 9 9 8 7 6 08/30 Read block 558 len 32768 09/30 Write block 511 len 20480 data: 7 6 7 6 5 10/30 Read block 581 len 28672 11/30 Read block 397 len 12288 12/30 Write block 15 len 16384 data: 7 7 6 5 13/30 Read block 405 len 4096 14/30 Write block 315 len 32768 data: 8 7 5 6 7 5 4 5 15/30 Flush 16/30 Write block 527 len 36864 data: 4 5 4 3 3 2 3 3 6 17/30 Read block 398 len 8192 18/30 Read block 294 len 16384 19/30 Write block 304 len 4096 data: 5 20/30 Flush 21/30 Write block 319 len 24576 data: 8 6 5 6 6 5 22/30 Write block 147 len 36864 data: 6 5 6 6 7 4 4 6 5 23/30 Read block 111 len 28672 24/30 Write block 259 len 28672 data: 6 5 5 6 7 7 9 25/30 Read block 241 len 40960 26/30 Read block 283 len 4096 27/30 Read block 419 len 16384 28/30 Write block 576 len 40960 data: 5 5 5 4 5 5 5 8 9 12 29/30 Read block 161 len 28672 30/30 Read block 505 len 8192 ---------------------------------------------------------------- Crucible gen:68 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1020 Acked Write 6 Done Done Done false 1021 Acked Write 9 Done Done Done false 1022 Acked Read 7 Done Done Done false 1023 Acked Write 7 Done Done Done false 1024 Acked Read 10 Done Done Done false 1025 Acked Read 1 Done Done Done false 1026 Acked Read 4 Done Done Done false 1027 Acked Write 10 Done Done Done false 1028 Acked Read 7 Done Done Done false 1029 Acked Read 2 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 10 10 10 30 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1019 1019 1019 Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:24:56.394885586Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1452} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 68 68 68 132 132 132 F F F 1 020-039 66 66 66 128 128 128 F F F 2 040-059 63 63 63 120 120 120 F F F 3 060-079 60 60 60 112 112 112 F F F 4 080-099 64 64 64 125 125 125 F F F 5 100-119 61 61 61 115 115 115 F F F 6 120-139 68 68 68 132 132 132 F F F 7 140-159 66 66 66 129 129 129 T T T 8 160-179 58 58 58 110 110 110 F F F 9 180-199 65 65 65 127 127 127 F F F 10 200-219 63 63 63 120 120 120 F F F 11 220-239 63 63 63 120 120 120 F F F 12 240-259 67 67 67 130 130 130 T T T 13 260-279 66 66 66 128 128 128 T T T 14 280-299 66 66 66 129 129 129 F F F 15 300-319 68 68 68 133 133 133 T T T 16 320-339 68 68 68 132 132 132 T T T 17 340-359 64 64 64 125 125 125 F F F 18 360-379 68 68 68 132 132 132 F F F 19 380-399 68 68 68 132 132 132 F F F 20 400-419 64 64 64 125 125 125 F F F 21 420-439 67 67 67 130 130 130 F F F 22 440-459 67 67 67 130 130 130 F F F 23 460-479 67 67 67 130 130 130 F F F 24 480-499 66 66 66 128 128 128 F F F 25 500-519 68 68 68 132 132 132 F F F 26 520-539 68 68 68 133 133 133 F F F 27 540-559 66 66 66 129 129 129 F F F 28 560-579 63 63 63 120 120 120 T T T 29 580-599 55 55 55 105 105 105 T T T Max gen: 68, Max flush: 133 A Difference in extent metadata is expected here On loop 34 Verifying data now No change after verify Loop: 34 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 68 68 68 132 132 132 F F F 1 020-039 66 66 66 128 128 128 F F F 2 040-059 63 63 63 120 120 120 F F F 3 060-079 60 60 60 112 112 112 F F F 4 080-099 64 64 64 125 125 125 F F F 5 100-119 61 61 61 115 115 115 F F F 6 120-139 68 68 68 132 132 132 F F F 7 140-159 69 69 69 134 134 134 F F F 8 160-179 58 58 58 110 110 110 F F F 9 180-199 65 65 65 127 127 127 F F F 10 200-219 63 63 63 120 120 120 F F F 11 220-239 63 63 63 120 120 120 F F F 12 240-259 69 69 69 134 134 134 F F F 13 260-279 69 69 69 134 134 134 F F F 14 280-299 66 66 66 129 129 129 F F F 15 300-319 69 69 69 134 134 134 F F F 16 320-339 69 69 69 134 134 134 F F F 17 340-359 64 64 64 125 125 125 F F F 18 360-379 68 68 68 132 132 132 F F F 19 380-399 68 68 68 132 132 132 F F F 20 400-419 64 64 64 125 125 125 F F F 21 420-439 67 67 67 130 130 130 F F F 22 440-459 67 67 67 130 130 130 F F F 23 460-479 67 67 67 130 130 130 F F F 24 480-499 66 66 66 128 128 128 F F F 25 500-519 68 68 68 132 132 132 F F F 26 520-539 68 68 68 133 133 133 F F F 27 540-559 66 66 66 129 129 129 F F F 28 560-579 69 69 69 134 134 134 F F F 29 580-599 69 69 69 134 134 134 F F F Max gen: 69, Max flush: 134 Begin loop 35 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:00.743464873Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1460,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:00.792485117Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1460,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:00.793106316Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1460} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"0fd98d0d5d909611f8b408ccda2559d35fe117bb\",\n git_commit_timestamp: \"2024-11-08T20:05:44.000000000Z\",\n git_branch: \"mkeeter/drop-skipped-backpressure-guard\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:00.793141665Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1460} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:00.793156034Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1460} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:00.793170393Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1460} {"msg":"Crucible 9f866139-dd2f-47aa-95ff-d724e157f140 has session id: 692302a5-3837-4128-bee3-845906d3c218","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:00.79365387Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1460,"session_id":"692302a5-3837-4128-bee3-845906d3c218"} {"msg":"Upstairs opts: Upstairs UUID: 9f866139-dd2f-47aa-95ff-d724e157f140, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:00.793687018Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1460,"session_id":"692302a5-3837-4128-bee3-845906d3c218"} {"msg":"Crucible stats registered with UUID: 9f866139-dd2f-47aa-95ff-d724e157f140","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:00.793702107Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1460,"session_id":"692302a5-3837-4128-bee3-845906d3c218"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:00.824463804Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1460,"task":"crutest"} {"msg":"The guest has requested activation with gen:70","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:00.824494922Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1460} {"msg":"9f866139-dd2f-47aa-95ff-d724e157f140 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:00.82454352Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1460,"session_id":"692302a5-3837-4128-bee3-845906d3c218"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:00.824563469Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1460,"client":"0","":"downstairs","session_id":"692302a5-3837-4128-bee3-845906d3c218"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:00.824591178Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1460,"client":"0","":"downstairs","session_id":"692302a5-3837-4128-bee3-845906d3c218"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:00.824608777Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1460,"client":"1","":"downstairs","session_id":"692302a5-3837-4128-bee3-845906d3c218"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:00.824635165Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1460,"client":"1","":"downstairs","session_id":"692302a5-3837-4128-bee3-845906d3c218"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:00.824667484Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1460,"client":"2","":"downstairs","session_id":"692302a5-3837-4128-bee3-845906d3c218"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:00.824698742Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1460,"client":"2","":"downstairs","session_id":"692302a5-3837-4128-bee3-845906d3c218"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:00.824726241Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1460,"":"io task","client":"2","":"downstairs","session_id":"692302a5-3837-4128-bee3-845906d3c218"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:00.82475212Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1460,"":"io task","client":"0","":"downstairs","session_id":"692302a5-3837-4128-bee3-845906d3c218"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:00.824778318Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1460,"":"io task","client":"1","":"downstairs","session_id":"692302a5-3837-4128-bee3-845906d3c218"} {"msg":"ds_connection connected from Ok(127.0.0.1:43812)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:00.824867794Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1460,"":"io task","client":"1","":"downstairs","session_id":"692302a5-3837-4128-bee3-845906d3c218"} {"msg":"ds_connection connected from Ok(127.0.0.1:50555)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:00.825021826Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1460,"":"io task","client":"2","":"downstairs","session_id":"692302a5-3837-4128-bee3-845906d3c218"} {"msg":"ds_connection connected from Ok(127.0.0.1:61018)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:00.825066054Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1460,"":"io task","client":"0","":"downstairs","session_id":"692302a5-3837-4128-bee3-845906d3c218"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:00.825630367Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1460,"client":"2","":"downstairs","session_id":"692302a5-3837-4128-bee3-845906d3c218"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:00.825662665Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1460,"client":"2","":"downstairs","session_id":"692302a5-3837-4128-bee3-845906d3c218"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:00.825678724Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1460,"client":"0","":"downstairs","session_id":"692302a5-3837-4128-bee3-845906d3c218"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:00.825693104Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1460,"client":"0","":"downstairs","session_id":"692302a5-3837-4128-bee3-845906d3c218"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:00.825706663Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1460,"client":"1","":"downstairs","session_id":"692302a5-3837-4128-bee3-845906d3c218"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:00.825728682Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1460,"client":"1","":"downstairs","session_id":"692302a5-3837-4128-bee3-845906d3c218"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:00.826299564Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1460,"client":"2","":"downstairs","session_id":"692302a5-3837-4128-bee3-845906d3c218"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:00.826449077Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1460,"client":"1","":"downstairs","session_id":"692302a5-3837-4128-bee3-845906d3c218"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:00.826601249Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1460,"client":"0","":"downstairs","session_id":"692302a5-3837-4128-bee3-845906d3c218"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:00.826837998Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1460,"client":"2","":"downstairs","session_id":"692302a5-3837-4128-bee3-845906d3c218"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:00.826861197Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1460,"client":"2","":"downstairs","session_id":"692302a5-3837-4128-bee3-845906d3c218"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:00.826875456Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1460,"session_id":"692302a5-3837-4128-bee3-845906d3c218"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:00.827039998Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1460,"client":"1","":"downstairs","session_id":"692302a5-3837-4128-bee3-845906d3c218"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:00.827065087Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1460,"client":"1","":"downstairs","session_id":"692302a5-3837-4128-bee3-845906d3c218"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:00.827079326Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1460,"session_id":"692302a5-3837-4128-bee3-845906d3c218"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:00.827106335Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1460,"client":"0","":"downstairs","session_id":"692302a5-3837-4128-bee3-845906d3c218"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:00.827120454Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1460,"client":"0","":"downstairs","session_id":"692302a5-3837-4128-bee3-845906d3c218"} {"msg":"[0]R flush_numbers[0..12]: [132, 128, 120, 112, 125, 115, 132, 134, 110, 127, 120, 120]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:00.827162092Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1460,"":"downstairs","":"downstairs","session_id":"692302a5-3837-4128-bee3-845906d3c218"} {"msg":"[0]R generation[0..12]: [68, 66, 63, 60, 64, 61, 68, 69, 58, 65, 63, 63]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:00.827179671Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1460,"":"downstairs","":"downstairs","session_id":"692302a5-3837-4128-bee3-845906d3c218"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:00.82719412Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1460,"":"downstairs","":"downstairs","session_id":"692302a5-3837-4128-bee3-845906d3c218"} {"msg":"[1]R flush_numbers[0..12]: [132, 128, 120, 112, 125, 115, 132, 134, 110, 127, 120, 120]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:00.8272083Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1460,"":"downstairs","":"downstairs","session_id":"692302a5-3837-4128-bee3-845906d3c218"} {"msg":"[1]R generation[0..12]: [68, 66, 63, 60, 64, 61, 68, 69, 58, 65, 63, 63]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:00.827222139Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1460,"":"downstairs","":"downstairs","session_id":"692302a5-3837-4128-bee3-845906d3c218"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:00.827236268Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1460,"":"downstairs","":"downstairs","session_id":"692302a5-3837-4128-bee3-845906d3c218"} {"msg":"[2]R flush_numbers[0..12]: [132, 128, 120, 112, 125, 115, 132, 134, 110, 127, 120, 120]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:00.827249988Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1460,"":"downstairs","":"downstairs","session_id":"692302a5-3837-4128-bee3-845906d3c218"} {"msg":"[2]R generation[0..12]: [68, 66, 63, 60, 64, 61, 68, 69, 58, 65, 63, 63]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:00.827263717Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1460,"":"downstairs","":"downstairs","session_id":"692302a5-3837-4128-bee3-845906d3c218"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:00.827277666Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1460,"":"downstairs","":"downstairs","session_id":"692302a5-3837-4128-bee3-845906d3c218"} {"msg":"Max found gen is 70","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:00.827290756Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1460,"":"downstairs","":"downstairs","session_id":"692302a5-3837-4128-bee3-845906d3c218"} {"msg":"Generation requested: 70 >= found:70","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:00.827304055Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1460,"":"downstairs","":"downstairs","session_id":"692302a5-3837-4128-bee3-845906d3c218"} {"msg":"Next flush: 135","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:00.827317124Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1460,"":"downstairs","":"downstairs","session_id":"692302a5-3837-4128-bee3-845906d3c218"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:00.827330154Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1460,"":"downstairs","":"downstairs","session_id":"692302a5-3837-4128-bee3-845906d3c218"} {"msg":"No downstairs reconciliation required"Wait for a query_work_queue command to finish before sending IO ,"v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:00.827354543Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1460,"session_id":"692302a5-3837-4128-bee3-845906d3c218"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:00.827366752Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1460,"client":"0","":"downstairs","session_id":"692302a5-3837-4128-bee3-845906d3c218"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:00.827380321Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1460,"client":"1","":"downstairs","session_id":"692302a5-3837-4128-bee3-845906d3c218"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:00.82739373Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1460,"client":"2","":"downstairs","session_id":"692302a5-3837-4128-bee3-845906d3c218"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:00.82740873Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1460,"session_id":"692302a5-3837-4128-bee3-845906d3c218"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:00.827421319Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1460,"session_id":"692302a5-3837-4128-bee3-845906d3c218"} {"msg":"9f866139-dd2f-47aa-95ff-d724e157f140 is now active with session: 692302a5-3837-4128-bee3-845906d3c218","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:00.827434199Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1460,"session_id":"692302a5-3837-4128-bee3-845906d3c218"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:00.827446258Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1460,"session_id":"692302a5-3837-4128-bee3-845906d3c218"} {"msg":"The guest has finished waiting for activation with:70","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:00.827458607Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1460} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 475 len 36864 data: 8 7 7 8 9 7 8 7 8 02/30 Write block 299 len 32768 data: 6 6 7 6 6 6 4 5 03/30 Write block 442 len 28672 data: 7 5 5 5 7 6 5 04/30 Read block 562 len 36864 05/30 Read block 541 len 24576 06/30 Read block 265 len 8192 07/30 Read block 22 len 40960 08/30 Read block 402 len 32768 09/30 Read block 451 len 24576 10/30 Flush 11/30 Write block 303 len 16384 data: 7 7 5 6 12/30 Write block 559 len 28672 data: 6 5 4 3 3 4 5 13/30 Read block 118 len 36864 14/30 Read block 206 len 32768 15/30 Read block 503 len 32768 16/30 Read block 75 len 36864 17/30 Write block 501 len 28672 data: 6 6 5 6 5 4 4 18/30 Read block 248 len 28672 19/30 Read block 211 len 16384 20/30 Read block 365 len 24576 21/30 Read block 223 len 28672 22/30 Write block 136 len 16384 data: 7 8 8 7 23/30 Write block 571 len 16384 data: 7 9 9 6 24/30 Write block 523 len 4096 data: 5 25/30 Read block 339 len 8192 26/30 Read block 578 len 12288 27/30 Write block 429 len 40960 data: 8 8 7 7 5 6 8 9 8 8 28/30 Write block 446 len 36864 data: 8 7 6 5 5 5 6 5 5 29/30 Write block 287 len 8192 data: 11 12 30/30 Read block 347 len 12288 ---------------------------------------------------------------- Crucible gen:70 GIO:true work queues: Upstairs:0 downstairs:20 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1010 Acked Write 4 Done Done Done false 1011 Acked Write 7 Done Done Done false 1012 Acked Read 9 Done Done Done false 1013 Acked Read 8 Done Done Done false 1014 Acked Read 8 Done Done Done false 1015 Acked Read 9 Done Done Done false 1016 Acked Write 7 Done Done Done false 1017 Acked Read 7 Done Done Done false 1018 Acked Read 4 Done Done Done false 1019 Acked Read 6 Done Done Done false 1020 Acked Read 7 Done Done Done false 1021 Acked Write 4 Done Done Done false 1022 Acked Write 4 Done Done Done false 1023 Acked Write 1 Done Done Done false 1024 Acked Read 2 Done Done Done false 1025 Acked Read 3 Done Done Done false 1026 Acked Write 10 Done Done Done false 1027 Acked Write 9 Sent Done Sent false 1028 Acked Write 2 Sent Done Sent false 1029 Acked Read 3 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 0 3 6 Done 17 20 17 54 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1009 1009 1009 Downstairs last five completed: 1009 1008 1007 1006 1005 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:25:00.851240295Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1460} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 68 68 68 132 132 132 F F F 1 020-039 66 66 66 128 128 128 F F F 2 040-059 63 63 63 120 120 120 F F F 3 060-079 60 60 60 112 112 112 F F F 4 080-099 64 64 64 125 125 125 F F F 5 100-119 61 61 61 115 115 115 F F F 6 120-139 68 68 68 132 132 132 T T T 7 140-159 69 69 69 134 134 134 F F F 8 160-179 58 58 58 110 110 110 F F F 9 180-199 65 65 65 127 127 127 F F F 10 200-219 63 63 63 120 120 120 F F F 11 220-239 63 63 63 120 120 120 F F F 12 240-259 69 69 69 134 134 134 F F F 13 260-279 69 69 69 134 134 134 F F F 14 280-299 70 70 70 135 135 135 T T T 15 300-319 70 70 70 135 135 135 T T T 16 320-339 69 69 69 134 134 134 F F F 17 340-359 64 64 64 125 125 125 F F F 18 360-379 68 68 68 132 132 132 F F F 19 380-399 68 68 68 132 132 132 F F F 20 400-419 64 64 64 125 125 125 F F F 21 420-439 67 67 67 130 130 130 T T T 22 440-459 70 70 70 135 135 135 T T T 23 460-479 70 70 70 135 135 135 F F F 24 480-499 70 70 70 135 135 135 F F F 25 500-519 68 68 68 132 132 132 T T T 26 520-539 68 68 68 133 133 133 T T T 27 540-559 66 66 66 129 129 129 T T T 28 560-579 69 69 69 134 134 134 T T T 29 580-599 69 69 69 134 134 134 F F F Max gen: 70, Max flush: 135 A Difference in extent metadata is expected here On loop 35 Verifying data now No change after verify Loop: 35 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 68 68 68 132 132 132 F F F 1 020-039 66 66 66 128 128 128 F F F 2 040-059 63 63 63 120 120 120 F F F 3 060-079 60 60 60 112 112 112 F F F 4 080-099 64 64 64 125 125 125 F F F 5 100-119 61 61 61 115 115 115 F F F 6 120-139 71 71 71 136 136 136 F F F 7 140-159 69 69 69 134 134 134 F F F 8 160-179 58 58 58 110 110 110 F F F 9 180-199 65 65 65 127 127 127 F F F 10 200-219 63 63 63 120 120 120 F F F 11 220-239 63 63 63 120 120 120 F F F 12 240-259 69 69 69 134 134 134 F F F 13 260-279 69 69 69 134 134 134 F F F 14 280-299 71 71 71 136 136 136 F F F 15 300-319 71 71 71 136 136 136 F F F 16 320-339 69 69 69 134 134 134 F F F 17 340-359 64 64 64 125 125 125 F F F 18 360-379 68 68 68 132 132 132 F F F 19 380-399 68 68 68 132 132 132 F F F 20 400-419 64 64 64 125 125 125 F F F 21 420-439 71 71 71 136 136 136 F F F 22 440-459 71 71 71 136 136 136 F F F 23 460-479 70 70 70 135 135 135 F F F 24 480-499 70 70 70 135 135 135 F F F 25 500-519 71 71 71 136 136 136 F F F 26 520-539 71 71 71 136 136 136 F F F 27 540-559 71 71 71 136 136 136 F F F 28 560-579 71 71 71 136 136 136 F F F 29 580-599 69 69 69 134 134 134 F F F Max gen: 71, Max flush: 136 Begin loop 36 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:05.238709062Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1468,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:05.283024036Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1468,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:05.283728961Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1468} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"0fd98d0d5d909611f8b408ccda2559d35fe117bb\",\n git_commit_timestamp: \"2024-11-08T20:05:44.000000000Z\",\n git_branch: \"mkeeter/drop-skipped-backpressure-guard\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:05.28376828Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1468} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:05.283783149Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1468} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:05.283797658Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1468} {"msg":"Crucible 0014e95c-81e0-4735-b72e-63d450d2c712 has session id: 06d739f0-66b9-4818-aa3d-7c71c42e1388","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:05.284235957Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1468,"session_id":"06d739f0-66b9-4818-aa3d-7c71c42e1388"} {"msg":"Upstairs opts: Upstairs UUID: 0014e95c-81e0-4735-b72e-63d450d2c712, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:05.284281644Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1468,"session_id":"06d739f0-66b9-4818-aa3d-7c71c42e1388"} {"msg":"Crucible stats registered with UUID: 0014e95c-81e0-4735-b72e-63d450d2c712","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:05.284299414Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1468,"session_id":"06d739f0-66b9-4818-aa3d-7c71c42e1388"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:05.315305178Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1468,"task":"crutest"} {"msg":"The guest has requested activation with gen:72","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:05.315343566Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1468} {"msg":"0014e95c-81e0-4735-b72e-63d450d2c712 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:05.315363525Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1468,"session_id":"06d739f0-66b9-4818-aa3d-7c71c42e1388"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:05.315379835Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1468,"client":"0","":"downstairs","session_id":"06d739f0-66b9-4818-aa3d-7c71c42e1388"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:05.315406063Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1468,"client":"0","":"downstairs","session_id":"06d739f0-66b9-4818-aa3d-7c71c42e1388"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:05.315425232Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1468,"client":"1","":"downstairs","session_id":"06d739f0-66b9-4818-aa3d-7c71c42e1388"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:05.315442291Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1468,"client":"1","":"downstairs","session_id":"06d739f0-66b9-4818-aa3d-7c71c42e1388"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:05.315458991Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1468,"client":"2","":"downstairs","session_id":"06d739f0-66b9-4818-aa3d-7c71c42e1388"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:05.315487249Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1468,"client":"2","":"downstairs","session_id":"06d739f0-66b9-4818-aa3d-7c71c42e1388"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:05.315518328Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1468,"":"io task","client":"2","":"downstairs","session_id":"06d739f0-66b9-4818-aa3d-7c71c42e1388"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:05.315544636Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1468,"":"io task","client":"0","":"downstairs","session_id":"06d739f0-66b9-4818-aa3d-7c71c42e1388"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:05.315585734Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1468,"":"io task","client":"1","":"downstairs","session_id":"06d739f0-66b9-4818-aa3d-7c71c42e1388"} {"msg":"ds_connection connected from Ok(127.0.0.1:62404)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:05.315620423Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1468,"":"io task","client":"2","":"downstairs","session_id":"06d739f0-66b9-4818-aa3d-7c71c42e1388"} {"msg":"ds_connection connected from Ok(127.0.0.1:51202)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:05.315650571Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1468,"":"io task","client":"0","":"downstairs","session_id":"06d739f0-66b9-4818-aa3d-7c71c42e1388"} {"msg":"ds_connection connected from Ok(127.0.0.1:43440)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:05.31567783Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1468,"":"io task","client":"1","":"downstairs","session_id":"06d739f0-66b9-4818-aa3d-7c71c42e1388"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:05.316356987Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1468,"client":"1","":"downstairs","session_id":"06d739f0-66b9-4818-aa3d-7c71c42e1388"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:05.316392605Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1468,"client":"1","":"downstairs","session_id":"06d739f0-66b9-4818-aa3d-7c71c42e1388"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:05.316513409Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1468,"client":"0","":"downstairs","session_id":"06d739f0-66b9-4818-aa3d-7c71c42e1388"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:05.316577446Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1468,"client":"0","":"downstairs","session_id":"06d739f0-66b9-4818-aa3d-7c71c42e1388"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:05.316602105Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1468,"client":"2","":"downstairs","session_id":"06d739f0-66b9-4818-aa3d-7c71c42e1388"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:05.316624684Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1468,"client":"2","":"downstairs","session_id":"06d739f0-66b9-4818-aa3d-7c71c42e1388"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:05.316984456Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1468,"client":"1","":"downstairs","session_id":"06d739f0-66b9-4818-aa3d-7c71c42e1388"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:05.317172997Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1468,"client":"0","":"downstairs","session_id":"06d739f0-66b9-4818-aa3d-7c71c42e1388"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:05.317485692Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1468,"client":"2","":"downstairs","session_id":"06d739f0-66b9-4818-aa3d-7c71c42e1388"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:05.317660363Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1468,"client":"1","":"downstairs","session_id":"06d739f0-66b9-4818-aa3d-7c71c42e1388"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:05.317695341Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1468,"client":"1","":"downstairs","session_id":"06d739f0-66b9-4818-aa3d-7c71c42e1388"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:05.31771457Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1468,"session_id":"06d739f0-66b9-4818-aa3d-7c71c42e1388"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:05.31772774Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1468,"client":"0","":"downstairs","session_id":"06d739f0-66b9-4818-aa3d-7c71c42e1388"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:05.317741359Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1468,"client":"0","":"downstairs","session_id":"06d739f0-66b9-4818-aa3d-7c71c42e1388"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:05.317754788Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1468,"session_id":"06d739f0-66b9-4818-aa3d-7c71c42e1388"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:05.317940819Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1468,"client":"2","":"downstairs","session_id":"06d739f0-66b9-4818-aa3d-7c71c42e1388"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:05.317964198Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1468,"client":"2","":"downstairs","session_id":"06d739f0-66b9-4818-aa3d-7c71c42e1388"} {"msg":"[0]R flush_numbers[0..12]: [132, 128, 120, 112, 125, 115, 136, 134, 110, 127, 120, 120]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:05.317982477Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1468,"":"downstairs","":"downstairs","session_id":"06d739f0-66b9-4818-aa3d-7c71c42e1388"} {"msg":"[0]R generation[0..12]: [68, 66, 63, 60, 64, 61, 71, 69, 58, 65, 63, 63]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:05.317997096Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1468,"":"downstairs","":"downstairs","session_id":"06d739f0-66b9-4818-aa3d-7c71c42e1388"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:05.318011426Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1468,"":"downstairs","":"downstairs","session_id":"06d739f0-66b9-4818-aa3d-7c71c42e1388"} {"msg":"[1]R flush_numbers[0..12]: [132, 128, 120, 112, 125, 115, 136, 134, 110, 127, 120, 120]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:05.318025445Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1468,"":"downstairs","":"downstairs","session_id":"06d739f0-66b9-4818-aa3d-7c71c42e1388"} {"msg":"[1]R generation[0..12]: [68, 66, 63, 60, 64, 61, 71, 69, 58, 65, 63, 63]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:05.318039114Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1468,"":"downstairs","":"downstairs","session_id":"06d739f0-66b9-4818-aa3d-7c71c42e1388"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:05.318053174Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1468,"":"downstairs","":"downstairs","session_id":"06d739f0-66b9-4818-aa3d-7c71c42e1388"} {"msg":"[2]R flush_numbers[0..12]: [132, 128, 120, 112, 125, 115, 136, 134, 110, 127, 120, 120]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:05.318066863Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1468,"":"downstairs","":"downstairs","session_id":"06d739f0-66b9-4818-aa3d-7c71c42e1388"} {"msg":"[2]R generation[0..12]: [68, 66, 63, 60, 64, 61, 71, 69, 58, 65, 63, 63]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:05.318080392Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1468,"":"downstairs","":"downstairs","session_id":"06d739f0-66b9-4818-aa3d-7c71c42e1388"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:05.318094412Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1468,"":"downstairs","":"downstairs","session_id":"06d739f0-66b9-4818-aa3d-7c71c42e1388"} {"msg":"Max found gen is 72","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:05.318107491Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1468,"":"downstairs","":"downstairs","session_id":"06d739f0-66b9-4818-aa3d-7c71c42e1388"} {"msg":"Generation requested: 72 >= found:72","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:05.318120821Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1468,"":"downstairs","":"downstairs","session_id":"06d739f0-66b9-4818-aa3d-7c71c42e1388"} {"msg":"Next flush: 137","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:05.31813379Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1468,"":"downstairs","":"downstairs","session_id":"06d739f0-66b9-4818-aa3d-7c71c42e1388"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:05.318146939Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1468,"":"downstairs","":"downstairs","session_id":"06d739f0-66b9-4818-aa3d-7c71c42e1388"Wait for a query_work_queue command to finish before sending IO } {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:05.318168548Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1468,"session_id":"06d739f0-66b9-4818-aa3d-7c71c42e1388"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:05.318182658Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1468,"client":"0","":"downstairs","session_id":"06d739f0-66b9-4818-aa3d-7c71c42e1388"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:05.318196107Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1468,"client":"1","":"downstairs","session_id":"06d739f0-66b9-4818-aa3d-7c71c42e1388"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:05.318209266Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1468,"client":"2","":"downstairs","session_id":"06d739f0-66b9-4818-aa3d-7c71c42e1388"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:05.318222485Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1468,"session_id":"06d739f0-66b9-4818-aa3d-7c71c42e1388"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:05.318234795Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1468,"session_id":"06d739f0-66b9-4818-aa3d-7c71c42e1388"} {"msg":"0014e95c-81e0-4735-b72e-63d450d2c712 is now active with session: 06d739f0-66b9-4818-aa3d-7c71c42e1388","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:05.318247634Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1468,"session_id":"06d739f0-66b9-4818-aa3d-7c71c42e1388"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:05.318259604Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1468,"session_id":"06d739f0-66b9-4818-aa3d-7c71c42e1388"} {"msg":"The guest has finished waiting for activation with:72","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:05.318272093Z","hostname":"Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1468} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 424 len 12288 data: 4 5 5 02/30 Read block 341 len 16384 03/30 Read block 157 len 16384 04/30 Write block 333 len 20480 data: 5 6 8 8 9 05/30 Read block 126 len 28672 06/30 Read block 371 len 8192 07/30 Write block 156 len 20480 data: 4 4 4 5 4 08/30 Write block 489 len 8192 data: 6 6 09/30 Write block 440 len 40960 data: 7 9 8 6 6 6 9 8 7 6 10/30 Read block 286 len 40960 11/30 Write block 297 len 40960 data: 3 5 7 7 8 7 8 8 6 7 12/30 Flush 13/30 Write block 574 len 16384 data: 7 7 6 6 14/30 Read block 270 len 24576 15/30 Read block 356 len 16384 16/30 Read block 282 len 20480 17/30 Read block 409 len 12288 18/30 Read block 449 len 16384 19/30 Read block 399 len 20480 20/30 Write block 540 len 4096 data: 9 21/30 Flush 22/30 Write block 238 len 12288 data: 9 8 8 23/30 Write block 525 len 4096 data: 4 24/30 Read block 30 len 32768 25/30 Write block 569 len 24576 data: 6 5 8 10 10 8 26/30 Read block 572 len 36864 27/30 Write block 495 len 32768 data: 7 7 8 8 7 6 7 7 28/30 Write block 190 len 40960 data: 10 7 7 7 7 6 7 8 8 7 29/30 Write block 367 len 40960 data: 8 10 8 7 8 10 6 6 6 6 30/30 Write block 361 len 8192 data: 13 11 ---------------------------------------------------------------- Crucible gen:72 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 3 Done Done Done false 1022 Acked Write 1 Done Done Done false 1023 Acked Read 8 Done Done Done false 1024 Acked Write 6 Done Done Done false 1025 Acked Read 9 Done Done Done false 1026 Acked Write 8 Done Done Done false 1027 Acked Write 10 Done Done Done false 1028 Acked Write 10 Sent Sent Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 7 7 7 21 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1020 1020 1020 Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:25:05.342440692Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1468} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 68 68 68 132 132 132 F F F 1 020-039 66 66 66 128 128 128 F F F 2 040-059 63 63 63 120 120 120 F F F 3 060-079 60 60 60 112 112 112 F F F 4 080-099 64 64 64 125 125 125 F F F 5 100-119 61 61 61 115 115 115 F F F 6 120-139 71 71 71 136 136 136 F F F 7 140-159 72 72 72 137 137 137 F F F 8 160-179 72 72 72 137 137 137 F F F 9 180-199 65 65 65 127 127 127 T T T 10 200-219 63 63 63 120 120 120 F F F 11 220-239 63 63 63 120 120 120 T T T 12 240-259 69 69 69 134 134 134 T T T 13 260-279 69 69 69 134 134 134 F F F 14 280-299 72 72 72 137 137 137 F F F 15 300-319 72 72 72 137 137 137 F F F 16 320-339 72 72 72 137 137 137 F F F 17 340-359 64 64 64 125 125 125 F F F 18 360-379 68 68 68 132 132 132 T T T 19 380-399 68 68 68 132 132 132 F F F 20 400-419 64 64 64 125 125 125 F F F 21 420-439 72 72 72 137 137 137 F F F 22 440-459 72 72 72 137 137 137 F F F 23 460-479 70 70 70 135 135 135 F F F 24 480-499 72 72 72 137 137 137 T T T 25 500-519 71 71 71 136 136 136 T T T 26 520-539 71 71 71 136 136 136 T T T 27 540-559 72 72 72 138 138 138 F F F 28 560-579 72 72 72 138 138 138 T T T 29 580-599 69 69 69 134 134 134 F F F Max gen: 72, Max flush: 138 A Difference in extent metadata is expected here On loop 36 Verifying data now No change after verify Loop: 36 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 68 68 68 132 132 132 F F F 1 020-039 66 66 66 128 128 128 F F F 2 040-059 63 63 63 120 120 120 F F F 3 060-079 60 60 60 112 112 112 F F F 4 080-099 64 64 64 125 125 125 F F F 5 100-119 61 61 61 115 115 115 F F F 6 120-139 71 71 71 136 136 136 F F F 7 140-159 72 72 72 137 137 137 F F F 8 160-179 72 72 72 137 137 137 F F F 9 180-199 73 73 73 139 139 139 F F F 10 200-219 63 63 63 120 120 120 F F F 11 220-239 73 73 73 139 139 139 F F F 12 240-259 73 73 73 139 139 139 F F F 13 260-279 69 69 69 134 134 134 F F F 14 280-299 72 72 72 137 137 137 F F F 15 300-319 72 72 72 137 137 137 F F F 16 320-339 72 72 72 137 137 137 F F F 17 340-359 64 64 64 125 125 125 F F F 18 360-379 73 73 73 139 139 139 F F F 19 380-399 68 68 68 132 132 132 F F F 20 400-419 64 64 64 125 125 125 F F F 21 420-439 72 72 72 137 137 137 F F F 22 440-459 72 72 72 137 137 137 F F F 23 460-479 70 70 70 135 135 135 F F F 24 480-499 73 73 73 139 139 139 F F F 25 500-519 73 73 73 139 139 139 F F F 26 520-539 73 73 73 139 139 139 F F F 27 540-559 72 72 72 138 138 138 F F F 28 560-579 73 73 73 139 139 139 F F F 29 580-599 69 69 69 134 134 134 F F F Max gen: 73, Max flush: 139 Begin loop 37 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:09.701947405Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1476,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:09.745980853Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1476,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:09.74686462Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1476} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"0fd98d0d5d909611f8b408ccda2559d35fe117bb\",\n git_commit_timestamp: \"2024-11-08T20:05:44.000000000Z\",\n git_branch: \"mkeeter/drop-skipped-backpressure-guard\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:09.746901398Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1476} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:09.746915977Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1476} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:09.746930727Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1476} {"msg":"Crucible c2a08b44-ab59-4486-add7-5849edf2ab2f has session id: aa7d0e2f-36a5-4e67-af0b-da031c43a973","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:09.747553796Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1476,"session_id":"aa7d0e2f-36a5-4e67-af0b-da031c43a973"} {"msg":"Upstairs opts: Upstairs UUID: c2a08b44-ab59-4486-add7-5849edf2ab2f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:09.747598034Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1476,"session_id":"aa7d0e2f-36a5-4e67-af0b-da031c43a973"} {"msg":"Crucible stats registered with UUID: c2a08b44-ab59-4486-add7-5849edf2ab2f","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:09.747616083Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1476,"session_id":"aa7d0e2f-36a5-4e67-af0b-da031c43a973"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:09.779135302Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1476,"task":"crutest"} {"msg":"The guest has requested activation with gen:74","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:09.779169171Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1476} {"msg":"c2a08b44-ab59-4486-add7-5849edf2ab2f active request set","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:09.779202289Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1476,"session_id":"aa7d0e2f-36a5-4e67-af0b-da031c43a973"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:09.779219168Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1476,"client":"0","":"downstairs","session_id":"aa7d0e2f-36a5-4e67-af0b-da031c43a973"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:09.779266626Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1476,"client":"0","":"downstairs","session_id":"aa7d0e2f-36a5-4e67-af0b-da031c43a973"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:09.779301434Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1476,"client":"1","":"downstairs","session_id":"aa7d0e2f-36a5-4e67-af0b-da031c43a973"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:09.779334732Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1476,"client":"1","":"downstairs","session_id":"aa7d0e2f-36a5-4e67-af0b-da031c43a973"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:09.779368621Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1476,"client":"2","":"downstairs","session_id":"aa7d0e2f-36a5-4e67-af0b-da031c43a973"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:09.779398849Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1476,"client":"2","":"downstairs","session_id":"aa7d0e2f-36a5-4e67-af0b-da031c43a973"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:09.779429678Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1476,"":"io task","client":"2","":"downstairs","session_id":"aa7d0e2f-36a5-4e67-af0b-da031c43a973"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:09.779464006Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1476,"":"io task","client":"0","":"downstairs","session_id":"aa7d0e2f-36a5-4e67-af0b-da031c43a973"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:09.779500184Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1476,"":"io task","client":"1","":"downstairs","session_id":"aa7d0e2f-36a5-4e67-af0b-da031c43a973"} {"msg":"ds_connection connected from Ok(127.0.0.1:56334)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:09.779536023Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1476,"":"io task","client":"0","":"downstairs","session_id":"aa7d0e2f-36a5-4e67-af0b-da031c43a973"} {"msg":"ds_connection connected from Ok(127.0.0.1:40433)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:09.779572541Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1476,"":"io task","client":"1","":"downstairs","session_id":"aa7d0e2f-36a5-4e67-af0b-da031c43a973"} {"msg":"ds_connection connected from Ok(127.0.0.1:42403)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:09.779604869Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1476,"":"io task","client":"2","":"downstairs","session_id":"aa7d0e2f-36a5-4e67-af0b-da031c43a973"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:09.780378892Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1476,"client":"2","":"downstairs","session_id":"aa7d0e2f-36a5-4e67-af0b-da031c43a973"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:09.78040884Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1476,"client":"2","":"downstairs","session_id":"aa7d0e2f-36a5-4e67-af0b-da031c43a973"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:09.780423799Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1476,"client":"0","":"downstairs","session_id":"aa7d0e2f-36a5-4e67-af0b-da031c43a973"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:09.780437329Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1476,"client":"0","":"downstairs","session_id":"aa7d0e2f-36a5-4e67-af0b-da031c43a973"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:09.780450988Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1476,"client":"1","":"downstairs","session_id":"aa7d0e2f-36a5-4e67-af0b-da031c43a973"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:09.780472797Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1476,"client":"1","":"downstairs","session_id":"aa7d0e2f-36a5-4e67-af0b-da031c43a973"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:09.781091137Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1476,"client":"1","":"downstairs","session_id":"aa7d0e2f-36a5-4e67-af0b-da031c43a973"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:09.781136504Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1476,"client":"2","":"downstairs","session_id":"aa7d0e2f-36a5-4e67-af0b-da031c43a973"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:09.781171763Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1476,"client":"0","":"downstairs","session_id":"aa7d0e2f-36a5-4e67-af0b-da031c43a973"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:09.781615021Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1476,"client":"1","":"downstairs","session_id":"aa7d0e2f-36a5-4e67-af0b-da031c43a973"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:09.78164497Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1476,"client":"1","":"downstairs","session_id":"aa7d0e2f-36a5-4e67-af0b-da031c43a973"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:09.781661249Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1476,"session_id":"aa7d0e2f-36a5-4e67-af0b-da031c43a973"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:09.781737895Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1476,"client":"0","":"downstairs","session_id":"aa7d0e2f-36a5-4e67-af0b-da031c43a973"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:09.781758274Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1476,"client":"0","":"downstairs","session_id":"aa7d0e2f-36a5-4e67-af0b-da031c43a973"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:09.781771913Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1476,"session_id":"aa7d0e2f-36a5-4e67-af0b-da031c43a973"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:09.781784193Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1476,"client":"2","":"downstairs","session_id":"aa7d0e2f-36a5-4e67-af0b-da031c43a973"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:09.781797612Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1476,"client":"2","":"downstairs","session_id":"aa7d0e2f-36a5-4e67-af0b-da031c43a973"} {"msg":"[0]R flush_numbers[0..12]: [132, 128, 120, 112, 125, 115, 136, 137, 137, 139, 120, 139]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:09.781811691Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1476,"":"downstairs","":"downstairs","session_id":"aa7d0e2f-36a5-4e67-af0b-da031c43a973"} {"msg":"[0]R generation[0..12]: [68, 66, 63, 60, 64, 61, 71, 72, 72, 73, 63, 73]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:09.781825591Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1476,"":"downstairs","":"downstairs","session_id":"aa7d0e2f-36a5-4e67-af0b-da031c43a973"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:09.78183967Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1476,"":"downstairs","":"downstairs","session_id":"aa7d0e2f-36a5-4e67-af0b-da031c43a973"} {"msg":"[1]R flush_numbers[0..12]: [132, 128, 120, 112, 125, 115, 136, 137, 137, 139, 120, 139]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:09.781857029Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1476,"":"downstairs","":"downstairs","session_id":"aa7d0e2f-36a5-4e67-af0b-da031c43a973"} {"msg":"[1]R generation[0..12]: [68, 66, 63, 60, 64, 61, 71, 72, 72, 73, 63, 73]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:09.781871219Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1476,"":"downstairs","":"downstairs","session_id":"aa7d0e2f-36a5-4e67-af0b-da031c43a973"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:09.781885108Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1476,"":"downstairs","":"downstairs","session_id":"aa7d0e2f-36a5-4e67-af0b-da031c43a973"} {"msg":"[2]R flush_numbers[0..12]: [132, 128, 120, 112, 125, 115, 136, 137, 137, 139, 120, 139]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:09.781899087Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1476,"":"downstairs","":"downstairs","session_id":"aa7d0e2f-36a5-4e67-af0b-da031c43a973"} {"msg":"[2]R generation[0..12]: [68, 66, 63, 60, 64, 61, 71, 72, 72, 73, 63, 73]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:09.781912657Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1476,"":"downstairs","":"downstairs","session_id":"aa7d0e2f-36a5-4e67-af0b-da031c43a973"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:09.781926696Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1476,"":"downstairs","":"downstairs","session_id":"aa7d0e2f-36a5-4e67-af0b-da031c43a973"} {"msg":"Max found gen is 74","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:09.781939635Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1476,"":"downstairs","":"downstairs","session_id":"aa7d0e2f-36a5-4e67-af0b-da031c43a973"} {"msg":"Generation requested: 74 >= found:74","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:09.781952895Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1476,"":"downstairs","":"downstairs","session_id":"aa7d0e2f-36a5-4e67-af0b-da031c43a973"} {"msg":"Next flush: 140","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:09.781965884Z",Wait for a query_work_queue command to finish before sending IO "hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1476,"":"downstairs","":"downstairs","session_id":"aa7d0e2f-36a5-4e67-af0b-da031c43a973"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:09.7820398Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1476,"":"downstairs","":"downstairs","session_id":"aa7d0e2f-36a5-4e67-af0b-da031c43a973"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:09.782056439Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1476,"session_id":"aa7d0e2f-36a5-4e67-af0b-da031c43a973"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:09.782068749Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1476,"client":"0","":"downstairs","session_id":"aa7d0e2f-36a5-4e67-af0b-da031c43a973"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:09.782082108Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1476,"client":"1","":"downstairs","session_id":"aa7d0e2f-36a5-4e67-af0b-da031c43a973"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:09.782095108Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1476,"client":"2","":"downstairs","session_id":"aa7d0e2f-36a5-4e67-af0b-da031c43a973"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:09.782110857Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1476,"session_id":"aa7d0e2f-36a5-4e67-af0b-da031c43a973"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:09.782123386Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1476,"session_id":"aa7d0e2f-36a5-4e67-af0b-da031c43a973"} {"msg":"c2a08b44-ab59-4486-add7-5849edf2ab2f is now active with session: aa7d0e2f-36a5-4e67-af0b-da031c43a973","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:09.782136196Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1476,"session_id":"aa7d0e2f-36a5-4e67-af0b-da031c43a973"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:09.782148135Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1476,"session_id":"aa7d0e2f-36a5-4e67-af0b-da031c43a973"} {"msg":"The guest has finished waiting for activation with:74","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:09.782160474Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1476} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 243 len 20480 data: 6 6 6 6 9 02/30 Write block 544 len 12288 data: 11 7 6 03/30 Write block 168 len 16384 data: 7 7 9 8 04/30 Read block 517 len 20480 05/30 Read block 171 len 8192 06/30 Read block 119 len 8192 07/30 Read block 453 len 36864 08/30 Read block 319 len 12288 09/30 Write block 230 len 8192 data: 7 5 10/30 Write block 57 len 40960 data: 5 4 4 5 5 5 6 6 6 5 11/30 Write block 222 len 12288 data: 6 6 6 12/30 Write block 541 len 8192 data: 9 9 13/30 Read block 98 len 8192 14/30 Write block 187 len 12288 data: 10 8 7 15/30 Write block 595 len 20480 data: 5 4 4 2 3 16/30 Flush 17/30 Write block 338 len 20480 data: 9 8 8 8 9 18/30 Read block 241 len 12288 19/30 Flush 20/30 Write block 206 len 28672 data: 9 8 6 7 6 8 7 21/30 Read block 479 len 24576 22/30 Read block 178 len 20480 23/30 Read block 458 len 28672 24/30 Flush 25/30 Write block 256 len 24576 data: 4 6 6 7 6 6 26/30 Write block 133 len 24576 data: 8 8 8 8 9 9 27/30 Read block 1 len 28672 28/30 Read block 4 len 12288 29/30 Read block 530 len 4096 30/30 Write block 361 len 28672 data: 14 12 10 9 8 7 9 ---------------------------------------------------------------- Crucible gen:74 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 6 Done Done Done false 1025 Acked Write 6 Done Done Done false 1026 Acked Read 7 Done Done Done false 1027 Acked Read 3 Done Done Done false 1028 Acked Read 1 Done Done Done false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:25:09.80741969Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1476} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 68 68 68 132 132 132 F F F 1 020-039 66 66 66 128 128 128 F F F 2 040-059 74 74 74 140 140 140 F F F 3 060-079 74 74 74 140 140 140 F F F 4 080-099 64 64 64 125 125 125 F F F 5 100-119 61 61 61 115 115 115 F F F 6 120-139 71 71 71 136 136 136 T T T 7 140-159 72 72 72 137 137 137 F F F 8 160-179 74 74 74 140 140 140 F F F 9 180-199 74 74 74 140 140 140 F F F 10 200-219 74 74 74 142 142 142 F F F 11 220-239 74 74 74 140 140 140 F F F 12 240-259 74 74 74 140 140 140 T T T 13 260-279 69 69 69 134 134 134 T T T 14 280-299 72 72 72 137 137 137 F F F 15 300-319 72 72 72 137 137 137 F F F 16 320-339 74 74 74 141 141 141 F F F 17 340-359 74 74 74 141 141 141 F F F 18 360-379 73 73 73 139 139 139 T T T 19 380-399 68 68 68 132 132 132 F F F 20 400-419 64 64 64 125 125 125 F F F 21 420-439 72 72 72 137 137 137 F F F 22 440-459 72 72 72 137 137 137 F F F 23 460-479 70 70 70 135 135 135 F F F 24 480-499 73 73 73 139 139 139 F F F 25 500-519 73 73 73 139 139 139 F F F 26 520-539 73 73 73 139 139 139 F F F 27 540-559 74 74 74 140 140 140 F F F 28 560-579 73 73 73 139 139 139 F F F 29 580-599 74 74 74 140 140 140 F F F Max gen: 74, Max flush: 142 A Difference in extent metadata is expected here On loop 37 Verifying data now No change after verify Loop: 37 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 68 68 68 132 132 132 F F F 1 020-039 66 66 66 128 128 128 F F F 2 040-059 74 74 74 140 140 140 F F F 3 060-079 74 74 74 140 140 140 F F F 4 080-099 64 64 64 125 125 125 F F F 5 100-119 61 61 61 115 115 115 F F F 6 120-139 75 75 75 143 143 143 F F F 7 140-159 72 72 72 137 137 137 F F F 8 160-179 74 74 74 140 140 140 F F F 9 180-199 74 74 74 140 140 140 F F F 10 200-219 74 74 74 142 142 142 F F F 11 220-239 74 74 74 140 140 140 F F F 12 240-259 75 75 75 143 143 143 F F F 13 260-279 75 75 75 143 143 143 F F F 14 280-299 72 72 72 137 137 137 F F F 15 300-319 72 72 72 137 137 137 F F F 16 320-339 74 74 74 141 141 141 F F F 17 340-359 74 74 74 141 141 141 F F F 18 360-379 75 75 75 143 143 143 F F F 19 380-399 68 68 68 132 132 132 F F F 20 400-419 64 64 64 125 125 125 F F F 21 420-439 72 72 72 137 137 137 F F F 22 440-459 72 72 72 137 137 137 F F F 23 460-479 70 70 70 135 135 135 F F F 24 480-499 73 73 73 139 139 139 F F F 25 500-519 73 73 73 139 139 139 F F F 26 520-539 73 73 73 139 139 139 F F F 27 540-559 74 74 74 140 140 140 F F F 28 560-579 73 73 73 139 139 139 F F F 29 580-599 74 74 74 140 140 140 F F F Max gen: 75, Max flush: 143 Begin loop 38 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:14.109556037Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1484,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:14.153352836Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1484,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:14.153913479Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1484} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"0fd98d0d5d909611f8b408ccda2559d35fe117bb\",\n git_commit_timestamp: \"2024-11-08T20:05:44.000000000Z\",\n git_branch: \"mkeeter/drop-skipped-backpressure-guard\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:14.153952847Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1484} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:14.153967586Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1484} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:14.153982996Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1484} {"msg":"Crucible 345698fe-90a7-4f54-b5e5-4d6d4bf8c906 has session id: 07fd7857-5be8-41a3-bd19-9615c2401999","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:14.154415754Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1484,"session_id":"07fd7857-5be8-41a3-bd19-9615c2401999"} {"msg":"Upstairs opts: Upstairs UUID: 345698fe-90a7-4f54-b5e5-4d6d4bf8c906, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:14.154462182Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1484,"session_id":"07fd7857-5be8-41a3-bd19-9615c2401999"} {"msg":"Crucible stats registered with UUID: 345698fe-90a7-4f54-b5e5-4d6d4bf8c906","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:14.154487301Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1484,"session_id":"07fd7857-5be8-41a3-bd19-9615c2401999"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:14.186186252Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1484,"task":"crutest"} {"msg":"The guest has requested activation with gen:76","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:14.18622026Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1484} {"msg":"345698fe-90a7-4f54-b5e5-4d6d4bf8c906 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:14.186284807Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1484,"session_id":"07fd7857-5be8-41a3-bd19-9615c2401999"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:14.186314555Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1484,"client":"0","":"downstairs","session_id":"07fd7857-5be8-41a3-bd19-9615c2401999"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:14.186339414Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1484,"client":"0","":"downstairs","session_id":"07fd7857-5be8-41a3-bd19-9615c2401999"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:14.186353993Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1484,"client":"1","":"downstairs","session_id":"07fd7857-5be8-41a3-bd19-9615c2401999"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:14.186370902Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1484,"client":"1","":"downstairs","session_id":"07fd7857-5be8-41a3-bd19-9615c2401999"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:14.186396761Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1484,"client":"2","":"downstairs","session_id":"07fd7857-5be8-41a3-bd19-9615c2401999"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:14.18642815Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1484,"client":"2","":"downstairs","session_id":"07fd7857-5be8-41a3-bd19-9615c2401999"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:14.186456778Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1484,"":"io task","client":"2","":"downstairs","session_id":"07fd7857-5be8-41a3-bd19-9615c2401999"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:14.186484327Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1484,"":"io task","client":"0","":"downstairs","session_id":"07fd7857-5be8-41a3-bd19-9615c2401999"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:14.186514075Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1484,"":"io task","client":"1","":"downstairs","session_id":"07fd7857-5be8-41a3-bd19-9615c2401999"} {"msg":"ds_connection connected from Ok(127.0.0.1:53478)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:14.186542464Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1484,"":"io task","client":"2","":"downstairs","session_id":"07fd7857-5be8-41a3-bd19-9615c2401999"} {"msg":"ds_connection connected from Ok(127.0.0.1:48374)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:14.186567773Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1484,"":"io task","client":"1","":"downstairs","session_id":"07fd7857-5be8-41a3-bd19-9615c2401999"} {"msg":"ds_connection connected from Ok(127.0.0.1:36469)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:14.186592732Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1484,"":"io task","client":"0","":"downstairs","session_id":"07fd7857-5be8-41a3-bd19-9615c2401999"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:14.187344245Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1484,"client":"1","":"downstairs","session_id":"07fd7857-5be8-41a3-bd19-9615c2401999"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:14.187372434Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1484,"client":"1","":"downstairs","session_id":"07fd7857-5be8-41a3-bd19-9615c2401999"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:14.187521226Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1484,"client":"0","":"downstairs","session_id":"07fd7857-5be8-41a3-bd19-9615c2401999"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:14.187556895Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1484,"client":"0","":"downstairs","session_id":"07fd7857-5be8-41a3-bd19-9615c2401999"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:14.187587083Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1484,"client":"2","":"downstairs","session_id":"07fd7857-5be8-41a3-bd19-9615c2401999"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:14.187620772Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1484,"client":"2","":"downstairs","session_id":"07fd7857-5be8-41a3-bd19-9615c2401999"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:14.188039441Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1484,"client":"1","":"downstairs","session_id":"07fd7857-5be8-41a3-bd19-9615c2401999"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:14.188070959Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1484,"client":"0","":"downstairs","session_id":"07fd7857-5be8-41a3-bd19-9615c2401999"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:14.188354056Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1484,"client":"2","":"downstairs","session_id":"07fd7857-5be8-41a3-bd19-9615c2401999"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:14.188644211Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1484,"client":"1","":"downstairs","session_id":"07fd7857-5be8-41a3-bd19-9615c2401999"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:14.18867518Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1484,"client":"1","":"downstairs","session_id":"07fd7857-5be8-41a3-bd19-9615c2401999"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:14.188701909Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1484,"session_id":"07fd7857-5be8-41a3-bd19-9615c2401999"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:14.188717488Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1484,"client":"0","":"downstairs","session_id":"07fd7857-5be8-41a3-bd19-9615c2401999"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:14.188731497Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1484,"client":"0","":"downstairs","session_id":"07fd7857-5be8-41a3-bd19-9615c2401999"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:14.188744706Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1484,"session_id":"07fd7857-5be8-41a3-bd19-9615c2401999"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:14.188864191Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1484,"client":"2","":"downstairs","session_id":"07fd7857-5be8-41a3-bd19-9615c2401999"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:14.18888412Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1484,"client":"2","":"downstairs","session_id":"07fd7857-5be8-41a3-bd19-9615c2401999"} {"msg":"[0]R flush_numbers[0..12]: [132, 128, 140, 140, 125, 115, 143, 137, 140, 140, 142, 140]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:14.188898729Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1484,"":"downstairs","":"downstairs","session_id":"07fd7857-5be8-41a3-bd19-9615c2401999"} {"msg":"[0]R generation[0..12]: [68, 66, 74, 74, 64, 61, 75, 72, 74, 74, 74, 74]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:14.188912898Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1484,"":"downstairs","":"downstairs","session_id":"07fd7857-5be8-41a3-bd19-9615c2401999"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:14.188926808Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1484,"":"downstairs","":"downstairs","session_id":"07fd7857-5be8-41a3-bd19-9615c2401999"} {"msg":"[1]R flush_numbers[0..12]: [132, 128, 140, 140, 125, 115, 143, 137, 140, 140, 142, 140]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:14.188940597Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1484,"":"downstairs","":"downstairs","session_id":"07fd7857-5be8-41a3-bd19-9615c2401999"} {"msg":"[1]R generation[0..12]: [68, 66, 74, 74, 64, 61, 75, 72, 74, 74, 74, 74]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:14.188953956Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1484,"":"downstairs","":"downstairs","session_id":"07fd7857-5be8-41a3-bd19-9615c2401999"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:14.188967756Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1484,"":"downstairs","":"downstairs","session_id":"07fd7857-5be8-41a3-bd19-9615c2401999"} {"msg":"[2]R flush_numbers[0..12]: [132, 128, 140, 140, 125, 115, 143, 137, 140, 140, 142, 140]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:14.188984715Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1484,"":"downstairs","":"downstairs","session_id":"07fd7857-5be8-41a3-bd19-9615c2401999"} {"msg":"[2]R generation[0..12]: [68, 66, 74, 74, 64, 61, 75, 72, 74, 74, 74, 74]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:14.189002714Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1484,"":"downstairs","":"downstairs","session_id":"07fd7857-5be8-41a3-bd19-9615c2401999"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:14.189027993Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1484,"":"downstairs","":"downstairs","session_id":"07fd7857-5be8-41a3-bd19-9615c2401999"} {"msg":"Max found gen is 76","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:14.189043932Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1484,"":"downstairs","":"downstairs","session_id":"07fd7857-5be8-41a3-bd19-9615c2401999"} {"msg":"Generation requested: 76 >= found:76","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:14.189057351Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1484,"":"downstairs","":"downstairs","session_id":"07fd7857-5be8-41a3-bd19-9615c2401999"} {"msg":"Next flush: 144","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:14.189070201Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1484,"":"downstairs","":"downstairs","session_id":"07fd7857-5be8-41a3-bd19-9615c2401999"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-08T20:25:14.18908304Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1484,"":"downstairs","":"downstairs","session_id":"07fd7857-5be8-41a3-bd19-9615c2401999"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:14.189102859Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1484,"session_id":"07fd7857-5be8-41a3-bd19-9615c2401999"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:14.189114828Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1484,"client":"0","":"downstairs","session_id":"07fd7857-5be8-41a3-bd19-9615c2401999"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:14.189127928Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1484,"client":"1","":"downstairs","session_id":"07fd7857-5be8-41a3-bd19-9615c2401999"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:14.189142097Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1484,"client":"2","":"downstairs","session_id":"07fd7857-5be8-41a3-bd19-9615c2401999"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:14.189155466Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1484,"session_id":"07fd7857-5be8-41a3-bd19-9615c2401999"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:14.189167746Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1484,"session_id":"07fd7857-5be8-41a3-bd19-9615c2401999"} {"msg":"345698fe-90a7-4f54-b5e5-4d6d4bf8c906 is now active with session: 07fd7857-5be8-41a3-bd19-9615c2401999","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:14.189180615Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1484,"session_id":"07fd7857-5be8-41a3-bd19-9615c2401999"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:14.189192695Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1484,"session_id":"Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 07fd7857-5be8-41a3-bd19-9615c2401999"} {"msg":"The guest has finished waiting for activation with:76","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:14.189211344Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1484} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 79 len 40960 data: 6 5 4 5 6 5 7 7 7 7 02/30 Read block 413 len 28672 03/30 Read block 554 len 24576 04/30 Write block 349 len 20480 data: 8 7 7 7 10 05/30 Read block 110 len 16384 06/30 Flush 07/30 Write block 12 len 24576 data: 9 8 6 8 8 7 08/30 Read block 243 len 36864 09/30 Write block 226 len 16384 data: 7 9 7 7 10/30 Read block 325 len 8192 11/30 Read block 341 len 16384 12/30 Flush 13/30 Flush 14/30 Write block 270 len 24576 data: 7 7 7 6 8 8 15/30 Read block 517 len 28672 16/30 Write block 28 len 4096 data: 6 17/30 Read block 505 len 32768 18/30 Read block 392 len 24576 19/30 Read block 367 len 40960 20/30 Write block 421 len 24576 data: 7 6 6 5 6 6 21/30 Read block 246 len 8192 22/30 Read block 122 len 20480 23/30 Write block 94 len 40960 data: 3 4 3 2 2 3 5 5 5 6 24/30 Read block 21 len 16384 25/30 Write block 363 len 28672 data: 11 10 9 8 10 11 9 26/30 Read block 538 len 36864 27/30 Read block 23 len 32768 28/30 Write block 266 len 8192 data: 8 5 29/30 Read block 332 len 32768 30/30 Write block 410 len 4096 data: 8 ---------------------------------------------------------------- Crucible gen:76 GIO:true work queues: Upstairs:0 downstairs:17 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1013 Acked Write 6 Done Done Done false 1014 Acked Read 7 Done Done Done false 1015 Acked Write 1 Done Done Done false 1016 Acked Read 8 Done Done Done false 1017 Acked Read 6 Done Done Done false 1018 Acked Read 10 Done Done Done false 1019 Acked Write 6 Done Done Done false 1020 Acked Read 2 Done Done Done false 1021 Acked Read 5 Done Done Done false 1022 Acked Write 10 Done Done Done false 1023 Acked Read 4 Done Done Done false 1024 Acked Write 7 Done Done Done false 1025 Acked Read 9 Done Done Done false 1026 Acked Read 8 Done Done Done false 1027 Acked Write 2 Done Done Done false 1028 Acked Read 8 Done Sent Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 2 2 5 Done 16 15 15 46 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1012 1012 1012 Downstairs last five completed: 1012 1011 1010 1009 1008 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:25:14.21547885Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1484} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 76 76 76 145 145 145 F F F 1 020-039 66 66 66 128 128 128 T T T 2 040-059 74 74 74 140 140 140 F F F 3 060-079 76 76 76 144 144 144 F F F 4 080-099 76 76 76 144 144 144 T T T 5 100-119 61 61 61 115 115 115 T T T 6 120-139 75 75 75 143 143 143 F F F 7 140-159 72 72 72 137 137 137 F F F 8 160-179 74 74 74 140 140 140 F F F 9 180-199 74 74 74 140 140 140 F F F 10 200-219 74 74 74 142 142 142 F F F 11 220-239 76 76 76 145 145 145 F F F 12 240-259 75 75 75 143 143 143 F F F 13 260-279 75 75 75 143 143 143 T T T 14 280-299 72 72 72 137 137 137 F F F 15 300-319 72 72 72 137 137 137 F F F 16 320-339 74 74 74 141 141 141 F F F 17 340-359 76 76 76 144 144 144 F F F 18 360-379 75 75 75 143 143 143 T T T 19 380-399 68 68 68 132 132 132 F F F 20 400-419 64 64 64 125 125 125 T T T 21 420-439 72 72 72 137 137 137 T T T 22 440-459 72 72 72 137 137 137 F F F 23 460-479 70 70 70 135 135 135 F F F 24 480-499 73 73 73 139 139 139 F F F 25 500-519 73 73 73 139 139 139 F F F 26 520-539 73 73 73 139 139 139 F F F 27 540-559 74 74 74 140 140 140 F F F 28 560-579 73 73 73 139 139 139 F F F 29 580-599 74 74 74 140 140 140 F F F Max gen: 76, Max flush: 145 A Difference in extent metadata is expected here On loop 38 Verifying data now No change after verify Loop: 38 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 76 76 76 145 145 145 F F F 1 020-039 77 77 77 146 146 146 F F F 2 040-059 74 74 74 140 140 140 F F F 3 060-079 76 76 76 144 144 144 F F F 4 080-099 77 77 77 146 146 146 F F F 5 100-119 77 77 77 146 146 146 F F F 6 120-139 75 75 75 143 143 143 F F F 7 140-159 72 72 72 137 137 137 F F F 8 160-179 74 74 74 140 140 140 F F F 9 180-199 74 74 74 140 140 140 F F F 10 200-219 74 74 74 142 142 142 F F F 11 220-239 76 76 76 145 145 145 F F F 12 240-259 75 75 75 143 143 143 F F F 13 260-279 77 77 77 146 146 146 F F F 14 280-299 72 72 72 137 137 137 F F F 15 300-319 72 72 72 137 137 137 F F F 16 320-339 74 74 74 141 141 141 F F F 17 340-359 76 76 76 144 144 144 F F F 18 360-379 77 77 77 146 146 146 F F F 19 380-399 68 68 68 132 132 132 F F F 20 400-419 77 77 77 146 146 146 F F F 21 420-439 77 77 77 146 146 146 F F F 22 440-459 72 72 72 137 137 137 F F F 23 460-479 70 70 70 135 135 135 F F F 24 480-499 73 73 73 139 139 139 F F F 25 500-519 73 73 73 139 139 139 F F F 26 520-539 73 73 73 139 139 139 F F F 27 540-559 74 74 74 140 140 140 F F F 28 560-579 73 73 73 139 139 139 F F F 29 580-599 74 74 74 140 140 140 F F F Max gen: 77, Max flush: 146 Begin loop 39 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:28.567341047Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1492,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:28.611411923Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1492,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:28.612046062Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1492} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"0fd98d0d5d909611f8b408ccda2559d35fe117bb\",\n git_commit_timestamp: \"2024-11-08T20:05:44.000000000Z\",\n git_branch: \"mkeeter/drop-skipped-backpressure-guard\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:28.61208552Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1492} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:28.612104349Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1492} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:28.612119349Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1492} {"msg":"Crucible d57bd5e3-5a19-4db9-93ad-6664db561e55 has session id: d1bdd4aa-82b5-4d94-9326-11a92da92211","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:28.612652032Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1492,"session_id":"d1bdd4aa-82b5-4d94-9326-11a92da92211"} {"msg":"Upstairs opts: Upstairs UUID: d57bd5e3-5a19-4db9-93ad-6664db561e55, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:28.61270164Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1492,"session_id":"d1bdd4aa-82b5-4d94-9326-11a92da92211"} {"msg":"Crucible stats registered with UUID: d57bd5e3-5a19-4db9-93ad-6664db561e55","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:28.612729229Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1492,"session_id":"d1bdd4aa-82b5-4d94-9326-11a92da92211"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:28.643639728Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1492,"task":"crutest"} {"msg":"The guest has requested activation with gen:78","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:28.643674806Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1492} {"msg":"d57bd5e3-5a19-4db9-93ad-6664db561e55 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:28.643748923Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1492,"session_id":"d1bdd4aa-82b5-4d94-9326-11a92da92211"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:28.643772411Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1492,"client":"0","":"downstairs","session_id":"d1bdd4aa-82b5-4d94-9326-11a92da92211"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:28.64380095Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1492,"client":"0","":"downstairs","session_id":"d1bdd4aa-82b5-4d94-9326-11a92da92211"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:28.643819039Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1492,"client":"1","":"downstairs","session_id":"d1bdd4aa-82b5-4d94-9326-11a92da92211"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:28.643836908Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1492,"client":"1","":"downstairs","session_id":"d1bdd4aa-82b5-4d94-9326-11a92da92211"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:28.643854107Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1492,"client":"2","":"downstairs","session_id":"d1bdd4aa-82b5-4d94-9326-11a92da92211"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:28.643871397Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1492,"client":"2","":"downstairs","session_id":"d1bdd4aa-82b5-4d94-9326-11a92da92211"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:28.643897645Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1492,"":"io task","client":"2","":"downstairs","session_id":"d1bdd4aa-82b5-4d94-9326-11a92da92211"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:28.643933814Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1492,"":"io task","client":"0","":"downstairs","session_id":"d1bdd4aa-82b5-4d94-9326-11a92da92211"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:28.643964492Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1492,"":"io task","client":"1","":"downstairs","session_id":"d1bdd4aa-82b5-4d94-9326-11a92da92211"} {"msg":"ds_connection connected from Ok(127.0.0.1:42112)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:28.644050708Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1492,"":"io task","client":"0","":"downstairs","session_id":"d1bdd4aa-82b5-4d94-9326-11a92da92211"} {"msg":"ds_connection connected from Ok(127.0.0.1:40031)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:28.644094796Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1492,"":"io task","client":"1","":"downstairs","session_id":"d1bdd4aa-82b5-4d94-9326-11a92da92211"} {"msg":"ds_connection connected from Ok(127.0.0.1:43474)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:28.644173102Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1492,"":"io task","client":"2","":"downstairs","session_id":"d1bdd4aa-82b5-4d94-9326-11a92da92211"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:28.644764633Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1492,"client":"0","":"downstairs","session_id":"d1bdd4aa-82b5-4d94-9326-11a92da92211"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:28.644793181Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1492,"client":"0","":"downstairs","session_id":"d1bdd4aa-82b5-4d94-9326-11a92da92211"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:28.644807941Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1492,"client":"1","":"downstairs","session_id":"d1bdd4aa-82b5-4d94-9326-11a92da92211"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:28.6448218Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1492,"client":"1","":"downstairs","session_id":"d1bdd4aa-82b5-4d94-9326-11a92da92211"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:28.644958544Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1492,"client":"2","":"downstairs","session_id":"d1bdd4aa-82b5-4d94-9326-11a92da92211"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:28.64503049Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1492,"client":"2","":"downstairs","session_id":"d1bdd4aa-82b5-4d94-9326-11a92da92211"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:28.645411761Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1492,"client":"1","":"downstairs","session_id":"d1bdd4aa-82b5-4d94-9326-11a92da92211"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:28.645447729Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1492,"client":"0","":"downstairs","session_id":"d1bdd4aa-82b5-4d94-9326-11a92da92211"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:28.645715407Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1492,"client":"2","":"downstairs","session_id":"d1bdd4aa-82b5-4d94-9326-11a92da92211"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:28.646078599Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1492,"client":"0","":"downstairs","session_id":"d1bdd4aa-82b5-4d94-9326-11a92da92211"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:28.646122996Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1492,"client":"0","":"downstairs","session_id":"d1bdd4aa-82b5-4d94-9326-11a92da92211"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:28.646150975Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1492,"session_id":"d1bdd4aa-82b5-4d94-9326-11a92da92211"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:28.646166165Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1492,"client":"1","":"downstairs","session_id":"d1bdd4aa-82b5-4d94-9326-11a92da92211"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:28.646180154Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1492,"client":"1","":"downstairs","session_id":"d1bdd4aa-82b5-4d94-9326-11a92da92211"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:28.646193903Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1492,"session_id":"d1bdd4aa-82b5-4d94-9326-11a92da92211"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:28.646206412Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1492,"client":"2","":"downstairs","session_id":"d1bdd4aa-82b5-4d94-9326-11a92da92211"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:28.646219932Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1492,"client":"2","":"downstairs","session_id":"d1bdd4aa-82b5-4d94-9326-11a92da92211"} {"msg":"[0]R flush_numbers[0..12]: [145, 146, 140, 144, 146, 146, 143, 137, 140, 140, 142, 145]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:28.646233921Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1492,"":"downstairs","":"downstairs","session_id":"d1bdd4aa-82b5-4d94-9326-11a92da92211"} {"msg":"[0]R generation[0..12]: [76, 77, 74, 76, 77, 77, 75, 72, 74, 74, 74, 76]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:28.64624775Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1492,"":"downstairs","":"downstairs","session_id":"d1bdd4aa-82b5-4d94-9326-11a92da92211"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:28.64626201Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1492,"":"downstairs","":"downstairs","session_id":"d1bdd4aa-82b5-4d94-9326-11a92da92211"} {"msg":"[1]R flush_numbers[0..12]: [145, 146, 140, 144, 146, 146, 143, 137, 140, 140, 142, 145]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:28.646275869Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1492,"":"downstairs","":"downstairs","session_id":"d1bdd4aa-82b5-4d94-9326-11a92da92211"} {"msg":"[1]R generation[0..12]: [76, 77, 74, 76, 77, 77, 75, 72, 74, 74, 74, 76]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:28.646289459Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1492,"":"downstairs","":"downstairs","session_id":"d1bdd4aa-82b5-4d94-9326-11a92da92211"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:28.646303208Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1492,"":"downstairs","":"downstairs","session_id":"d1bdd4aa-82b5-4d94-9326-11a92da92211"} {"msg":"[2]R flush_numbers[0..12]: [145, 146, 140, 144, 146, 146, 143, 137, 140, 140, 142, 145]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:28.646317247Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1492,"":"downstairs","":"downstairs","session_id":"d1bdd4aa-82b5-4d94-9326-11a92da92211"} {"msg":"[2]R generation[0..12]: [76, 77, 74, 76, 77, 77, 75, 72, 74, 74, 74, 76]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:28.646332876Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1492,"":"downstairs","":"downstairs","session_id":"d1bdd4aa-82b5-4d94-9326-11a92da92211"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:28.646347136Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1492,"":"downstairs","":"downstairs","session_id":"d1bdd4aa-82b5-4d94-9326-11a92da92211"} {"msg":"Max found gen is 78","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:28.646360315Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1492,"":"downstairs","":"downstairs","session_id":"d1bdd4aa-82b5-4d94-9326-11a92da92211"} {"msg":"Generation requested: 78 >= found:78","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:28.646373614Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1492,"":"downstairs","":"downstairs","session_id":"d1bdd4aa-82b5-4d94-9326-11a92da92211"} {"msg":"Next flush: 147","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:28.646386724Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1492,"":"downstairs","":"downstairs","session_id":"d1bdd4aa-82b5-4d94-9326-11a92da92211"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:28.646399663Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1492,"":"downstairs","":"downstairs","session_id":"d1bdd4aa-82b5-4d94-9326-11a92da92211"} {Wait for a query_work_queue command to finish before sending IO "msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:28.646420762Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1492,"session_id":"d1bdd4aa-82b5-4d94-9326-11a92da92211"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:28.646433192Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1492,"client":"0","":"downstairs","session_id":"d1bdd4aa-82b5-4d94-9326-11a92da92211"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:28.646446521Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1492,"client":"1","":"downstairs","session_id":"d1bdd4aa-82b5-4d94-9326-11a92da92211"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:28.64645985Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1492,"client":"2","":"downstairs","session_id":"d1bdd4aa-82b5-4d94-9326-11a92da92211"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:28.64647317Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1492,"session_id":"d1bdd4aa-82b5-4d94-9326-11a92da92211"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:28.646485479Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1492,"session_id":"d1bdd4aa-82b5-4d94-9326-11a92da92211"} {"msg":"d57bd5e3-5a19-4db9-93ad-6664db561e55 is now active with session: d1bdd4aa-82b5-4d94-9326-11a92da92211","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:28.646498258Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1492,"session_id":"d1bdd4aa-82b5-4d94-9326-11a92da92211"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:28.646510268Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1492,"session_id":"d1bdd4aa-82b5-4d94-9326-11a92da92211"} {"msg":"The guest has finished waiting for activation with:78","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:28.646522467Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1492} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Flush 03/30 Flush 04/30 Write block 553 len 32768 data: 4 5 6 6 6 6 7 6 05/30 Read block 441 len 4096 06/30 Read block 445 len 16384 07/30 Flush 08/30 Write block 384 len 4096 data: 5 09/30 Write block 575 len 40960 data: 8 7 7 6 5 6 6 6 9 10 10/30 Read block 460 len 36864 11/30 Flush 12/30 Write block 165 len 28672 data: 4 4 3 8 8 10 9 13/30 Read block 563 len 28672 14/30 Read block 379 len 4096 15/30 Read block 429 len 28672 16/30 Write block 526 len 40960 data: 4 5 6 5 4 4 3 4 4 7 17/30 Read block 567 len 4096 18/30 Write block 190 len 12288 data: 11 8 8 19/30 Write block 299 len 28672 data: 8 8 9 8 9 9 7 20/30 Read block 122 len 28672 21/30 Write block 222 len 36864 data: 7 7 7 7 8 10 8 8 8 22/30 Read block 183 len 12288 23/30 Write block 119 len 40960 data: 7 8 8 9 10 9 6 5 4 4 24/30 Write block 441 len 8192 data: 10 9 25/30 Read block 274 len 12288 26/30 Flush 27/30 Write block 489 len 20480 data: 7 7 5 4 6 28/30 Read block 185 len 36864 29/30 Write block 172 len 24576 data: 5 4 4 5 3 5 30/30 Write block 38 len 32768 data: 6 7 7 5 6 7 7 6 ---------------------------------------------------------------- Crucible gen:78 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 5 Done Done Done false 1027 Acked Read 9 Done Done Done false 1028 Acked Write 6 Sent Sent Sent false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 2 2 2 6 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:25:28.677054815Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1492} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 76 76 76 145 145 145 F F F 1 020-039 77 77 77 146 146 146 T T T 2 040-059 74 74 74 140 140 140 T T T 3 060-079 76 76 76 144 144 144 F F F 4 080-099 77 77 77 146 146 146 F F F 5 100-119 78 78 78 152 152 152 F F F 6 120-139 78 78 78 152 152 152 F F F 7 140-159 72 72 72 137 137 137 F F F 8 160-179 78 78 78 152 152 152 T T T 9 180-199 78 78 78 152 152 152 F F F 10 200-219 74 74 74 142 142 142 F F F 11 220-239 78 78 78 152 152 152 F F F 12 240-259 75 75 75 143 143 143 F F F 13 260-279 77 77 77 146 146 146 F F F 14 280-299 78 78 78 152 152 152 F F F 15 300-319 78 78 78 152 152 152 F F F 16 320-339 74 74 74 141 141 141 F F F 17 340-359 76 76 76 144 144 144 F F F 18 360-379 77 77 77 146 146 146 F F F 19 380-399 78 78 78 151 151 151 F F F 20 400-419 77 77 77 146 146 146 F F F 21 420-439 77 77 77 146 146 146 F F F 22 440-459 78 78 78 152 152 152 F F F 23 460-479 70 70 70 135 135 135 F F F 24 480-499 73 73 73 139 139 139 T T T 25 500-519 73 73 73 139 139 139 F F F 26 520-539 78 78 78 152 152 152 F F F 27 540-559 78 78 78 150 150 150 F F F 28 560-579 78 78 78 151 151 151 F F F 29 580-599 78 78 78 151 151 151 F F F Max gen: 78, Max flush: 152 A Difference in extent metadata is expected here On loop 39 Verifying data now No change after verify Loop: 39 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 76 76 76 145 145 145 F F F 1 020-039 79 79 79 153 153 153 F F F 2 040-059 79 79 79 153 153 153 F F F 3 060-079 76 76 76 144 144 144 F F F 4 080-099 77 77 77 146 146 146 F F F 5 100-119 78 78 78 152 152 152 F F F 6 120-139 78 78 78 152 152 152 F F F 7 140-159 72 72 72 137 137 137 F F F 8 160-179 79 79 79 153 153 153 F F F 9 180-199 78 78 78 152 152 152 F F F 10 200-219 74 74 74 142 142 142 F F F 11 220-239 78 78 78 152 152 152 F F F 12 240-259 75 75 75 143 143 143 F F F 13 260-279 77 77 77 146 146 146 F F F 14 280-299 78 78 78 152 152 152 F F F 15 300-319 78 78 78 152 152 152 F F F 16 320-339 74 74 74 141 141 141 F F F 17 340-359 76 76 76 144 144 144 F F F 18 360-379 77 77 77 146 146 146 F F F 19 380-399 78 78 78 151 151 151 F F F 20 400-419 77 77 77 146 146 146 F F F 21 420-439 77 77 77 146 146 146 F F F 22 440-459 78 78 78 152 152 152 F F F 23 460-479 70 70 70 135 135 135 F F F 24 480-499 79 79 79 153 153 153 F F F 25 500-519 73 73 73 139 139 139 F F F 26 520-539 78 78 78 152 152 152 F F F 27 540-559 78 78 78 150 150 150 F F F 28 560-579 78 78 78 151 151 151 F F F 29 580-599 78 78 78 151 151 151 F F F Max gen: 79, Max flush: 153 Begin loop 40 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:32.989565144Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1500,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:33.034517217Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1500,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:33.035122587Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1500} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"0fd98d0d5d909611f8b408ccda2559d35fe117bb\",\n git_commit_timestamp: \"2024-11-08T20:05:44.000000000Z\",\n git_branch: \"mkeeter/drop-skipped-backpressure-guard\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:33.035169945Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1500} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:33.035188494Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1500} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:33.035207783Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1500} {"msg":"Crucible 01a25872-f279-4247-9f92-32a387b1194b has session id: 6ee5a070-4bea-469d-af87-248464630eb8","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:33.035703909Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1500,"session_id":"6ee5a070-4bea-469d-af87-248464630eb8"} {"msg":"Upstairs opts: Upstairs UUID: 01a25872-f279-4247-9f92-32a387b1194b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:33.035754136Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1500,"session_id":"6ee5a070-4bea-469d-af87-248464630eb8"} {"msg":"Crucible stats registered with UUID: 01a25872-f279-4247-9f92-32a387b1194b","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:33.035786265Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1500,"session_id":"6ee5a070-4bea-469d-af87-248464630eb8"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:33.066947872Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1500,"task":"crutest"} {"msg":"The guest has requested activation with gen:80","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:33.0669805Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1500} {"msg":"01a25872-f279-4247-9f92-32a387b1194b active request set","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:33.067030938Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1500,"session_id":"6ee5a070-4bea-469d-af87-248464630eb8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:33.067050837Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1500,"client":"0","":"downstairs","session_id":"6ee5a070-4bea-469d-af87-248464630eb8"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:33.067078526Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1500,"client":"0","":"downstairs","session_id":"6ee5a070-4bea-469d-af87-248464630eb8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:33.067096455Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1500,"client":"1","":"downstairs","session_id":"6ee5a070-4bea-469d-af87-248464630eb8"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:33.067113414Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1500,"client":"1","":"downstairs","session_id":"6ee5a070-4bea-469d-af87-248464630eb8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:33.067130093Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1500,"client":"2","":"downstairs","session_id":"6ee5a070-4bea-469d-af87-248464630eb8"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:33.067147662Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1500,"client":"2","":"downstairs","session_id":"6ee5a070-4bea-469d-af87-248464630eb8"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:33.067163932Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1500,"":"io task","client":"2","":"downstairs","session_id":"6ee5a070-4bea-469d-af87-248464630eb8"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:33.067267656Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1500,"":"io task","client":"0","":"downstairs","session_id":"6ee5a070-4bea-469d-af87-248464630eb8"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:33.067307904Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1500,"":"io task","client":"1","":"downstairs","session_id":"6ee5a070-4bea-469d-af87-248464630eb8"} {"msg":"ds_connection connected from Ok(127.0.0.1:48335)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:33.067344633Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1500,"":"io task","client":"2","":"downstairs","session_id":"6ee5a070-4bea-469d-af87-248464630eb8"} {"msg":"ds_connection connected from Ok(127.0.0.1:34943)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:33.067485876Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1500,"":"io task","client":"1","":"downstairs","session_id":"6ee5a070-4bea-469d-af87-248464630eb8"} {"msg":"ds_connection connected from Ok(127.0.0.1:35473)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:33.067520794Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1500,"":"io task","client":"0","":"downstairs","session_id":"6ee5a070-4bea-469d-af87-248464630eb8"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:33.068097266Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1500,"client":"0","":"downstairs","session_id":"6ee5a070-4bea-469d-af87-248464630eb8"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:33.068139144Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1500,"client":"0","":"downstairs","session_id":"6ee5a070-4bea-469d-af87-248464630eb8"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:33.068170492Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1500,"client":"1","":"downstairs","session_id":"6ee5a070-4bea-469d-af87-248464630eb8"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:33.068200701Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1500,"client":"1","":"downstairs","session_id":"6ee5a070-4bea-469d-af87-248464630eb8"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:33.068232309Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1500,"client":"2","":"downstairs","session_id":"6ee5a070-4bea-469d-af87-248464630eb8"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:33.068261458Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1500,"client":"2","":"downstairs","session_id":"6ee5a070-4bea-469d-af87-248464630eb8"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:33.068786532Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1500,"client":"0","":"downstairs","session_id":"6ee5a070-4bea-469d-af87-248464630eb8"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:33.068865408Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1500,"client":"2","":"downstairs","session_id":"6ee5a070-4bea-469d-af87-248464630eb8"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:33.068907466Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1500,"client":"1","":"downstairs","session_id":"6ee5a070-4bea-469d-af87-248464630eb8"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:33.069301647Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1500,"client":"0","":"downstairs","session_id":"6ee5a070-4bea-469d-af87-248464630eb8"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:33.069343085Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1500,"client":"0","":"downstairs","session_id":"6ee5a070-4bea-469d-af87-248464630eb8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:33.069370913Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1500,"session_id":"6ee5a070-4bea-469d-af87-248464630eb8"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:33.069385973Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1500,"client":"2","":"downstairs","session_id":"6ee5a070-4bea-469d-af87-248464630eb8"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:33.069400262Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1500,"client":"2","":"downstairs","session_id":"6ee5a070-4bea-469d-af87-248464630eb8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:33.069417011Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1500,"session_id":"6ee5a070-4bea-469d-af87-248464630eb8"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:33.06943866Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1500,"client":"1","":"downstairs","session_id":"6ee5a070-4bea-469d-af87-248464630eb8"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:33.06945272Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1500,"client":"1","":"downstairs","session_id":"6ee5a070-4bea-469d-af87-248464630eb8"} {"msg":"[0]R flush_numbers[0..12]: [145, 153, 153, 144, 146, 152, 152, 137, 153, 152, 142, 152]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:33.069493478Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1500,"":"downstairs","":"downstairs","session_id":"6ee5a070-4bea-469d-af87-248464630eb8"} {"msg":"[0]R generation[0..12]: [76, 79, 79, 76, 77, 78, 78, 72, 79, 78, 74, 78]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:33.069510667Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1500,"":"downstairs","":"downstairs","session_id":"6ee5a070-4bea-469d-af87-248464630eb8"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:33.069524866Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1500,"":"downstairs","":"downstairs","session_id":"6ee5a070-4bea-469d-af87-248464630eb8"} {"msg":"[1]R flush_numbers[0..12]: [145, 153, 153, 144, 146, 152, 152, 137, 153, 152, 142, 152]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:33.069538815Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1500,"":"downstairs","":"downstairs","session_id":"6ee5a070-4bea-469d-af87-248464630eb8"} {"msg":"[1]R generation[0..12]: [76, 79, 79, 76, 77, 78, 78, 72, 79, 78, 74, 78]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:33.069552365Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1500,"":"downstairs","":"downstairs","session_id":"6ee5a070-4bea-469d-af87-248464630eb8"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:33.069566154Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1500,"":"downstairs","":"downstairs","session_id":"6ee5a070-4bea-469d-af87-248464630eb8"} {"msg":"[2]R flush_numbers[0..12]: [145, 153, 153, 144, 146, 152, 152, 137, 153, 152, 142, 152]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:33.069579893Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1500,"":"downstairs","":"downstairs","session_id":"6ee5a070-4bea-469d-af87-248464630eb8"} {"msg":"[2]R generation[0..12]: [76, 79, 79, 76, 77, 78, 78, 72, 79, 78, 74, 78]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:33.069593373Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1500,"":"downstairs","":"downstairs","session_id":"6ee5a070-4bea-469d-af87-248464630eb8"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:33.069607272Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1500,"":"downstairs","":"downstairs","session_id":"6ee5a070-4bea-469d-af87-248464630eb8"} {"msg":"Max found gen is 80","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:33.069620531Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1500,"":"downstairs","":"downstairs","session_id":"6ee5a070-4bea-469d-af87-248464630eb8"} {"msg":"Generation requested: 80 >= found:80","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:33.069633581Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1500,"":"downstairs","":"downstairs","session_id":"6ee5a070-4bea-469d-af87-248464630eb8"} {"msg":"Next flush: 154","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:33.06964933Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1500,"":"downstairs","":"downstairs","session_id":"6ee5a070-4bea-469d-af87-248464630eb8"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:33.069662489Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1500,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO 6ee5a070-4bea-469d-af87-248464630eb8"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:33.069699277Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1500,"session_id":"6ee5a070-4bea-469d-af87-248464630eb8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:33.069712647Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1500,"client":"0","":"downstairs","session_id":"6ee5a070-4bea-469d-af87-248464630eb8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:33.069726036Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1500,"client":"1","":"downstairs","session_id":"6ee5a070-4bea-469d-af87-248464630eb8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:33.069739166Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1500,"client":"2","":"downstairs","session_id":"6ee5a070-4bea-469d-af87-248464630eb8"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:33.069752515Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1500,"session_id":"6ee5a070-4bea-469d-af87-248464630eb8"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:33.069764704Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1500,"session_id":"6ee5a070-4bea-469d-af87-248464630eb8"} {"msg":"01a25872-f279-4247-9f92-32a387b1194b is now active with session: 6ee5a070-4bea-469d-af87-248464630eb8","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:33.069777364Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1500,"session_id":"6ee5a070-4bea-469d-af87-248464630eb8"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:33.069789303Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1500,"session_id":"Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 6ee5a070-4bea-469d-af87-248464630eb8"} {"msg":"The guest has finished waiting for activation with:80","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:33.069805522Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1500} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 484 len 8192 data: 8 6 02/30 Flush 03/30 Write block 252 len 8192 data: 12 10 04/30 Read block 137 len 24576 05/30 Write block 430 len 16384 data: 9 8 8 6 06/30 Flush 07/30 Write block 508 len 28672 data: 6 6 7 8 7 8 7 08/30 Write block 315 len 40960 data: 9 8 6 7 9 7 6 7 7 6 09/30 Read block 488 len 40960 10/30 Write block 68 len 4096 data: 6 11/30 Write block 127 len 24576 data: 5 5 4 6 5 6 12/30 Read block 575 len 4096 13/30 Write block 126 len 8192 data: 6 6 14/30 Write block 541 len 32768 data: 10 10 10 12 8 7 4 6 15/30 Read block 437 len 4096 16/30 Write block 395 len 36864 data: 7 7 9 9 9 7 6 5 5 17/30 Read block 475 len 32768 18/30 Write block 437 len 36864 data: 9 9 8 8 11 10 7 7 7 19/30 Read block 214 len 28672 20/30 Write block 271 len 24576 data: 8 8 7 9 9 9 21/30 Read block 153 len 28672 22/30 Flush 23/30 Write block 21 len 16384 data: 9 10 10 11 24/30 Write block 128 len 40960 data: 6 5 7 6 7 9 9 9 9 10 25/30 Read block 324 len 28672 26/30 Write block 489 len 12288 data: 8 8 6 27/30 Read block 302 len 40960 28/30 Write block 295 len 12288 data: 5 3 4 29/30 Write block 562 len 32768 data: 4 4 5 6 5 5 6 7 30/30 Write block 498 len 16384 data: 9 8 7 8 ---------------------------------------------------------------- Crucible gen:80 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 4 Done Done Done false 1023 Acked Write 10 Done Done Done false 1024 Acked Read 7 Done Done Done false 1025 Acked Write 3 Done Done Done false 1026 Acked Read 10 Done Done Done false 1027 Acked Write 3 Sent Done Done false 1028 Acked Write 8 Sent Sent Sent false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 2 2 7 Done 5 6 6 17 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1021 1021 1021 Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:25:33.098782536Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1500} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 76 76 76 145 145 145 F F F 1 020-039 79 79 79 153 153 153 T T T 2 040-059 79 79 79 153 153 153 F F F 3 060-079 80 80 80 156 156 156 F F F 4 080-099 77 77 77 146 146 146 F F F 5 100-119 78 78 78 152 152 152 F F F 6 120-139 80 80 80 156 156 156 T T T 7 140-159 72 72 72 137 137 137 F F F 8 160-179 79 79 79 153 153 153 F F F 9 180-199 78 78 78 152 152 152 F F F 10 200-219 74 74 74 142 142 142 F F F 11 220-239 78 78 78 152 152 152 F F F 12 240-259 80 80 80 155 155 155 F F F 13 260-279 80 80 80 156 156 156 F F F 14 280-299 78 78 78 152 152 152 T T T 15 300-319 80 80 80 156 156 156 F F F 16 320-339 80 80 80 156 156 156 F F F 17 340-359 76 76 76 144 144 144 F F F 18 360-379 77 77 77 146 146 146 F F F 19 380-399 80 80 80 156 156 156 F F F 20 400-419 80 80 80 156 156 156 F F F 21 420-439 80 80 80 156 156 156 F F F 22 440-459 80 80 80 156 156 156 F F F 23 460-479 70 70 70 135 135 135 F F F 24 480-499 80 80 80 154 154 154 T T T 25 500-519 80 80 80 156 156 156 T T T 26 520-539 78 78 78 152 152 152 F F F 27 540-559 80 80 80 156 156 156 F F F 28 560-579 78 78 78 151 151 151 T T T 29 580-599 78 78 78 151 151 151 F F F Max gen: 80, Max flush: 156 A Difference in extent metadata is expected here On loop 40 Verifying data now No change after verify Loop: 40 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 76 76 76 145 145 145 F F F 1 020-039 81 81 81 157 157 157 F F F 2 040-059 79 79 79 153 153 153 F F F 3 060-079 80 80 80 156 156 156 F F F 4 080-099 77 77 77 146 146 146 F F F 5 100-119 78 78 78 152 152 152 F F F 6 120-139 81 81 81 157 157 157 F F F 7 140-159 72 72 72 137 137 137 F F F 8 160-179 79 79 79 153 153 153 F F F 9 180-199 78 78 78 152 152 152 F F F 10 200-219 74 74 74 142 142 142 F F F 11 220-239 78 78 78 152 152 152 F F F 12 240-259 80 80 80 155 155 155 F F F 13 260-279 80 80 80 156 156 156 F F F 14 280-299 81 81 81 157 157 157 F F F 15 300-319 80 80 80 156 156 156 F F F 16 320-339 80 80 80 156 156 156 F F F 17 340-359 76 76 76 144 144 144 F F F 18 360-379 77 77 77 146 146 146 F F F 19 380-399 80 80 80 156 156 156 F F F 20 400-419 80 80 80 156 156 156 F F F 21 420-439 80 80 80 156 156 156 F F F 22 440-459 80 80 80 156 156 156 F F F 23 460-479 70 70 70 135 135 135 F F F 24 480-499 81 81 81 157 157 157 F F F 25 500-519 81 81 81 157 157 157 F F F 26 520-539 78 78 78 152 152 152 F F F 27 540-559 80 80 80 156 156 156 F F F 28 560-579 81 81 81 157 157 157 F F F 29 580-599 78 78 78 151 151 151 F F F Max gen: 81, Max flush: 157 Begin loop 41 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:37.443386337Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1508,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:37.487760578Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1508,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:37.488394397Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1508} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"0fd98d0d5d909611f8b408ccda2559d35fe117bb\",\n git_commit_timestamp: \"2024-11-08T20:05:44.000000000Z\",\n git_branch: \"mkeeter/drop-skipped-backpressure-guard\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:37.488439815Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1508} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:37.488457124Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1508} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:37.488474644Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1508} {"msg":"Crucible a609600d-1b90-4746-8ce0-d4d9245c18e2 has session id: 122b25f3-3cdd-4c12-b1b5-0941b0314769","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:37.488974169Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1508,"session_id":"122b25f3-3cdd-4c12-b1b5-0941b0314769"} {"msg":"Upstairs opts: Upstairs UUID: a609600d-1b90-4746-8ce0-d4d9245c18e2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:37.489053885Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1508,"session_id":"122b25f3-3cdd-4c12-b1b5-0941b0314769"} {"msg":"Crucible stats registered with UUID: a609600d-1b90-4746-8ce0-d4d9245c18e2","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:37.489089513Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1508,"session_id":"122b25f3-3cdd-4c12-b1b5-0941b0314769"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:37.52006464Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1508,"task":"crutest"} {"msg":"The guest has requested activation with gen:82","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:37.520114227Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1508} {"msg":"a609600d-1b90-4746-8ce0-d4d9245c18e2 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:37.520149655Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1508,"session_id":"122b25f3-3cdd-4c12-b1b5-0941b0314769"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:37.520180204Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1508,"client":"0","":"downstairs","session_id":"122b25f3-3cdd-4c12-b1b5-0941b0314769"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:37.520227041Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1508,"client":"0","":"downstairs","session_id":"122b25f3-3cdd-4c12-b1b5-0941b0314769"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:37.52026222Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1508,"client":"1","":"downstairs","session_id":"122b25f3-3cdd-4c12-b1b5-0941b0314769"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:37.520288749Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1508,"client":"1","":"downstairs","session_id":"122b25f3-3cdd-4c12-b1b5-0941b0314769"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:37.520315087Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1508,"client":"2","":"downstairs","session_id":"122b25f3-3cdd-4c12-b1b5-0941b0314769"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:37.520342696Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1508,"client":"2","":"downstairs","session_id":"122b25f3-3cdd-4c12-b1b5-0941b0314769"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:37.520367345Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1508,"":"io task","client":"2","":"downstairs","session_id":"122b25f3-3cdd-4c12-b1b5-0941b0314769"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:37.520393513Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1508,"":"io task","client":"0","":"downstairs","session_id":"122b25f3-3cdd-4c12-b1b5-0941b0314769"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:37.520418552Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1508,"":"io task","client":"1","":"downstairs","session_id":"122b25f3-3cdd-4c12-b1b5-0941b0314769"} {"msg":"ds_connection connected from Ok(127.0.0.1:50989)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:37.520444641Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1508,"":"io task","client":"2","":"downstairs","session_id":"122b25f3-3cdd-4c12-b1b5-0941b0314769"} {"msg":"ds_connection connected from Ok(127.0.0.1:33870)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:37.52047202Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1508,"":"io task","client":"0","":"downstairs","session_id":"122b25f3-3cdd-4c12-b1b5-0941b0314769"} {"msg":"ds_connection connected from Ok(127.0.0.1:45191)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:37.520696169Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1508,"":"io task","client":"1","":"downstairs","session_id":"122b25f3-3cdd-4c12-b1b5-0941b0314769"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:37.52108533Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1508,"client":"2","":"downstairs","session_id":"122b25f3-3cdd-4c12-b1b5-0941b0314769"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:37.521131307Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1508,"client":"2","":"downstairs","session_id":"122b25f3-3cdd-4c12-b1b5-0941b0314769"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:37.521174255Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1508,"client":"0","":"downstairs","session_id":"122b25f3-3cdd-4c12-b1b5-0941b0314769"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:37.521210953Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1508,"client":"0","":"downstairs","session_id":"122b25f3-3cdd-4c12-b1b5-0941b0314769"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:37.521330898Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1508,"client":"1","":"downstairs","session_id":"122b25f3-3cdd-4c12-b1b5-0941b0314769"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:37.521365766Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1508,"client":"1","":"downstairs","session_id":"122b25f3-3cdd-4c12-b1b5-0941b0314769"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:37.52189881Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1508,"client":"0","":"downstairs","session_id":"122b25f3-3cdd-4c12-b1b5-0941b0314769"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:37.521941378Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1508,"client":"2","":"downstairs","session_id":"122b25f3-3cdd-4c12-b1b5-0941b0314769"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:37.521974976Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1508,"client":"1","":"downstairs","session_id":"122b25f3-3cdd-4c12-b1b5-0941b0314769"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:37.522522409Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1508,"client":"0","":"downstairs","session_id":"122b25f3-3cdd-4c12-b1b5-0941b0314769"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:37.522562158Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1508,"client":"0","":"downstairs","session_id":"122b25f3-3cdd-4c12-b1b5-0941b0314769"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:37.522585846Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1508,"session_id":"122b25f3-3cdd-4c12-b1b5-0941b0314769"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:37.522599566Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1508,"client":"2","":"downstairs","session_id":"122b25f3-3cdd-4c12-b1b5-0941b0314769"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:37.522613445Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1508,"client":"2","":"downstairs","session_id":"122b25f3-3cdd-4c12-b1b5-0941b0314769"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:37.522627014Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1508,"session_id":"122b25f3-3cdd-4c12-b1b5-0941b0314769"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:37.522639224Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1508,"client":"1","":"downstairs","session_id":"122b25f3-3cdd-4c12-b1b5-0941b0314769"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:37.522652463Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1508,"client":"1","":"downstairs","session_id":"122b25f3-3cdd-4c12-b1b5-0941b0314769"} {"msg":"[0]R flush_numbers[0..12]: [145, 157, 153, 156, 146, 152, 157, 137, 153, 152, 142, 152]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:37.522669422Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1508,"":"downstairs","":"downstairs","session_id":"122b25f3-3cdd-4c12-b1b5-0941b0314769"} {"msg":"[0]R generation[0..12]: [76, 81, 79, 80, 77, 78, 81, 72, 79, 78, 74, 78]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:37.522685101Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1508,"":"downstairs","":"downstairs","session_id":"122b25f3-3cdd-4c12-b1b5-0941b0314769"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:37.522699611Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1508,"":"downstairs","":"downstairs","session_id":"122b25f3-3cdd-4c12-b1b5-0941b0314769"} {"msg":"[1]R flush_numbers[0..12]: [145, 157, 153, 156, 146, 152, 157, 137, 153, 152, 142, 152]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:37.52271345Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1508,"":"downstairs","":"downstairs","session_id":"122b25f3-3cdd-4c12-b1b5-0941b0314769"} {"msg":"[1]R generation[0..12]: [76, 81, 79, 80, 77, 78, 81, 72, 79, 78, 74, 78]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:37.522727219Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1508,"":"downstairs","":"downstairs","session_id":"122b25f3-3cdd-4c12-b1b5-0941b0314769"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:37.522741269Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1508,"":"downstairs","":"downstairs","session_id":"122b25f3-3cdd-4c12-b1b5-0941b0314769"} {"msg":"[2]R flush_numbers[0..12]: [145, 157, 153, 156, 146, 152, 157, 137, 153, 152, 142, 152]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:37.522765077Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1508,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO 122b25f3-3cdd-4c12-b1b5-0941b0314769"} {"msg":"[2]R generation[0..12]: [76, 81, 79, 80, 77, 78, 81, 72, 79, 78, 74, 78]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:37.522800356Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1508,"":"downstairs","":"downstairs","session_id":"122b25f3-3cdd-4c12-b1b5-0941b0314769"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:37.522824355Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1508,"":"downstairs","":"downstairs","session_id":"122b25f3-3cdd-4c12-b1b5-0941b0314769"} {"msg":"Max found gen is 82","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:37.522838614Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1508,"":"downstairs","":"downstairs","session_id":"122b25f3-3cdd-4c12-b1b5-0941b0314769"} {"msg":"Generation requested: 82 >= found:82","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:37.522851783Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1508,"":"downstairs","":"downstairs","session_id":"122b25f3-3cdd-4c12-b1b5-0941b0314769"} {"msg":"Next flush: 158","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:37.522864583Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1508,"":"downstairs","":"downstairs","session_id":"122b25f3-3cdd-4c12-b1b5-0941b0314769"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:37.522877642Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1508,"":"downstairs","":"downstairs","session_id":"122b25f3-3cdd-4c12-b1b5-0941b0314769"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:37.522891071Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1508,"session_id":"122b25f3-3cdd-4c12-b1b5-0941b0314769"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:37.522905651Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1508,"client":"0","":"downstairs","session_id":"122b25f3-3cdd-4c12-b1b5-0941b0314769"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:37.52291935Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1508,"client":"1","":"downstairs","session_id":"122b25f3-3cdd-4c12-b1b5-0941b0314769"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:37.522932509Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1508,"client":"2","":"downstairs","session_id":"122b25f3-3cdd-4c12-b1b5-0941b0314769"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:37.522945679Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1508,"session_id":"122b25f3-3cdd-4c12-b1b5-0941b0314769"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:37.522957908Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1508,"session_id":"122b25f3-3cdd-4c12-b1b5-0941b0314769"} {"msg":"a609600d-1b90-4746-8ce0-d4d9245c18e2 is now active with session: 122b25f3-3cdd-4c12-b1b5-0941b0314769","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:37.522970667Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1508,"session_id":"122b25f3-3cdd-4c12-b1b5-0941b0314769"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:37.522982517Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1508,"session_id":"122b25f3-3cdd-4c12-b1b5-0941b0314769"} {"msg":"The guest has finished waiting for activation with:82","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:37.522994626Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1508} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 587 len 4096 data: 11 02/30 Read block 257 len 28672 03/30 Read block 106 len 16384 04/30 Read block 566 len 40960 05/30 Write block 525 len 40960 data: 5 5 6 7 6 5 5 4 5 5 06/30 Write block 452 len 32768 data: 7 6 6 4 5 5 4 4 07/30 Write block 35 len 40960 data: 5 4 5 7 8 8 6 7 8 8 08/30 Write block 253 len 24576 data: 11 10 8 5 7 7 09/30 Write block 449 len 24576 data: 7 6 6 8 7 7 10/30 Write block 150 len 20480 data: 7 8 5 5 7 11/30 Read block 65 len 8192 12/30 Read block 246 len 36864 13/30 Read block 589 len 32768 14/30 Read block 276 len 28672 15/30 Read block 110 len 32768 16/30 Write block 32 len 24576 data: 7 7 7 6 5 6 17/30 Write block 483 len 8192 data: 9 9 18/30 Flush 19/30 Write block 546 len 40960 data: 8 5 7 6 5 3 5 5 6 7 20/30 Write block 38 len 32768 data: 8 9 9 7 8 9 9 7 21/30 Write block 516 len 24576 data: 5 3 4 5 3 4 22/30 Read block 345 len 4096 23/30 Read block 570 len 24576 24/30 Read block 458 len 32768 25/30 Read block 558 len 24576 26/30 Write block 62 len 24576 data: 6 7 7 7 6 5 27/30 Read block 501 len 40960 28/30 Write block 242 len 4096 data: 6 29/30 Write block 36 len 20480 data: 6 7 9 10 10 30/30 Write block 471 len 32768 data: 8 8 8 8 9 8 8 9 ---------------------------------------------------------------- Crucible gen:82 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1018 Acked Write 10 Done Done Done false 1019 Acked Write 8 Done Done Done false 1020 Acked Write 6 Done Done Done false 1021 Acked Read 1 Done Done Done false 1022 Acked Read 6 Done Done Done false 1023 Acked Read 8 Done Done Done false 1024 Acked Read 6 Done Done Done false 1025 Acked Write 6 Done Done Done false 1026 Acked Read 10 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Write 5 Sent Sent Sent false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 10 10 10 30 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1017 1017 1017 Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:25:37.553248128Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1508} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 76 76 76 145 145 145 F F F 1 020-039 82 82 82 158 158 158 T T T 2 040-059 82 82 82 158 158 158 T T T 3 060-079 80 80 80 156 156 156 T T T 4 080-099 77 77 77 146 146 146 F F F 5 100-119 78 78 78 152 152 152 F F F 6 120-139 81 81 81 157 157 157 F F F 7 140-159 82 82 82 158 158 158 F F F 8 160-179 79 79 79 153 153 153 F F F 9 180-199 78 78 78 152 152 152 F F F 10 200-219 74 74 74 142 142 142 F F F 11 220-239 78 78 78 152 152 152 F F F 12 240-259 82 82 82 158 158 158 T T T 13 260-279 80 80 80 156 156 156 F F F 14 280-299 81 81 81 157 157 157 F F F 15 300-319 80 80 80 156 156 156 F F F 16 320-339 80 80 80 156 156 156 F F F 17 340-359 76 76 76 144 144 144 F F F 18 360-379 77 77 77 146 146 146 F F F 19 380-399 80 80 80 156 156 156 F F F 20 400-419 80 80 80 156 156 156 F F F 21 420-439 80 80 80 156 156 156 F F F 22 440-459 82 82 82 158 158 158 F F F 23 460-479 70 70 70 135 135 135 T T T 24 480-499 82 82 82 158 158 158 F F F 25 500-519 81 81 81 157 157 157 T T T 26 520-539 82 82 82 158 158 158 T T T 27 540-559 80 80 80 156 156 156 T T T 28 560-579 81 81 81 157 157 157 F F F 29 580-599 82 82 82 158 158 158 F F F Max gen: 82, Max flush: 158 A Difference in extent metadata is expected here On loop 41 Verifying data now No change after verify Loop: 41 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 76 76 76 145 145 145 F F F 1 020-039 83 83 83 159 159 159 F F F 2 040-059 83 83 83 159 159 159 F F F 3 060-079 83 83 83 159 159 159 F F F 4 080-099 77 77 77 146 146 146 F F F 5 100-119 78 78 78 152 152 152 F F F 6 120-139 81 81 81 157 157 157 F F F 7 140-159 82 82 82 158 158 158 F F F 8 160-179 79 79 79 153 153 153 F F F 9 180-199 78 78 78 152 152 152 F F F 10 200-219 74 74 74 142 142 142 F F F 11 220-239 78 78 78 152 152 152 F F F 12 240-259 83 83 83 159 159 159 F F F 13 260-279 80 80 80 156 156 156 F F F 14 280-299 81 81 81 157 157 157 F F F 15 300-319 80 80 80 156 156 156 F F F 16 320-339 80 80 80 156 156 156 F F F 17 340-359 76 76 76 144 144 144 F F F 18 360-379 77 77 77 146 146 146 F F F 19 380-399 80 80 80 156 156 156 F F F 20 400-419 80 80 80 156 156 156 F F F 21 420-439 80 80 80 156 156 156 F F F 22 440-459 82 82 82 158 158 158 F F F 23 460-479 83 83 83 159 159 159 F F F 24 480-499 82 82 82 158 158 158 F F F 25 500-519 83 83 83 159 159 159 F F F 26 520-539 83 83 83 159 159 159 F F F 27 540-559 83 83 83 159 159 159 F F F 28 560-579 81 81 81 157 157 157 F F F 29 580-599 82 82 82 158 158 158 F F F Max gen: 83, Max flush: 159 Begin loop 42 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:41.909137977Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1516,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:25:41.952146335Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1516,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:41.952550585Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1516,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:41.984549141Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1516,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:41.985147482Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1516} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"0fd98d0d5d909611f8b408ccda2559d35fe117bb\",\n git_commit_timestamp: \"2024-11-08T20:05:44.000000000Z\",\n git_branch: \"mkeeter/drop-skipped-backpressure-guard\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:41.98519138Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1516} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:41.985208849Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1516} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:41.985226818Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1516} {"msg":"Crucible 86706ea5-b6a4-4196-9d18-b80586e74b00 has session id: d850a2c4-9c41-4aed-b3d2-345515ee4f66","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:41.985632728Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1516,"session_id":"d850a2c4-9c41-4aed-b3d2-345515ee4f66"} {"msg":"Upstairs opts: Upstairs UUID: 86706ea5-b6a4-4196-9d18-b80586e74b00, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:41.985686966Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1516,"session_id":"d850a2c4-9c41-4aed-b3d2-345515ee4f66"} {"msg":"Crucible stats registered with UUID: 86706ea5-b6a4-4196-9d18-b80586e74b00","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:41.985716654Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1516,"session_id":"d850a2c4-9c41-4aed-b3d2-345515ee4f66"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:42.016559867Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1516,"task":"crutest"} {"msg":"The guest has requested activation with gen:84","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:42.016605544Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1516} {"msg":"86706ea5-b6a4-4196-9d18-b80586e74b00 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:42.016636583Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1516,"session_id":"d850a2c4-9c41-4aed-b3d2-345515ee4f66"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:42.016661172Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1516,"client":"0","":"downstairs","session_id":"d850a2c4-9c41-4aed-b3d2-345515ee4f66"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:42.016706259Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1516,"client":"0","":"downstairs","session_id":"d850a2c4-9c41-4aed-b3d2-345515ee4f66"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:42.016743658Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1516,"client":"1","":"downstairs","session_id":"d850a2c4-9c41-4aed-b3d2-345515ee4f66"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:42.016781206Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1516,"client":"1","":"downstairs","session_id":"d850a2c4-9c41-4aed-b3d2-345515ee4f66"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:42.016814624Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1516,"client":"2","":"downstairs","session_id":"d850a2c4-9c41-4aed-b3d2-345515ee4f66"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:42.016847903Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1516,"client":"2","":"downstairs","session_id":"d850a2c4-9c41-4aed-b3d2-345515ee4f66"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:42.016880031Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1516,"":"io task","client":"2","":"downstairs","session_id":"d850a2c4-9c41-4aed-b3d2-345515ee4f66"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:42.01691098Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1516,"":"io task","client":"0","":"downstairs","session_id":"d850a2c4-9c41-4aed-b3d2-345515ee4f66"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:42.016945908Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1516,"":"io task","client":"1","":"downstairs","session_id":"d850a2c4-9c41-4aed-b3d2-345515ee4f66"} {"msg":"ds_connection connected from Ok(127.0.0.1:60613)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:42.017087011Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1516,"":"io task","client":"2","":"downstairs","session_id":"d850a2c4-9c41-4aed-b3d2-345515ee4f66"} {"msg":"ds_connection connected from Ok(127.0.0.1:35385)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:42.017128069Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1516,"":"io task","client":"0","":"downstairs","session_id":"d850a2c4-9c41-4aed-b3d2-345515ee4f66"} {"msg":"ds_connection connected from Ok(127.0.0.1:37642)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:42.017156657Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1516,"":"io task","client":"1","":"downstairs","session_id":"d850a2c4-9c41-4aed-b3d2-345515ee4f66"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:42.017684122Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1516,"client":"2","":"downstairs","session_id":"d850a2c4-9c41-4aed-b3d2-345515ee4f66"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:42.01772585Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1516,"client":"2","":"downstairs","session_id":"d850a2c4-9c41-4aed-b3d2-345515ee4f66"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:42.017760768Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1516,"client":"1","":"downstairs","session_id":"d850a2c4-9c41-4aed-b3d2-345515ee4f66"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:42.017835874Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1516,"client":"1","":"downstairs","session_id":"d850a2c4-9c41-4aed-b3d2-345515ee4f66"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:42.017854003Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1516,"client":"0","":"downstairs","session_id":"d850a2c4-9c41-4aed-b3d2-345515ee4f66"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:42.017867923Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1516,"client":"0","":"downstairs","session_id":"d850a2c4-9c41-4aed-b3d2-345515ee4f66"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:42.018344359Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1516,"client":"1","":"downstairs","session_id":"d850a2c4-9c41-4aed-b3d2-345515ee4f66"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:42.018387487Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1516,"client":"2","":"downstairs","session_id":"d850a2c4-9c41-4aed-b3d2-345515ee4f66"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:42.0185331Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1516,"client":"0","":"downstairs","session_id":"d850a2c4-9c41-4aed-b3d2-345515ee4f66"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:42.018913632Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1516,"client":"2","":"downstairs","session_id":"d850a2c4-9c41-4aed-b3d2-345515ee4f66"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:42.01895293Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1516,"client":"2","":"downstairs","session_id":"d850a2c4-9c41-4aed-b3d2-345515ee4f66"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:42.018981448Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1516,"session_id":"d850a2c4-9c41-4aed-b3d2-345515ee4f66"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:42.018999927Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1516,"client":"1","":"downstairs","session_id":"d850a2c4-9c41-4aed-b3d2-345515ee4f66"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:42.019014597Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1516,"client":"1","":"downstairs","session_id":"d850a2c4-9c41-4aed-b3d2-345515ee4f66"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:42.019028426Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1516,"session_id":"d850a2c4-9c41-4aed-b3d2-345515ee4f66"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:42.019044865Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1516,"client":"0","":"downstairs","session_id":"d850a2c4-9c41-4aed-b3d2-345515ee4f66"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:42.019058644Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1516,"client":"0","":"downstairs","session_id":"d850a2c4-9c41-4aed-b3d2-345515ee4f66"} {"msg":"[0]R flush_numbers[0..12]: [145, 159, 159, 159, 146, 152, 157, 158, 153, 152, 142, 152]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:42.019091153Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1516,"":"downstairs","":"downstairs","session_id":"d850a2c4-9c41-4aed-b3d2-345515ee4f66"} {"msg":"[0]R generation[0..12]: [76, 83, 83, 83, 77, 78, 81, 82, 79, 78, 74, 78]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:42.019106292Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1516,"":"downstairs","":"downstairs","session_id":"d850a2c4-9c41-4aed-b3d2-345515ee4f66"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:42.019120592Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1516,"":"downstairs","":"downstairs","session_id":"d850a2c4-9c41-4aed-b3d2-345515ee4f66"} {"msg":"[1]R flush_numbers[0..12]: [145, 159, 159, 159, 146, 152, 157, 158, 153, 152, 142, 152]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:42.019134501Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1516,"":"downstairs","":"downstairs","session_id":"d850a2c4-9c41-4aed-b3d2-345515ee4f66"} {"msg":"[1]R generation[0..12]: [76, 83, 83, 83, 77, 78, 81, 82, 79, 78, 74, 78]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:42.01914821Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1516,"":"downstairs","":"downstairs","session_id":"d850a2c4-9c41-4aed-b3d2-345515ee4f66"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:42.019162399Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1516,"":"downstairs","":"downstairs","session_id":"d850a2c4-9c41-4aed-b3d2-345515ee4f66"} {"msg":"[2]R flush_numbers[0..12]: [145, 159, 159, 159, 146, 152, 157, 158, 153, 152, 142, 152]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:42.019176339Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1516,"":"downstairs","":"downstairs","session_id":"d850a2c4-9c41-4aed-b3d2-345515ee4f66"} {"msg":"[2]R generation[0..12]: [76, 83, 83, 83, 77, 78, 81, 82, 79, 78, 74, 78]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:42.019189938Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1516,"":"downstairs","":"downstairs","session_id":"d850a2c4-9c41-4aed-b3d2-345515ee4f66"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:42.019203957Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1516,"":"downstairs","":"downstairs","session_id":"d850a2c4-9c41-4aed-b3d2-345515ee4f66"} {"msg":"Max found gen is 84","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:42.019216827Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1516,"":"downstairs","":"downstairs","session_id":"d850a2c4-9c41-4aed-b3d2-345515ee4f66"} {"msg":"Generation requested: 84 >= found:84","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:42.019229966Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1516,"":"downstairs","":"downstairs","session_id":"d850a2c4-9c41-4aed-b3d2-345515ee4f66"} {"msg":"Next flush: 160","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:42.019242915Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1516,"":"downstairs","":"downstairs","session_id":"d850a2c4-9c41-4aed-b3d2-345515ee4f66"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:42.019255725Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1516Wait for a query_work_queue command to finish before sending IO ,"":"downstairs","":"downstairs","session_id":"d850a2c4-9c41-4aed-b3d2-345515ee4f66"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:42.019278034Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1516,"session_id":"d850a2c4-9c41-4aed-b3d2-345515ee4f66"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:42.019292213Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1516,"client":"0","":"downstairs","session_id":"d850a2c4-9c41-4aed-b3d2-345515ee4f66"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:42.019305703Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1516,"client":"1","":"downstairs","session_id":"d850a2c4-9c41-4aed-b3d2-345515ee4f66"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:42.019318942Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1516,"client":"2","":"downstairs","session_id":"d850a2c4-9c41-4aed-b3d2-345515ee4f66"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:42.019332601Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1516,"session_id":"d850a2c4-9c41-4aed-b3d2-345515ee4f66"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:42.01934511Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1516,"session_id":"d850a2c4-9c41-4aed-b3d2-345515ee4f66"} {"msg":"86706ea5-b6a4-4196-9d18-b80586e74b00 is now active with session: d850a2c4-9c41-4aed-b3d2-345515ee4f66","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:42.01935811Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1516,"session_id":"d850a2c4-9c41-4aed-b3d2-345515ee4f66"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:42.019370119Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1516,"session_id":"d850a2c4-9c41-4aed-b3d2-345515ee4f66"} {"msg":"The guest has finished waiting for activation with:84","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:42.019382549Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1516} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 326 len 40960 data: 6 4 4 4 4 4 5 6 7 9 02/30 Write block 518 len 24576 data: 5 6 4 5 5 6 03/30 Read block 433 len 8192 04/30 Read block 36 len 32768 05/30 Write block 435 len 8192 data: 9 10 06/30 Read block 410 len 8192 07/30 Read block 307 len 16384 08/30 Write block 163 len 36864 data: 2 4 5 5 4 9 9 11 10 09/30 Read block 323 len 8192 10/30 Flush 11/30 Write block 572 len 20480 data: 11 11 9 9 8 12/30 Read block 116 len 4096 13/30 Write block 70 len 16384 data: 6 4 5 6 14/30 Read block 202 len 36864 15/30 Read block 144 len 20480 16/30 Read block 322 len 24576 17/30 Read block 532 len 36864 18/30 Read block 188 len 28672 19/30 Read block 386 len 20480 20/30 Write block 338 len 24576 data: 10 9 9 9 10 8 21/30 Write block 513 len 36864 data: 9 8 6 6 4 6 7 5 6 22/30 Write block 164 len 16384 data: 5 6 6 5 23/30 Write block 500 len 32768 data: 8 9 8 6 7 6 5 5 24/30 Read block 317 len 12288 25/30 Read block 179 len 24576 26/30 Read block 434 len 4096 27/30 Write block 433 len 40960 data: 7 7 10 11 10 10 9 9 12 11 28/30 Write block 147 len 28672 data: 7 6 7 8 9 6 6 29/30 Write block 262 len 32768 data: 7 8 8 10 9 6 6 7 30/30 Write block 297 len 28672 data: 5 6 9 9 10 9 10 ---------------------------------------------------------------- Crucible gen:84 GIO:true work queues: Upstairs:0 downstairs:20 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1010 Acked Write 5 Done Done Done false 1011 Acked Read 1 Done Done Done false 1012 Acked Write 4 Done Done Done false 1013 Acked Read 9 Done Done Done false 1014 Acked Read 5 Done Done Done false 1015 Acked Read 6 Done Done Done false 1016 Acked Read 9 Done Done Done false 1017 Acked Read 7 Done Done Done false 1018 Acked Read 5 Done Done Done false 1019 Acked Write 6 Done Done Done false 1020 Acked Write 9 Done Done Done false 1021 Acked Write 4 Done Done Done false 1022 Acked Write 8 Done Done Done false 1023 Acked Read 3 Done Done Done false 1024 Acked Read 6 Done Done Done false 1025 Acked Read 1 Done Done Done false 1026 Acked Write 10 Done Done Done false 1027 Acked Write 7 Done Done Done false 1028 Acked Write 8 Sent Sent Sent false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 18 18 18 54 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1009 1009 1009 Downstairs last five completed: 1009 1008 1007 1006 1005 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:25:42.04228084Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1516} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 76 76 76 145 145 145 F F F 1 020-039 83 83 83 159 159 159 F F F 2 040-059 83 83 83 159 159 159 F F F 3 060-079 83 83 83 159 159 159 T T T 4 080-099 77 77 77 146 146 146 F F F 5 100-119 78 78 78 152 152 152 F F F 6 120-139 81 81 81 157 157 157 F F F 7 140-159 82 82 82 158 158 158 T T T 8 160-179 84 84 84 160 160 160 T T T 9 180-199 78 78 78 152 152 152 F F F 10 200-219 74 74 74 142 142 142 F F F 11 220-239 78 78 78 152 152 152 F F F 12 240-259 83 83 83 159 159 159 F F F 13 260-279 80 80 80 156 156 156 T T T 14 280-299 81 81 81 157 157 157 T T T 15 300-319 80 80 80 156 156 156 T T T 16 320-339 84 84 84 160 160 160 T T T 17 340-359 76 76 76 144 144 144 T T T 18 360-379 77 77 77 146 146 146 F F F 19 380-399 80 80 80 156 156 156 F F F 20 400-419 80 80 80 156 156 156 F F F 21 420-439 84 84 84 160 160 160 T T T 22 440-459 82 82 82 158 158 158 T T T 23 460-479 83 83 83 159 159 159 F F F 24 480-499 82 82 82 158 158 158 F F F 25 500-519 84 84 84 160 160 160 T T T 26 520-539 84 84 84 160 160 160 T T T 27 540-559 83 83 83 159 159 159 F F F 28 560-579 81 81 81 157 157 157 T T T 29 580-599 82 82 82 158 158 158 F F F Max gen: 84, Max flush: 160 A Difference in extent metadata is expected here On loop 42 Verifying data now No change after verify Loop: 42 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 76 76 76 145 145 145 F F F 1 020-039 83 83 83 159 159 159 F F F 2 040-059 83 83 83 159 159 159 F F F 3 060-079 85 85 85 161 161 161 F F F 4 080-099 77 77 77 146 146 146 F F F 5 100-119 78 78 78 152 152 152 F F F 6 120-139 81 81 81 157 157 157 F F F 7 140-159 85 85 85 161 161 161 F F F 8 160-179 85 85 85 161 161 161 F F F 9 180-199 78 78 78 152 152 152 F F F 10 200-219 74 74 74 142 142 142 F F F 11 220-239 78 78 78 152 152 152 F F F 12 240-259 83 83 83 159 159 159 F F F 13 260-279 85 85 85 161 161 161 F F F 14 280-299 85 85 85 161 161 161 F F F 15 300-319 85 85 85 161 161 161 F F F 16 320-339 85 85 85 161 161 161 F F F 17 340-359 85 85 85 161 161 161 F F F 18 360-379 77 77 77 146 146 146 F F F 19 380-399 80 80 80 156 156 156 F F F 20 400-419 80 80 80 156 156 156 F F F 21 420-439 85 85 85 161 161 161 F F F 22 440-459 85 85 85 161 161 161 F F F 23 460-479 83 83 83 159 159 159 F F F 24 480-499 82 82 82 158 158 158 F F F 25 500-519 85 85 85 161 161 161 F F F 26 520-539 85 85 85 161 161 161 F F F 27 540-559 83 83 83 159 159 159 F F F 28 560-579 85 85 85 161 161 161 F F F 29 580-599 82 82 82 158 158 158 F F F Max gen: 85, Max flush: 161 Begin loop 43 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:46.51356786Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1524,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:25:46.556430345Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1524,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:46.556836425Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1524,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:46.588650181Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1524,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:46.589298279Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1524} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"0fd98d0d5d909611f8b408ccda2559d35fe117bb\",\n git_commit_timestamp: \"2024-11-08T20:05:44.000000000Z\",\n git_branch: \"mkeeter/drop-skipped-backpressure-guard\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:46.589333877Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1524} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:46.589347806Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1524} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:46.589361756Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1524} {"msg":"Crucible 75eba4fa-a4ca-4fa0-a6a2-11684240a64d has session id: aca99988-688c-4f43-840f-fb35c041a998","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:46.589837272Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1524,"session_id":"aca99988-688c-4f43-840f-fb35c041a998"} {"msg":"Upstairs opts: Upstairs UUID: 75eba4fa-a4ca-4fa0-a6a2-11684240a64d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:46.589874191Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1524,"session_id":"aca99988-688c-4f43-840f-fb35c041a998"} {"msg":"Crucible stats registered with UUID: 75eba4fa-a4ca-4fa0-a6a2-11684240a64d","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:46.58989516Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1524,"session_id":"aca99988-688c-4f43-840f-fb35c041a998"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:46.621631458Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1524,"task":"crutest"} {"msg":"The guest has requested activation with gen:86","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:46.621663217Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1524} {"msg":"75eba4fa-a4ca-4fa0-a6a2-11684240a64d active request set","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:46.621722684Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1524,"session_id":"aca99988-688c-4f43-840f-fb35c041a998"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:46.621751083Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1524,"client":"0","":"downstairs","session_id":"aca99988-688c-4f43-840f-fb35c041a998"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:46.621787631Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1524,"client":"0","":"downstairs","session_id":"aca99988-688c-4f43-840f-fb35c041a998"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:46.62181714Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1524,"client":"1","":"downstairs","session_id":"aca99988-688c-4f43-840f-fb35c041a998"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:46.621847488Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1524,"client":"1","":"downstairs","session_id":"aca99988-688c-4f43-840f-fb35c041a998"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:46.621875977Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1524,"client":"2","":"downstairs","session_id":"aca99988-688c-4f43-840f-fb35c041a998"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:46.621906405Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1524,"client":"2","":"downstairs","session_id":"aca99988-688c-4f43-840f-fb35c041a998"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:46.621932174Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1524,"":"io task","client":"2","":"downstairs","session_id":"aca99988-688c-4f43-840f-fb35c041a998"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:46.621959593Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1524,"":"io task","client":"0","":"downstairs","session_id":"aca99988-688c-4f43-840f-fb35c041a998"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:46.62202337Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1524,"":"io task","client":"1","":"downstairs","session_id":"aca99988-688c-4f43-840f-fb35c041a998"} {"msg":"ds_connection connected from Ok(127.0.0.1:42484)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:46.622069647Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1524,"":"io task","client":"2","":"downstairs","session_id":"aca99988-688c-4f43-840f-fb35c041a998"} {"msg":"ds_connection connected from Ok(127.0.0.1:34032)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:46.622106385Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1524,"":"io task","client":"1","":"downstairs","session_id":"aca99988-688c-4f43-840f-fb35c041a998"} {"msg":"ds_connection connected from Ok(127.0.0.1:39884)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:46.62220852Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1524,"":"io task","client":"0","":"downstairs","session_id":"aca99988-688c-4f43-840f-fb35c041a998"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:46.622811031Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1524,"client":"1","":"downstairs","session_id":"aca99988-688c-4f43-840f-fb35c041a998"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:46.622840119Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1524,"client":"1","":"downstairs","session_id":"aca99988-688c-4f43-840f-fb35c041a998"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:46.622855039Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1524,"client":"0","":"downstairs","session_id":"aca99988-688c-4f43-840f-fb35c041a998"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:46.622870338Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1524,"client":"0","":"downstairs","session_id":"aca99988-688c-4f43-840f-fb35c041a998"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:46.622884087Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1524,"client":"2","":"downstairs","session_id":"aca99988-688c-4f43-840f-fb35c041a998"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:46.622897487Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1524,"client":"2","":"downstairs","session_id":"aca99988-688c-4f43-840f-fb35c041a998"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:46.623598333Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1524,"client":"2","":"downstairs","session_id":"aca99988-688c-4f43-840f-fb35c041a998"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:46.623629321Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1524,"client":"0","":"downstairs","session_id":"aca99988-688c-4f43-840f-fb35c041a998"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:46.62365506Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1524,"client":"1","":"downstairs","session_id":"aca99988-688c-4f43-840f-fb35c041a998"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:46.624184484Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1524,"client":"1","":"downstairs","session_id":"aca99988-688c-4f43-840f-fb35c041a998"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:46.624214552Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1524,"client":"1","":"downstairs","session_id":"aca99988-688c-4f43-840f-fb35c041a998"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:46.624229572Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1524,"session_id":"aca99988-688c-4f43-840f-fb35c041a998"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:46.624298568Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1524,"client":"0","":"downstairs","session_id":"aca99988-688c-4f43-840f-fb35c041a998"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:46.624319017Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1524,"client":"0","":"downstairs","session_id":"aca99988-688c-4f43-840f-fb35c041a998"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:46.624332657Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1524,"session_id":"aca99988-688c-4f43-840f-fb35c041a998"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:46.624344906Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1524,"client":"2","":"downstairs","session_id":"aca99988-688c-4f43-840f-fb35c041a998"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:46.624361255Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1524,"client":"2","":"downstairs","session_id":"aca99988-688c-4f43-840f-fb35c041a998"} {"msg":"[0]R flush_numbers[0..12]: [145, 159, 159, 161, 146, 152, 157, 161, 161, 152, 142, 152]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:46.624375255Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1524,"":"downstairs","":"downstairs","session_id":"aca99988-688c-4f43-840f-fb35c041a998"} {"msg":"[0]R generation[0..12]: [76, 83, 83, 85, 77, 78, 81, 85, 85, 78, 74, 78]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:46.624389034Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1524,"":"downstairs","":"downstairs","session_id":"aca99988-688c-4f43-840f-fb35c041a998"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:46.624403063Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1524,"":"downstairs","":"downstairs","session_id":"aca99988-688c-4f43-840f-fb35c041a998"} {"msg":"[1]R flush_numbers[0..12]: [145, 159, 159, 161, 146, 152, 157, 161, 161, 152, 142, 152]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:46.624416762Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1524,"":"downstairs","":"downstairs","session_id":"aca99988-688c-4f43-840f-fb35c041a998"} {"msg":"[1]R generation[0..12]: [76, 83, 83, 85, 77, 78, 81, 85, 85, 78, 74, 78]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:46.624430152Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1524,"":"downstairs","":"downstairs","session_id":"aca99988-688c-4f43-840f-fb35c041a998"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:46.624443951Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1524,"":"downstairs","":"downstairs","session_id":"aca99988-688c-4f43-840f-fb35c041a998"} {"msg":"[2]R flush_numbers[0..12]: [145, 159, 159, 161, 146, 152, 157, 161, 161, 152, 142, 152]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:46.624457431Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1524,"":"downstairs","":"downstairs","session_id":"aca99988-688c-4f43-840f-fb35c041a998"} {"msg":"[2]R generation[0..12]: [76, 83, 83, 85, 77, 78, 81, 85, 85, 78, 74, 78]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:46.62447073Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1524,"":"downstairs","":"downstairs","session_id":"aca99988-688c-4f43-840f-fb35c041a998"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:46.624484579Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1524,"":"downstairs","":"downstairs","session_id":"aca99988-688c-4f43-840f-fb35c041a998"} {"msg":"Max found gen is 86","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:46.624497718Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1524,"":"downstairs","":"downstairs","session_id":"aca99988-688c-4f43-840f-fb35c041a998"} {"msg":"Generation requested: 86 >= found:86","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:46.624510768Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1524,"":"downstairs","":"downstairs","session_id":"aca99988-688c-4f43-840f-fb35c041a998"} {"msg":"Next flush: 162","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:46.624523827Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1524,"":"downstairs","":"downstairs","session_id":"aca99988-688c-4f43-840f-fb35c041a998"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:46.624536796Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW"Wait for a query_work_queue command to finish before sending IO ,"pid":1524,"":"downstairs","":"downstairs","session_id":"aca99988-688c-4f43-840f-fb35c041a998"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:46.624560735Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1524,"session_id":"aca99988-688c-4f43-840f-fb35c041a998"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:46.624573005Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1524,"client":"0","":"downstairs","session_id":"aca99988-688c-4f43-840f-fb35c041a998"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:46.624586164Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1524,"client":"1","":"downstairs","session_id":"aca99988-688c-4f43-840f-fb35c041a998"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:46.624599173Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1524,"client":"2","":"downstairs","session_id":"aca99988-688c-4f43-840f-fb35c041a998"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:46.624612463Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1524,"session_id":"aca99988-688c-4f43-840f-fb35c041a998"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:46.624624812Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1524,"session_id":"aca99988-688c-4f43-840f-fb35c041a998"} {"msg":"75eba4fa-a4ca-4fa0-a6a2-11684240a64d is now active with session: aca99988-688c-4f43-840f-fb35c041a998","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:46.624637612Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1524,"session_id":"aca99988-688c-4f43-840f-fb35c041a998"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:46.624649621Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1524,"session_id":"aca99988-688c-4f43-840f-fb35c041a998"} {"msg":"The guest has finished waiting for activation with:86","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:46.62466203Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1524} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 573 len 20480 data: 12 10 10 9 8 02/30 Write block 212 len 40960 data: 8 7 6 5 6 5 7 6 5 8 03/30 Flush 04/30 Write block 467 len 8192 data: 5 7 05/30 Read block 487 len 32768 06/30 Write block 339 len 4096 data: 10 07/30 Write block 318 len 12288 data: 8 10 8 08/30 Write block 121 len 24576 data: 9 10 11 10 7 7 09/30 Write block 145 len 36864 data: 7 6 8 7 8 9 10 7 7 10/30 Write block 341 len 40960 data: 10 11 9 7 7 10 8 6 9 8 11/30 Read block 334 len 20480 12/30 Read block 417 len 20480 13/30 Read block 205 len 8192 14/30 Read block 525 len 8192 15/30 Read block 119 len 12288 16/30 Read block 486 len 20480 17/30 Write block 199 len 20480 data: 8 7 6 6 5 18/30 Write block 236 len 4096 data: 7 19/30 Write block 102 len 16384 data: 6 7 6 5 20/30 Write block 440 len 20480 data: 10 13 12 8 8 21/30 Write block 195 len 8192 data: 7 8 22/30 Read block 77 len 40960 23/30 Write block 466 len 28672 data: 5 6 8 8 8 9 9 24/30 Write block 565 len 8192 data: 7 6 25/30 Write block 457 len 28672 data: 6 5 5 4 3 2 2 26/30 Write block 251 len 20480 data: 10 13 12 11 9 27/30 Read block 12 len 12288 28/30 Read block 313 len 16384 29/30 Write block 427 len 40960 data: 4 4 9 10 9 9 8 8 11 12 30/30 Read block 231 len 16384 ---------------------------------------------------------------- Crucible gen:86 GIO:true work queues: Upstairs:0 downstairs:27 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1003 Acked Write 2 Done Done Done false 1004 Acked Read 8 Done Done Done false 1005 Acked Write 1 Done Done Done false 1006 Acked Write 3 Done Done Done false 1007 Acked Write 6 Done Done Done false 1008 Acked Write 9 Done Done Done false 1009 Acked Write 10 Done Done Done false 1010 Acked Read 5 Done Done Done false 1011 Acked Read 5 Done Done Done false 1012 Acked Read 2 Done Done Done false 1013 Acked Read 2 Done Done Done false 1014 Acked Read 3 Done Done Done false 1015 Acked Read 5 Done Done Done false 1016 Acked Write 5 Done Done Done false 1017 Acked Write 1 Done Done Done false 1018 Acked Write 4 Done Done Done false 1019 Acked Write 5 Done Done Done false 1020 Acked Write 2 Done Done Done false 1021 Acked Read 10 Done Done Done false 1022 Acked Write 7 Done Done Done false 1023 Acked Write 2 Done Done Done false 1024 Acked Write 7 Done Done Done false 1025 Acked Write 5 Done Done Done false 1026 Acked Read 3 Done Done Done false 1027 Acked Read 4 Done Done Done false 1028 Acked Write 10 Done Sent Done false 1029 Acked Read 4 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 2 1 3 Done 27 25 26 78 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1002 1002 1002 Downstairs last five completed: 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:25:46.644265482Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1524} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 76 76 76 145 145 145 F F F 1 020-039 83 83 83 159 159 159 F F F 2 040-059 83 83 83 159 159 159 F F F 3 060-079 85 85 85 161 161 161 F F F 4 080-099 77 77 77 146 146 146 F F F 5 100-119 78 78 78 152 152 152 T T T 6 120-139 81 81 81 157 157 157 T T T 7 140-159 85 85 85 161 161 161 T T T 8 160-179 85 85 85 161 161 161 F F F 9 180-199 78 78 78 152 152 152 T T T 10 200-219 86 86 86 162 162 162 T T T 11 220-239 86 86 86 162 162 162 T T T 12 240-259 83 83 83 159 159 159 T T T 13 260-279 85 85 85 161 161 161 F F F 14 280-299 85 85 85 161 161 161 F F F 15 300-319 85 85 85 161 161 161 T T T 16 320-339 85 85 85 161 161 161 T T T 17 340-359 85 85 85 161 161 161 T T T 18 360-379 77 77 77 146 146 146 F F F 19 380-399 80 80 80 156 156 156 F F F 20 400-419 80 80 80 156 156 156 F F F 21 420-439 85 85 85 161 161 161 T T T 22 440-459 85 85 85 161 161 161 T T T 23 460-479 83 83 83 159 159 159 T T T 24 480-499 82 82 82 158 158 158 F F F 25 500-519 85 85 85 161 161 161 F F F 26 520-539 85 85 85 161 161 161 F F F 27 540-559 83 83 83 159 159 159 F F F 28 560-579 86 86 86 162 162 162 T T T 29 580-599 82 82 82 158 158 158 F F F Max gen: 86, Max flush: 162 A Difference in extent metadata is expected here On loop 43 Verifying data now No change after verify Loop: 43 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 76 76 76 145 145 145 F F F 1 020-039 83 83 83 159 159 159 F F F 2 040-059 83 83 83 159 159 159 F F F 3 060-079 85 85 85 161 161 161 F F F 4 080-099 77 77 77 146 146 146 F F F 5 100-119 87 87 87 163 163 163 F F F 6 120-139 87 87 87 163 163 163 F F F 7 140-159 87 87 87 163 163 163 F F F 8 160-179 85 85 85 161 161 161 F F F 9 180-199 87 87 87 163 163 163 F F F 10 200-219 87 87 87 163 163 163 F F F 11 220-239 87 87 87 163 163 163 F F F 12 240-259 87 87 87 163 163 163 F F F 13 260-279 85 85 85 161 161 161 F F F 14 280-299 85 85 85 161 161 161 F F F 15 300-319 87 87 87 163 163 163 F F F 16 320-339 87 87 87 163 163 163 F F F 17 340-359 87 87 87 163 163 163 F F F 18 360-379 77 77 77 146 146 146 F F F 19 380-399 80 80 80 156 156 156 F F F 20 400-419 80 80 80 156 156 156 F F F 21 420-439 87 87 87 163 163 163 F F F 22 440-459 87 87 87 163 163 163 F F F 23 460-479 87 87 87 163 163 163 F F F 24 480-499 82 82 82 158 158 158 F F F 25 500-519 85 85 85 161 161 161 F F F 26 520-539 85 85 85 161 161 161 F F F 27 540-559 83 83 83 159 159 159 F F F 28 560-579 87 87 87 163 163 163 F F F 29 580-599 82 82 82 158 158 158 F F F Max gen: 87, Max flush: 163 Begin loop 44 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:51.127153774Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1532,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:51.171536964Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1532,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:51.172104197Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1532} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"0fd98d0d5d909611f8b408ccda2559d35fe117bb\",\n git_commit_timestamp: \"2024-11-08T20:05:44.000000000Z\",\n git_branch: \"mkeeter/drop-skipped-backpressure-guard\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:51.172148314Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1532} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:51.172165964Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1532} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:51.172184423Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1532} {"msg":"Crucible f7b844e7-93d5-4ba8-bd3e-9c698ab68ce6 has session id: a65e1fdc-cf45-4333-b18b-8c172fb7d1fa","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:51.172630031Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1532,"session_id":"a65e1fdc-cf45-4333-b18b-8c172fb7d1fa"} {"msg":"Upstairs opts: Upstairs UUID: f7b844e7-93d5-4ba8-bd3e-9c698ab68ce6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:51.172678929Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1532,"session_id":"a65e1fdc-cf45-4333-b18b-8c172fb7d1fa"} {"msg":"Crucible stats registered with UUID: f7b844e7-93d5-4ba8-bd3e-9c698ab68ce6","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:51.172708697Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1532,"session_id":"a65e1fdc-cf45-4333-b18b-8c172fb7d1fa"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:51.204504033Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1532,"task":"crutest"} {"msg":"The guest has requested activation with gen:88","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:51.204539582Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1532} {"msg":"f7b844e7-93d5-4ba8-bd3e-9c698ab68ce6 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:51.20457627Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1532,"session_id":"a65e1fdc-cf45-4333-b18b-8c172fb7d1fa"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:51.204599568Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1532,"client":"0","":"downstairs","session_id":"a65e1fdc-cf45-4333-b18b-8c172fb7d1fa"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:51.204644926Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1532,"client":"0","":"downstairs","session_id":"a65e1fdc-cf45-4333-b18b-8c172fb7d1fa"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:51.204681784Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1532,"client":"1","":"downstairs","session_id":"a65e1fdc-cf45-4333-b18b-8c172fb7d1fa"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:51.204719343Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1532,"client":"1","":"downstairs","session_id":"a65e1fdc-cf45-4333-b18b-8c172fb7d1fa"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:51.204750681Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1532,"client":"2","":"downstairs","session_id":"a65e1fdc-cf45-4333-b18b-8c172fb7d1fa"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:51.20477442Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1532,"client":"2","":"downstairs","session_id":"a65e1fdc-cf45-4333-b18b-8c172fb7d1fa"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:51.204802199Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1532,"":"io task","client":"2","":"downstairs","session_id":"a65e1fdc-cf45-4333-b18b-8c172fb7d1fa"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:51.204826837Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1532,"":"io task","client":"0","":"downstairs","session_id":"a65e1fdc-cf45-4333-b18b-8c172fb7d1fa"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:51.204849066Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1532,"":"io task","client":"1","":"downstairs","session_id":"a65e1fdc-cf45-4333-b18b-8c172fb7d1fa"} {"msg":"ds_connection connected from Ok(127.0.0.1:41114)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:51.204876785Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1532,"":"io task","client":"0","":"downstairs","session_id":"a65e1fdc-cf45-4333-b18b-8c172fb7d1fa"} {"msg":"ds_connection connected from Ok(127.0.0.1:63906)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:51.204904154Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1532,"":"io task","client":"2","":"downstairs","session_id":"a65e1fdc-cf45-4333-b18b-8c172fb7d1fa"} {"msg":"ds_connection connected from Ok(127.0.0.1:64093)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:51.205128393Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1532,"":"io task","client":"1","":"downstairs","session_id":"a65e1fdc-cf45-4333-b18b-8c172fb7d1fa"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:51.205671646Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1532,"client":"2","":"downstairs","session_id":"a65e1fdc-cf45-4333-b18b-8c172fb7d1fa"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:51.205705614Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1532,"client":"2","":"downstairs","session_id":"a65e1fdc-cf45-4333-b18b-8c172fb7d1fa"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:51.205841928Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1532,"client":"1","":"downstairs","session_id":"a65e1fdc-cf45-4333-b18b-8c172fb7d1fa"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:51.205877986Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1532,"client":"1","":"downstairs","session_id":"a65e1fdc-cf45-4333-b18b-8c172fb7d1fa"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:51.205933563Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1532,"client":"0","":"downstairs","session_id":"a65e1fdc-cf45-4333-b18b-8c172fb7d1fa"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:51.205968002Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1532,"client":"0","":"downstairs","session_id":"a65e1fdc-cf45-4333-b18b-8c172fb7d1fa"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:51.206258987Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1532,"client":"2","":"downstairs","session_id":"a65e1fdc-cf45-4333-b18b-8c172fb7d1fa"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:51.206371802Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1532,"client":"0","":"downstairs","session_id":"a65e1fdc-cf45-4333-b18b-8c172fb7d1fa"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:51.206684227Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1532,"client":"1","":"downstairs","session_id":"a65e1fdc-cf45-4333-b18b-8c172fb7d1fa"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:51.206882287Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1532,"client":"2","":"downstairs","session_id":"a65e1fdc-cf45-4333-b18b-8c172fb7d1fa"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:51.206913106Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1532,"client":"2","":"downstairs","session_id":"a65e1fdc-cf45-4333-b18b-8c172fb7d1fa"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:51.206929385Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1532,"session_id":"a65e1fdc-cf45-4333-b18b-8c172fb7d1fa"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:51.206954024Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1532,"client":"0","":"downstairs","session_id":"a65e1fdc-cf45-4333-b18b-8c172fb7d1fa"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:51.206969703Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1532,"client":"0","":"downstairs","session_id":"a65e1fdc-cf45-4333-b18b-8c172fb7d1fa"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:51.206987782Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1532,"session_id":"a65e1fdc-cf45-4333-b18b-8c172fb7d1fa"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:51.207198301Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1532,"client":"1","":"downstairs","session_id":"a65e1fdc-cf45-4333-b18b-8c172fb7d1fa"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:51.207220491Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1532,"client":"1","":"downstairs","session_id":"a65e1fdc-cf45-4333-b18b-8c172fb7d1fa"} {"msg":"[0]R flush_numbers[0..12]: [145, 159, 159, 161, 146, 163, 163, 163, 161, 163, 163, 163]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:51.20723574Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1532,"":"downstairs","":"downstairs","session_id":"a65e1fdc-cf45-4333-b18b-8c172fb7d1fa"} {"msg":"[0]R generation[0..12]: [76, 83, 83, 85, 77, 87, 87, 87, 85, 87, 87, 87]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:51.207250429Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1532,"":"downstairs","":"downstairs","session_id":"a65e1fdc-cf45-4333-b18b-8c172fb7d1fa"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:51.207265238Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1532,"":"downstairs","":"downstairs","session_id":"a65e1fdc-cf45-4333-b18b-8c172fb7d1fa"} {"msg":"[1]R flush_numbers[0..12]: [145, 159, 159, 161, 146, 163, 163, 163, 161, 163, 163, 163]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:51.207279957Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1532,"":"downstairs","":"downstairs","session_id":"a65e1fdc-cf45-4333-b18b-8c172fb7d1fa"} {"msg":"[1]R generation[0..12]: [76, 83, 83, 85, 77, 87, 87, 87, 85, 87, 87, 87]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:51.207293617Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1532,"":"downstairs","":"downstairs","session_id":"a65e1fdc-cf45-4333-b18b-8c172fb7d1fa"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:51.207307626Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1532,"":"downstairs","":"downstairs","session_id":"a65e1fdc-cf45-4333-b18b-8c172fb7d1fa"} {"msg":"[2]R flush_numbers[0..12]: [145, 159, 159, 161, 146, 163, 163, 163, 161, 163, 163, 163]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:51.207321986Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1532,"":"downstairs","":"downstairs","session_id":"a65e1fdc-cf45-4333-b18b-8c172fb7d1fa"} {"msg":"[2]R generation[0..12]: [76, 83, 83, 85, 77, 87, 87, 87, 85, 87, 87, 87]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:51.207335525Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1532,"":"downstairs","":"downstairs","session_id":"a65e1fdc-cf45-4333-b18b-8c172fb7d1fa"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:51.207350334Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1532,"":"downstairs","":"downstairs","session_id":"a65e1fdc-cf45-4333-b18b-8c172fb7d1fa"} {"msg":"Max found gen is 88","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:51.207363413Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1532,"":"downstairs","":"downstairs","session_id":"a65e1fdc-cf45-4333-b18b-8c172fb7d1fa"} {"msg":"Generation requested: 88 >= found:88","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:51.207377323Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1532,"":"downstairs","":"downstairs","session_id":"a65e1fdc-cf45-4333-b18b-8c172fb7d1fa"} {"msg":"Next flush: 164","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:51.207394092Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1532,"":"downstairs","":"downstairs","session_id":"a65e1fdc-cf45-4333-b18b-8c172fb7d1fa"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-08T20:25:51.207408841Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1532,"":"downstairs","":"downstairs","session_id":"a65e1fdc-cf45-4333-b18b-8c172fb7d1fa"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:51.20742805Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1532,"session_id":"a65e1fdc-cf45-4333-b18b-8c172fb7d1fa"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:51.20744109Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1532,"client":"0","":"downstairs","session_id":"a65e1fdc-cf45-4333-b18b-8c172fb7d1fa"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:51.207455179Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1532,"client":"1","":"downstairs","session_id":"a65e1fdc-cf45-4333-b18b-8c172fb7d1fa"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:51.207469028Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1532,"client":"2","":"downstairs","session_id":"a65e1fdc-cf45-4333-b18b-8c172fb7d1fa"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:51.207482998Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1532,"session_id":"a65e1fdc-cf45-4333-b18b-8c172fb7d1fa"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:51.207495307Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1532,"session_id":"a65e1fdc-cf45-4333-b18b-8c172fb7d1fa"} {"msg":"f7b844e7-93d5-4ba8-bd3e-9c698ab68ce6 is now active with session: a65e1fdc-cf45-4333-b18b-8c172fb7d1fa","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:51.207508216Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1532,"session_id":"a65e1fdc-cf45-4333-b18b-8c172fb7d1fa"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:51.207521136Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1532,"session_id":"a65e1fdc-cf45-4333-b18b-8c172fb7d1fa"} {"msg":"The guest has finished waiting for activation with:88","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:51.207533455Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1532} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 247 len 40960 data: 10 10 11 12 11 14 13 12 10 6 02/30 Write block 376 len 20480 data: 7 9 10 10 9 03/30 Read block 497 len 12288 04/30 Read block 222 len 24576 05/30 Flush 06/30 Write block 321 len 36864 data: 7 8 8 7 6 7 5 5 5 07/30 Write block 286 len 36864 data: 11 12 13 10 9 8 9 8 8 08/30 Write block 568 len 16384 data: 7 8 6 9 09/30 Write block 151 len 8192 data: 11 8 10/30 Write block 32 len 12288 data: 8 8 8 11/30 Read block 317 len 24576 12/30 Read block 209 len 32768 13/30 Write block 351 len 40960 data: 8 8 11 9 12 12 12 13 13 13 14/30 Write block 475 len 36864 data: 10 9 9 10 10 8 9 8 10 15/30 Write block 295 len 36864 data: 6 4 6 7 10 10 11 10 11 16/30 Write block 222 len 4096 data: 8 17/30 Write block 113 len 28672 data: 5 6 5 5 5 5 8 18/30 Read block 308 len 20480 19/30 Write block 409 len 16384 data: 7 9 10 10 20/30 Read block 396 len 36864 21/30 Read block 70 len 4096 22/30 Read block 93 len 24576 23/30 Write block 465 len 16384 data: 3 6 7 9 24/30 Write block 53 len 24576 data: 4 7 7 6 6 5 25/30 Read block 149 len 36864 26/30 Read block 200 len 32768 27/30 Write block 442 len 28672 data: 13 9 9 8 10 9 8 28/30 Read block 378 len 20480 29/30 Write block 27 len 12288 data: 9 7 5 30/30 Write block 11 len 36864 data: 6 10 9 7 9 9 8 6 7 ---------------------------------------------------------------- Crucible gen:88 GIO:true work queues: Upstairs:0 downstairs:25 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1005 Acked Write 9 Done Done Done false 1006 Acked Write 9 Done Done Done false 1007 Acked Write 4 Done Done Done false 1008 Acked Write 2 Done Done Done false 1009 Acked Write 3 Done Done Done false 1010 Acked Read 6 Done Done Done false 1011 Acked Read 8 Done Done Done false 1012 Acked Write 10 Done Done Done false 1013 Acked Write 9 Done Done Done false 1014 Acked Write 9 Done Done Done false 1015 Acked Write 1 Done Done Done false 1016 Acked Write 7 Done Done Done false 1017 Acked Read 5 Done Done Done false 1018 Acked Write 4 Done Done Done false 1019 Acked Read 9 Done Done Done false 1020 Acked Read 1 Done Done Done false 1021 Acked Read 6 Done Done Done false 1022 Acked Write 4 Done Done Done false 1023 Acked Write 6 Done Done Done false 1024 Acked Read 9 Done Done Done false 1025 Acked Read 8 Done Done Done false 1026 Acked Write 7 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Write 3 Done Done Done false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 24 24 24 72 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1004 1004 1004 Downstairs last five completed: 1004 1003 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:25:51.228120669Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1532} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 76 76 76 145 145 145 T T T 1 020-039 83 83 83 159 159 159 T T T 2 040-059 83 83 83 159 159 159 T T T 3 060-079 85 85 85 161 161 161 F F F 4 080-099 77 77 77 146 146 146 F F F 5 100-119 87 87 87 163 163 163 T T T 6 120-139 87 87 87 163 163 163 F F F 7 140-159 87 87 87 163 163 163 T T T 8 160-179 85 85 85 161 161 161 F F F 9 180-199 87 87 87 163 163 163 F F F 10 200-219 87 87 87 163 163 163 F F F 11 220-239 87 87 87 163 163 163 T T T 12 240-259 88 88 88 164 164 164 F F F 13 260-279 85 85 85 161 161 161 F F F 14 280-299 85 85 85 161 161 161 T T T 15 300-319 87 87 87 163 163 163 T T T 16 320-339 87 87 87 163 163 163 T T T 17 340-359 87 87 87 163 163 163 T T T 18 360-379 88 88 88 164 164 164 T T T 19 380-399 88 88 88 164 164 164 F F F 20 400-419 80 80 80 156 156 156 T T T 21 420-439 87 87 87 163 163 163 F F F 22 440-459 87 87 87 163 163 163 T T T 23 460-479 87 87 87 163 163 163 T T T 24 480-499 82 82 82 158 158 158 T T T 25 500-519 85 85 85 161 161 161 F F F 26 520-539 85 85 85 161 161 161 F F F 27 540-559 83 83 83 159 159 159 F F F 28 560-579 87 87 87 163 163 163 T T T 29 580-599 82 82 82 158 158 158 F F F Max gen: 88, Max flush: 164 A Difference in extent metadata is expected here On loop 44 Verifying data now No change after verify Loop: 44 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 89 89 89 165 165 165 F F F 1 020-039 89 89 89 165 165 165 F F F 2 040-059 89 89 89 165 165 165 F F F 3 060-079 85 85 85 161 161 161 F F F 4 080-099 77 77 77 146 146 146 F F F 5 100-119 89 89 89 165 165 165 F F F 6 120-139 87 87 87 163 163 163 F F F 7 140-159 89 89 89 165 165 165 F F F 8 160-179 85 85 85 161 161 161 F F F 9 180-199 87 87 87 163 163 163 F F F 10 200-219 87 87 87 163 163 163 F F F 11 220-239 89 89 89 165 165 165 F F F 12 240-259 88 88 88 164 164 164 F F F 13 260-279 85 85 85 161 161 161 F F F 14 280-299 89 89 89 165 165 165 F F F 15 300-319 89 89 89 165 165 165 F F F 16 320-339 89 89 89 165 165 165 F F F 17 340-359 89 89 89 165 165 165 F F F 18 360-379 89 89 89 165 165 165 F F F 19 380-399 88 88 88 164 164 164 F F F 20 400-419 89 89 89 165 165 165 F F F 21 420-439 87 87 87 163 163 163 F F F 22 440-459 89 89 89 165 165 165 F F F 23 460-479 89 89 89 165 165 165 F F F 24 480-499 89 89 89 165 165 165 F F F 25 500-519 85 85 85 161 161 161 F F F 26 520-539 85 85 85 161 161 161 F F F 27 540-559 83 83 83 159 159 159 F F F 28 560-579 89 89 89 165 165 165 F F F 29 580-599 82 82 82 158 158 158 F F F Max gen: 89, Max flush: 165 Begin loop 45 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:55.713769838Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1540,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:55.758136639Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1540,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:55.758767788Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1540} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"0fd98d0d5d909611f8b408ccda2559d35fe117bb\",\n git_commit_timestamp: \"2024-11-08T20:05:44.000000000Z\",\n git_branch: \"mkeeter/drop-skipped-backpressure-guard\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:55.758805286Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1540} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:55.758819546Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1540} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:55.758834145Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1540} {"msg":"Crucible f3bac724-a991-4a9b-810e-2c57f67e1de5 has session id: 13878dd2-da17-43f8-9a6d-c4c51ba1238e","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:55.759290803Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1540,"session_id":"13878dd2-da17-43f8-9a6d-c4c51ba1238e"} {"msg":"Upstairs opts: Upstairs UUID: f3bac724-a991-4a9b-810e-2c57f67e1de5, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:55.759330691Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1540,"session_id":"13878dd2-da17-43f8-9a6d-c4c51ba1238e"} {"msg":"Crucible stats registered with UUID: f3bac724-a991-4a9b-810e-2c57f67e1de5","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:55.75934663Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1540,"session_id":"13878dd2-da17-43f8-9a6d-c4c51ba1238e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:55.791208472Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1540,"task":"crutest"} {"msg":"The guest has requested activation with gen:90","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:55.791283318Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1540} {"msg":"f3bac724-a991-4a9b-810e-2c57f67e1de5 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:55.791302548Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1540,"session_id":"13878dd2-da17-43f8-9a6d-c4c51ba1238e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:55.791318957Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1540,"client":"0","":"downstairs","session_id":"13878dd2-da17-43f8-9a6d-c4c51ba1238e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:55.791342946Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1540,"client":"0","":"downstairs","session_id":"13878dd2-da17-43f8-9a6d-c4c51ba1238e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:55.791360515Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1540,"client":"1","":"downstairs","session_id":"13878dd2-da17-43f8-9a6d-c4c51ba1238e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:55.791377444Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1540,"client":"1","":"downstairs","session_id":"13878dd2-da17-43f8-9a6d-c4c51ba1238e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:55.791394073Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1540,"client":"2","":"downstairs","session_id":"13878dd2-da17-43f8-9a6d-c4c51ba1238e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:55.791410742Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1540,"client":"2","":"downstairs","session_id":"13878dd2-da17-43f8-9a6d-c4c51ba1238e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:55.791436331Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1540,"":"io task","client":"2","":"downstairs","session_id":"13878dd2-da17-43f8-9a6d-c4c51ba1238e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:55.791495078Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1540,"":"io task","client":"0","":"downstairs","session_id":"13878dd2-da17-43f8-9a6d-c4c51ba1238e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:55.791542236Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1540,"":"io task","client":"1","":"downstairs","session_id":"13878dd2-da17-43f8-9a6d-c4c51ba1238e"} {"msg":"ds_connection connected from Ok(127.0.0.1:34974)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:55.791573054Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1540,"":"io task","client":"0","":"downstairs","session_id":"13878dd2-da17-43f8-9a6d-c4c51ba1238e"} {"msg":"ds_connection connected from Ok(127.0.0.1:35696)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:55.791751875Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1540,"":"io task","client":"2","":"downstairs","session_id":"13878dd2-da17-43f8-9a6d-c4c51ba1238e"} {"msg":"ds_connection connected from Ok(127.0.0.1:64627)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:55.791810563Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1540,"":"io task","client":"1","":"downstairs","session_id":"13878dd2-da17-43f8-9a6d-c4c51ba1238e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:55.792325427Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1540,"client":"1","":"downstairs","session_id":"13878dd2-da17-43f8-9a6d-c4c51ba1238e"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:55.792360916Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1540,"client":"1","":"downstairs","session_id":"13878dd2-da17-43f8-9a6d-c4c51ba1238e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:55.792392594Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1540,"client":"2","":"downstairs","session_id":"13878dd2-da17-43f8-9a6d-c4c51ba1238e"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:55.792426953Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1540,"client":"2","":"downstairs","session_id":"13878dd2-da17-43f8-9a6d-c4c51ba1238e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:55.792506509Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1540,"client":"0","":"downstairs","session_id":"13878dd2-da17-43f8-9a6d-c4c51ba1238e"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:55.792539897Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1540,"client":"0","":"downstairs","session_id":"13878dd2-da17-43f8-9a6d-c4c51ba1238e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:55.793057072Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1540,"client":"0","":"downstairs","session_id":"13878dd2-da17-43f8-9a6d-c4c51ba1238e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:55.79308688Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1540,"client":"2","":"downstairs","session_id":"13878dd2-da17-43f8-9a6d-c4c51ba1238e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:55.793102589Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1540,"client":"1","":"downstairs","session_id":"13878dd2-da17-43f8-9a6d-c4c51ba1238e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:55.793645193Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1540,"client":"1","":"downstairs","session_id":"13878dd2-da17-43f8-9a6d-c4c51ba1238e"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:55.793671692Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1540,"client":"1","":"downstairs","session_id":"13878dd2-da17-43f8-9a6d-c4c51ba1238e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:55.793685881Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1540,"session_id":"13878dd2-da17-43f8-9a6d-c4c51ba1238e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:55.79369838Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1540,"client":"0","":"downstairs","session_id":"13878dd2-da17-43f8-9a6d-c4c51ba1238e"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:55.793715399Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1540,"client":"0","":"downstairs","session_id":"13878dd2-da17-43f8-9a6d-c4c51ba1238e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:55.793729119Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1540,"session_id":"13878dd2-da17-43f8-9a6d-c4c51ba1238e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:55.793741488Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1540,"client":"2","":"downstairs","session_id":"13878dd2-da17-43f8-9a6d-c4c51ba1238e"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:55.793755138Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1540,"client":"2","":"downstairs","session_id":"13878dd2-da17-43f8-9a6d-c4c51ba1238e"} {"msg":"[0]R flush_numbers[0..12]: [165, 165, 165, 161, 146, 165, 163, 165, 161, 163, 163, 165]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:55.793769047Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1540,"":"downstairs","":"downstairs","session_id":"13878dd2-da17-43f8-9a6d-c4c51ba1238e"} {"msg":"[0]R generation[0..12]: [89, 89, 89, 85, 77, 89, 87, 89, 85, 87, 87, 89]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:55.793783026Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1540,"":"downstairs","":"downstairs","session_id":"13878dd2-da17-43f8-9a6d-c4c51ba1238e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:55.793796995Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1540,"":"downstairs","":"downstairs","session_id":"13878dd2-da17-43f8-9a6d-c4c51ba1238e"} {"msg":"[1]R flush_numbers[0..12]: [165, 165, 165, 161, 146, 165, 163, 165, 161, 163, 163, 165]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:55.793810895Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1540,"":"downstairs","":"downstairs","session_id":"13878dd2-da17-43f8-9a6d-c4c51ba1238e"} {"msg":"[1]R generation[0..12]: [89, 89, 89, 85, 77, 89, 87, 89, 85, 87, 87, 89]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:55.793824434Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1540,"":"downstairs","":"downstairs","session_id":"13878dd2-da17-43f8-9a6d-c4c51ba1238e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:55.793838153Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1540,"":"downstairs","":"downstairs","session_id":"13878dd2-da17-43f8-9a6d-c4c51ba1238e"} {"msg":"[2]R flush_numbers[0..12]: [165, 165, 165, 161, 146, 165, 163, 165, 161, 163, 163, 165]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:55.793851933Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1540,"":"downstairs","":"downstairs","session_id":"13878dd2-da17-43f8-9a6d-c4c51ba1238e"} {"msg":"[2]R generation[0..12]: [89, 89, 89, 85, 77, 89, 87, 89, 85, 87, 87, 89]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:55.793865482Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1540,"":"downstairs","":"downstairs","session_id":"13878dd2-da17-43f8-9a6d-c4c51ba1238e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:55.793879282Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1540,"":"downstairs","":"downstairs","session_id":"13878dd2-da17-43f8-9a6d-c4c51ba1238e"} {"msg":"Max found gen is 90","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:55.793892271Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1540,"":"downstairs","":"downstairs","session_id":"13878dd2-da17-43f8-9a6d-c4c51ba1238e"} {"msg":"Generation requested: 90 >= found:90","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:55.79390695Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1540,"":"downstairs","":"downstairs","session_id":"13878dd2-da17-43f8-9a6d-c4c51ba1238e"} {"msg":"Next flush: 166","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:55.7939201Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1540,"":"downstairs","":"downstairs","session_id":"13878dd2-da17-43f8-9a6d-c4c51ba1238e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:55.793933359Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1540,"":"downstairs","":"downstairs","session_id":"13878dd2-da17-43f8-9a6d-c4c51ba1238e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:55.793946818Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":Wait for a query_work_queue command to finish before sending IO 1540,"session_id":"13878dd2-da17-43f8-9a6d-c4c51ba1238e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:55.793967807Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1540,"client":"0","":"downstairs","session_id":"13878dd2-da17-43f8-9a6d-c4c51ba1238e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:55.793988006Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1540,"client":"1","":"downstairs","session_id":"13878dd2-da17-43f8-9a6d-c4c51ba1238e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:55.794033184Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1540,"client":"2","":"downstairs","session_id":"13878dd2-da17-43f8-9a6d-c4c51ba1238e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:55.794052213Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1540,"session_id":"13878dd2-da17-43f8-9a6d-c4c51ba1238e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:55.794064962Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1540,"session_id":"13878dd2-da17-43f8-9a6d-c4c51ba1238e"} {"msg":"f3bac724-a991-4a9b-810e-2c57f67e1de5 is now active with session: 13878dd2-da17-43f8-9a6d-c4c51ba1238e","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:55.794086981Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1540,"session_id":"13878dd2-da17-43f8-9a6d-c4c51ba1238e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:55.79411193Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1540,"session_id":"13878dd2-da17-43f8-9a6d-c4c51ba1238e"} {"msg":"The guest has finished waiting for activation with:90","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:25:55.794137419Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1540} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 418 len 16384 data: 6 6 6 8 02/30 Read block 250 len 24576 03/30 Write block 546 len 28672 data: 9 6 8 7 6 4 6 04/30 Read block 241 len 36864 05/30 Read block 511 len 24576 06/30 Read block 329 len 28672 07/30 Read block 450 len 4096 08/30 Read block 93 len 32768 09/30 Write block 333 len 8192 data: 7 8 10/30 Write block 417 len 36864 data: 5 7 7 7 9 7 7 6 7 11/30 Flush 12/30 Write block 585 len 12288 data: 13 11 12 13/30 Write block 290 len 36864 data: 10 9 10 9 9 7 5 7 8 14/30 Flush 15/30 Write block 392 len 28672 data: 8 9 9 8 8 10 10 16/30 Read block 485 len 16384 17/30 Read block 259 len 20480 18/30 Read block 586 len 40960 19/30 Write block 381 len 8192 data: 8 7 20/30 Flush 21/30 Write block 293 len 40960 data: 10 10 8 6 8 9 11 11 12 11 22/30 Read block 215 len 40960 23/30 Flush 24/30 Write block 415 len 24576 data: 7 6 6 8 8 8 25/30 Read block 572 len 40960 26/30 Read block 53 len 4096 27/30 Write block 241 len 16384 data: 6 7 7 7 28/30 Read block 294 len 28672 29/30 Read block 555 len 40960 30/30 Read block 127 len 40960 ---------------------------------------------------------------- Crucible gen:90 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 6 Done Done Done false 1024 Acked Read 10 Done Done Done false 1025 Acked Read 1 Done Done Done false 1026 Acked Write 4 Done Done Done false 1027 Acked Read 7 Done Done Done false 1028 Acked Read 10 Done Done Done false 1029 Acked Read 10 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 0 1 Done 7 6 7 20 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:25:55.826631521Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1540} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 89 89 89 165 165 165 F F F 1 020-039 89 89 89 165 165 165 F F F 2 040-059 89 89 89 165 165 165 F F F 3 060-079 85 85 85 161 161 161 F F F 4 080-099 77 77 77 146 146 146 F F F 5 100-119 89 89 89 165 165 165 F F F 6 120-139 87 87 87 163 163 163 F F F 7 140-159 89 89 89 165 165 165 F F F 8 160-179 85 85 85 161 161 161 F F F 9 180-199 87 87 87 163 163 163 F F F 10 200-219 87 87 87 163 163 163 F F F 11 220-239 89 89 89 165 165 165 F F F 12 240-259 88 88 88 164 164 164 T T T 13 260-279 85 85 85 161 161 161 F F F 14 280-299 90 90 90 169 169 169 F F F 15 300-319 90 90 90 169 169 169 F F F 16 320-339 90 90 90 166 166 166 F F F 17 340-359 89 89 89 165 165 165 F F F 18 360-379 89 89 89 165 165 165 F F F 19 380-399 90 90 90 168 168 168 F F F 20 400-419 90 90 90 166 166 166 T T T 21 420-439 90 90 90 166 166 166 T T T 22 440-459 89 89 89 165 165 165 F F F 23 460-479 89 89 89 165 165 165 F F F 24 480-499 89 89 89 165 165 165 F F F 25 500-519 85 85 85 161 161 161 F F F 26 520-539 85 85 85 161 161 161 F F F 27 540-559 90 90 90 166 166 166 F F F 28 560-579 89 89 89 165 165 165 F F F 29 580-599 90 90 90 167 167 167 F F F Max gen: 90, Max flush: 169 A Difference in extent metadata is expected here On loop 45 Verifying data now No change after verify Loop: 45 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 89 89 89 165 165 165 F F F 1 020-039 89 89 89 165 165 165 F F F 2 040-059 89 89 89 165 165 165 F F F 3 060-079 85 85 85 161 161 161 F F F 4 080-099 77 77 77 146 146 146 F F F 5 100-119 89 89 89 165 165 165 F F F 6 120-139 87 87 87 163 163 163 F F F 7 140-159 89 89 89 165 165 165 F F F 8 160-179 85 85 85 161 161 161 F F F 9 180-199 87 87 87 163 163 163 F F F 10 200-219 87 87 87 163 163 163 F F F 11 220-239 89 89 89 165 165 165 F F F 12 240-259 91 91 91 170 170 170 F F F 13 260-279 85 85 85 161 161 161 F F F 14 280-299 90 90 90 169 169 169 F F F 15 300-319 90 90 90 169 169 169 F F F 16 320-339 90 90 90 166 166 166 F F F 17 340-359 89 89 89 165 165 165 F F F 18 360-379 89 89 89 165 165 165 F F F 19 380-399 90 90 90 168 168 168 F F F 20 400-419 91 91 91 170 170 170 F F F 21 420-439 91 91 91 170 170 170 F F F 22 440-459 89 89 89 165 165 165 F F F 23 460-479 89 89 89 165 165 165 F F F 24 480-499 89 89 89 165 165 165 F F F 25 500-519 85 85 85 161 161 161 F F F 26 520-539 85 85 85 161 161 161 F F F 27 540-559 90 90 90 166 166 166 F F F 28 560-579 89 89 89 165 165 165 F F F 29 580-599 90 90 90 167 167 167 F F F Max gen: 91, Max flush: 170 Begin loop 46 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:00.115911455Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1548,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:00.160320555Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1548,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:00.160982183Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1548} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"0fd98d0d5d909611f8b408ccda2559d35fe117bb\",\n git_commit_timestamp: \"2024-11-08T20:05:44.000000000Z\",\n git_branch: \"mkeeter/drop-skipped-backpressure-guard\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:00.161017811Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1548} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:00.16103211Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1548} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:00.161046849Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1548} {"msg":"Crucible a8658c84-36f4-449e-95e8-a88c316c41c7 has session id: 2c17e291-f10f-4b54-9857-30b4dc468553","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:00.16163762Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1548,"session_id":"2c17e291-f10f-4b54-9857-30b4dc468553"} {"msg":"Upstairs opts: Upstairs UUID: a8658c84-36f4-449e-95e8-a88c316c41c7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:00.161685648Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1548,"session_id":"2c17e291-f10f-4b54-9857-30b4dc468553"} {"msg":"Crucible stats registered with UUID: a8658c84-36f4-449e-95e8-a88c316c41c7","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:00.161711797Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1548,"session_id":"2c17e291-f10f-4b54-9857-30b4dc468553"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:00.192712252Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1548,"task":"crutest"} {"msg":"The guest has requested activation with gen:92","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:00.192759449Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1548} {"msg":"a8658c84-36f4-449e-95e8-a88c316c41c7 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:00.192795557Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1548,"session_id":"2c17e291-f10f-4b54-9857-30b4dc468553"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:00.192826306Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1548,"client":"0","":"downstairs","session_id":"2c17e291-f10f-4b54-9857-30b4dc468553"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:00.192872864Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1548,"client":"0","":"downstairs","session_id":"2c17e291-f10f-4b54-9857-30b4dc468553"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:00.192910622Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1548,"client":"1","":"downstairs","session_id":"2c17e291-f10f-4b54-9857-30b4dc468553"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:00.19294523Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1548,"client":"1","":"downstairs","session_id":"2c17e291-f10f-4b54-9857-30b4dc468553"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:00.192977849Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1548,"client":"2","":"downstairs","session_id":"2c17e291-f10f-4b54-9857-30b4dc468553"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:00.193011457Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1548,"client":"2","":"downstairs","session_id":"2c17e291-f10f-4b54-9857-30b4dc468553"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:00.193042496Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1548,"":"io task","client":"2","":"downstairs","session_id":"2c17e291-f10f-4b54-9857-30b4dc468553"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:00.193076324Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1548,"":"io task","client":"0","":"downstairs","session_id":"2c17e291-f10f-4b54-9857-30b4dc468553"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:00.193112062Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1548,"":"io task","client":"1","":"downstairs","session_id":"2c17e291-f10f-4b54-9857-30b4dc468553"} {"msg":"ds_connection connected from Ok(127.0.0.1:65020)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:00.19314851Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1548,"":"io task","client":"0","":"downstairs","session_id":"2c17e291-f10f-4b54-9857-30b4dc468553"} {"msg":"ds_connection connected from Ok(127.0.0.1:52958)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:00.193182489Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1548,"":"io task","client":"1","":"downstairs","session_id":"2c17e291-f10f-4b54-9857-30b4dc468553"} {"msg":"ds_connection connected from Ok(127.0.0.1:58220)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:00.193214157Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1548,"":"io task","client":"2","":"downstairs","session_id":"2c17e291-f10f-4b54-9857-30b4dc468553"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:00.193860416Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1548,"client":"2","":"downstairs","session_id":"2c17e291-f10f-4b54-9857-30b4dc468553"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:00.193897754Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1548,"client":"2","":"downstairs","session_id":"2c17e291-f10f-4b54-9857-30b4dc468553"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:00.193935672Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1548,"client":"0","":"downstairs","session_id":"2c17e291-f10f-4b54-9857-30b4dc468553"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:00.19396718Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1548,"client":"0","":"downstairs","session_id":"2c17e291-f10f-4b54-9857-30b4dc468553"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:00.193998349Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1548,"client":"1","":"downstairs","session_id":"2c17e291-f10f-4b54-9857-30b4dc468553"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:00.194026117Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1548,"client":"1","":"downstairs","session_id":"2c17e291-f10f-4b54-9857-30b4dc468553"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:00.194545442Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1548,"client":"2","":"downstairs","session_id":"2c17e291-f10f-4b54-9857-30b4dc468553"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:00.194577641Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1548,"client":"1","":"downstairs","session_id":"2c17e291-f10f-4b54-9857-30b4dc468553"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:00.194605539Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1548,"client":"0","":"downstairs","session_id":"2c17e291-f10f-4b54-9857-30b4dc468553"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:00.195088985Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1548,"client":"2","":"downstairs","session_id":"2c17e291-f10f-4b54-9857-30b4dc468553"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:00.195130123Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1548,"client":"2","":"downstairs","session_id":"2c17e291-f10f-4b54-9857-30b4dc468553"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:00.195160982Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1548,"session_id":"2c17e291-f10f-4b54-9857-30b4dc468553"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:00.195223759Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1548,"client":"1","":"downstairs","session_id":"2c17e291-f10f-4b54-9857-30b4dc468553"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:00.195251788Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1548,"client":"1","":"downstairs","session_id":"2c17e291-f10f-4b54-9857-30b4dc468553"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:00.195267217Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1548,"session_id":"2c17e291-f10f-4b54-9857-30b4dc468553"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:00.195280726Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1548,"client":"0","":"downstairs","session_id":"2c17e291-f10f-4b54-9857-30b4dc468553"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:00.195294735Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1548,"client":"0","":"downstairs","session_id":"2c17e291-f10f-4b54-9857-30b4dc468553"} {"msg":"[0]R flush_numbers[0..12]: [165, 165, 165, 161, 146, 165, 163, 165, 161, 163, 163, 165]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:00.195313045Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1548,"":"downstairs","":"downstairs","session_id":"2c17e291-f10f-4b54-9857-30b4dc468553"} {"msg":"[0]R generation[0..12]: [89, 89, 89, 85, 77, 89, 87, 89, 85, 87, 87, 89]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:00.195327274Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1548,"":"downstairs","":"downstairs","session_id":"2c17e291-f10f-4b54-9857-30b4dc468553"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:00.195341783Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1548,"":"downstairs","":"downstairs","session_id":"2c17e291-f10f-4b54-9857-30b4dc468553"} {"msg":"[1]R flush_numbers[0..12]: [165, 165, 165, 161, 146, 165, 163, 165, 161, 163, 163, 165]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:00.195355832Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1548,"":"downstairs","":"downstairs","session_id":"2c17e291-f10f-4b54-9857-30b4dc468553"} {"msg":"[1]R generation[0..12]: [89, 89, 89, 85, 77, 89, 87, 89, 85, 87, 87, 89]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:00.195369722Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1548,"":"downstairs","":"downstairs","session_id":"2c17e291-f10f-4b54-9857-30b4dc468553"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:00.195383771Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1548,"":"downstairs","":"downstairs","session_id":"2c17e291-f10f-4b54-9857-30b4dc468553"} {"msg":"[2]R flush_numbers[0..12]: [165, 165, 165, 161, 146, 165, 163, 165, 161, 163, 163, 165]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:00.19539833Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1548,"":"downstairs","":"downstairs","session_id":"2c17e291-f10f-4b54-9857-30b4dc468553"} {"msg":"[2]R generation[0..12]: [89, 89, 89, 85, 77, 89, 87, 89, 85, 87, 87, 89]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:00.19541217Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1548,"":"downstairs","":"downstairs","session_id":"2c17e291-f10f-4b54-9857-30b4dc468553"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:00.195425849Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1548,"":"downstairs","":"downstairs","session_id":"2c17e291-f10f-4b54-9857-30b4dc468553"} {"msg":"Max found gen is 92","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:00.195438988Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1548,"":"downstairs","":"downstairs","session_id":"2c17e291-f10f-4b54-9857-30b4dc468553"} {"msg":"Generation requested: 92 >= found:92","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:00.195452258Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1548,"":"Wait for a query_work_queue command to finish before sending IO downstairs","":"downstairs","session_id":"2c17e291-f10f-4b54-9857-30b4dc468553"} {"msg":"Next flush: 171","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:00.195474607Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1548,"":"downstairs","":"downstairs","session_id":"2c17e291-f10f-4b54-9857-30b4dc468553"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:00.195487866Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1548,"":"downstairs","":"downstairs","session_id":"2c17e291-f10f-4b54-9857-30b4dc468553"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:00.195501155Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1548,"session_id":"2c17e291-f10f-4b54-9857-30b4dc468553"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:00.195515445Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1548,"client":"0","":"downstairs","session_id":"2c17e291-f10f-4b54-9857-30b4dc468553"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:00.195528854Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1548,"client":"1","":"downstairs","session_id":"2c17e291-f10f-4b54-9857-30b4dc468553"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:00.195548243Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1548,"client":"2","":"downstairs","session_id":"2c17e291-f10f-4b54-9857-30b4dc468553"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2024-11-08T20:26:00.195569012Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1548,"session_id":"2c17e291-f10f-4b54-9857-30b4dc468553"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:00.195584381Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1548,"session_id":"2c17e291-f10f-4b54-9857-30b4dc468553"} {"msg":"a8658c84-36f4-449e-95e8-a88c316c41c7 is now active with session: 2c17e291-f10f-4b54-9857-30b4dc468553","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:00.195597501Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1548,"session_id":"2c17e291-f10f-4b54-9857-30b4dc468553"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:00.19560978Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1548,"session_id":"2c17e291-f10f-4b54-9857-30b4dc468553"} {"msg":"The guest has finished waiting for activation with:92","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:00.195622239Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1548} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 242 len 36864 data: 8 8 8 7 7 11 11 12 13 03/30 Write block 241 len 12288 data: 7 9 9 04/30 Write block 10 len 36864 data: 6 7 11 10 8 10 10 9 7 05/30 Read block 369 len 28672 06/30 Read block 492 len 36864 07/30 Read block 576 len 32768 08/30 Write block 116 len 8192 data: 6 6 09/30 Write block 213 len 20480 data: 8 7 6 7 6 10/30 Write block 516 len 20480 data: 7 5 7 8 6 11/30 Write block 11 len 36864 data: 8 12 11 9 11 11 10 8 8 12/30 Read block 158 len 40960 13/30 Read block 407 len 32768 14/30 Write block 238 len 24576 data: 10 9 9 8 10 10 15/30 Write block 468 len 36864 data: 10 9 9 10 10 9 9 11 10 16/30 Read block 187 len 12288 17/30 Write block 580 len 24576 data: 7 7 7 10 11 14 18/30 Write block 71 len 12288 data: 5 6 7 19/30 Read block 594 len 16384 20/30 Read block 324 len 8192 21/30 Read block 586 len 24576 22/30 Write block 345 len 40960 data: 8 11 9 7 10 9 9 9 12 10 23/30 Read block 255 len 12288 24/30 Read block 358 len 12288 25/30 Read block 475 len 12288 26/30 Read block 166 len 36864 27/30 Read block 171 len 16384 28/30 Read block 315 len 32768 29/30 Read block 364 len 12288 30/30 Write block 463 len 20480 data: 3 2 4 7 8 ---------------------------------------------------------------- Crucible gen:92 GIO:true work queues: Upstairs:0 downstairs:29 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1001 Acked Write 9 Done Done Done false 1002 Acked Write 3 Done Done Done false 1003 Acked Write 9 Done Done Done false 1004 Acked Read 7 Done Done Done false 1005 Acked Read 9 Done Done Done false 1006 Acked Read 8 Done Done Done false 1007 Acked Write 2 Done Done Done false 1008 Acked Write 5 Done Done Done false 1009 Acked Write 5 Done Done Done false 1010 Acked Write 9 Done Done Done false 1011 Acked Read 10 Done Done Done false 1012 Acked Read 8 Done Done Done false 1013 Acked Write 6 Done Done Done false 1014 Acked Write 9 Done Done Done false 1015 Acked Read 3 Done Done Done false 1016 Acked Write 6 Done Done Done false 1017 Acked Write 3 Done Done Done false 1018 Acked Read 4 Done Done Done false 1019 Acked Read 2 Done Done Done false 1020 Acked Read 6 Done Done Done false 1021 Acked Write 10 Done Done Done false 1022 Acked Read 3 Done Done Done false 1023 Acked Read 3 Done Done Done false 1024 Acked Read 3 Done Done Done false 1025 Acked Read 9 Done Done Done false 1026 Acked Read 4 Done Done Done false 1027 Acked Read 8 Done Done Done false 1028 Acked Read 3 Done Done Done false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 28 28 28 84 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1000 1000 1000 Downstairs last five completed: 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:26:00.217941808Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1548} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 89 89 89 165 165 165 T T T 1 020-039 89 89 89 165 165 165 F F F 2 040-059 89 89 89 165 165 165 F F F 3 060-079 85 85 85 161 161 161 T T T 4 080-099 77 77 77 146 146 146 F F F 5 100-119 89 89 89 165 165 165 T T T 6 120-139 87 87 87 163 163 163 F F F 7 140-159 89 89 89 165 165 165 F F F 8 160-179 85 85 85 161 161 161 F F F 9 180-199 87 87 87 163 163 163 F F F 10 200-219 87 87 87 163 163 163 T T T 11 220-239 89 89 89 165 165 165 T T T 12 240-259 91 91 91 170 170 170 T T T 13 260-279 85 85 85 161 161 161 F F F 14 280-299 90 90 90 169 169 169 F F F 15 300-319 90 90 90 169 169 169 F F F 16 320-339 90 90 90 166 166 166 F F F 17 340-359 89 89 89 165 165 165 T T T 18 360-379 89 89 89 165 165 165 F F F 19 380-399 90 90 90 168 168 168 F F F 20 400-419 91 91 91 170 170 170 F F F 21 420-439 91 91 91 170 170 170 F F F 22 440-459 89 89 89 165 165 165 F F F 23 460-479 89 89 89 165 165 165 T T T 24 480-499 89 89 89 165 165 165 F F F 25 500-519 85 85 85 161 161 161 T T T 26 520-539 85 85 85 161 161 161 T T T 27 540-559 90 90 90 166 166 166 F F F 28 560-579 89 89 89 165 165 165 F F F 29 580-599 90 90 90 167 167 167 T T T Max gen: 91, Max flush: 170 A Difference in extent metadata is expected here On loop 46 Verifying data now No change after verify Loop: 46 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 92 92 92 171 171 171 F F F 1 020-039 89 89 89 165 165 165 F F F 2 040-059 89 89 89 165 165 165 F F F 3 060-079 92 92 92 171 171 171 F F F 4 080-099 77 77 77 146 146 146 F F F 5 100-119 92 92 92 171 171 171 F F F 6 120-139 87 87 87 163 163 163 F F F 7 140-159 89 89 89 165 165 165 F F F 8 160-179 85 85 85 161 161 161 F F F 9 180-199 87 87 87 163 163 163 F F F 10 200-219 92 92 92 171 171 171 F F F 11 220-239 92 92 92 171 171 171 F F F 12 240-259 92 92 92 171 171 171 F F F 13 260-279 85 85 85 161 161 161 F F F 14 280-299 90 90 90 169 169 169 F F F 15 300-319 90 90 90 169 169 169 F F F 16 320-339 90 90 90 166 166 166 F F F 17 340-359 92 92 92 171 171 171 F F F 18 360-379 89 89 89 165 165 165 F F F 19 380-399 90 90 90 168 168 168 F F F 20 400-419 91 91 91 170 170 170 F F F 21 420-439 91 91 91 170 170 170 F F F 22 440-459 89 89 89 165 165 165 F F F 23 460-479 92 92 92 171 171 171 F F F 24 480-499 89 89 89 165 165 165 F F F 25 500-519 92 92 92 171 171 171 F F F 26 520-539 92 92 92 171 171 171 F F F 27 540-559 90 90 90 166 166 166 F F F 28 560-579 89 89 89 165 165 165 F F F 29 580-599 92 92 92 171 171 171 F F F Max gen: 92, Max flush: 171 Begin loop 47 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:04.631347968Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1556,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:04.676826115Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1556,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:04.677499492Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1556} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"0fd98d0d5d909611f8b408ccda2559d35fe117bb\",\n git_commit_timestamp: \"2024-11-08T20:05:44.000000000Z\",\n git_branch: \"mkeeter/drop-skipped-backpressure-guard\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:04.6775376Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1556} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:04.67755244Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1556} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:04.677566789Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1556} {"msg":"Crucible ae46ba37-ba44-4014-91ec-2df2d17521da has session id: d1d5886e-edaf-48aa-87b9-3b35bdb1fe24","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:04.678042596Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1556,"session_id":"d1d5886e-edaf-48aa-87b9-3b35bdb1fe24"} {"msg":"Upstairs opts: Upstairs UUID: ae46ba37-ba44-4014-91ec-2df2d17521da, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:04.678089173Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1556,"session_id":"d1d5886e-edaf-48aa-87b9-3b35bdb1fe24"} {"msg":"Crucible stats registered with UUID: ae46ba37-ba44-4014-91ec-2df2d17521da","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:04.678117422Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1556,"session_id":"d1d5886e-edaf-48aa-87b9-3b35bdb1fe24"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:04.709912778Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1556,"task":"crutest"} {"msg":"The guest has requested activation with gen:94","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:04.709944916Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1556} {"msg":"ae46ba37-ba44-4014-91ec-2df2d17521da active request set","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:04.710018373Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1556,"session_id":"d1d5886e-edaf-48aa-87b9-3b35bdb1fe24"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:04.710040932Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1556,"client":"0","":"downstairs","session_id":"d1d5886e-edaf-48aa-87b9-3b35bdb1fe24"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:04.71006953Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1556,"client":"0","":"downstairs","session_id":"d1d5886e-edaf-48aa-87b9-3b35bdb1fe24"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:04.710087709Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1556,"client":"1","":"downstairs","session_id":"d1d5886e-edaf-48aa-87b9-3b35bdb1fe24"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:04.710104798Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1556,"client":"1","":"downstairs","session_id":"d1d5886e-edaf-48aa-87b9-3b35bdb1fe24"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:04.710121598Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1556,"client":"2","":"downstairs","session_id":"d1d5886e-edaf-48aa-87b9-3b35bdb1fe24"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:04.710138587Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1556,"client":"2","":"downstairs","session_id":"d1d5886e-edaf-48aa-87b9-3b35bdb1fe24"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:04.710155386Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1556,"":"io task","client":"2","":"downstairs","session_id":"d1d5886e-edaf-48aa-87b9-3b35bdb1fe24"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:04.710172925Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1556,"":"io task","client":"0","":"downstairs","session_id":"d1d5886e-edaf-48aa-87b9-3b35bdb1fe24"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:04.710201144Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1556,"":"io task","client":"1","":"downstairs","session_id":"d1d5886e-edaf-48aa-87b9-3b35bdb1fe24"} {"msg":"ds_connection connected from Ok(127.0.0.1:44876)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:04.710311668Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1556,"":"io task","client":"0","":"downstairs","session_id":"d1d5886e-edaf-48aa-87b9-3b35bdb1fe24"} {"msg":"ds_connection connected from Ok(127.0.0.1:44054)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:04.710354426Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1556,"":"io task","client":"1","":"downstairs","session_id":"d1d5886e-edaf-48aa-87b9-3b35bdb1fe24"} {"msg":"ds_connection connected from Ok(127.0.0.1:43431)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:04.710379575Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1556,"":"io task","client":"2","":"downstairs","session_id":"d1d5886e-edaf-48aa-87b9-3b35bdb1fe24"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:04.71109749Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1556,"client":"2","":"downstairs","session_id":"d1d5886e-edaf-48aa-87b9-3b35bdb1fe24"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:04.711125639Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1556,"client":"2","":"downstairs","session_id":"d1d5886e-edaf-48aa-87b9-3b35bdb1fe24"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:04.711140608Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1556,"client":"0","":"downstairs","session_id":"d1d5886e-edaf-48aa-87b9-3b35bdb1fe24"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:04.711154477Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1556,"client":"0","":"downstairs","session_id":"d1d5886e-edaf-48aa-87b9-3b35bdb1fe24"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:04.7113019Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1556,"client":"1","":"downstairs","session_id":"d1d5886e-edaf-48aa-87b9-3b35bdb1fe24"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:04.711345358Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1556,"client":"1","":"downstairs","session_id":"d1d5886e-edaf-48aa-87b9-3b35bdb1fe24"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:04.711844233Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1556,"client":"0","":"downstairs","session_id":"d1d5886e-edaf-48aa-87b9-3b35bdb1fe24"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:04.711869892Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1556,"client":"2","":"downstairs","session_id":"d1d5886e-edaf-48aa-87b9-3b35bdb1fe24"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:04.711884551Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1556,"client":"1","":"downstairs","session_id":"d1d5886e-edaf-48aa-87b9-3b35bdb1fe24"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:04.712443264Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1556,"client":"1","":"downstairs","session_id":"d1d5886e-edaf-48aa-87b9-3b35bdb1fe24"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:04.712473513Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1556,"client":"1","":"downstairs","session_id":"d1d5886e-edaf-48aa-87b9-3b35bdb1fe24"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:04.712489472Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1556,"session_id":"d1d5886e-edaf-48aa-87b9-3b35bdb1fe24"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:04.712502351Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1556,"client":"2","":"downstairs","session_id":"d1d5886e-edaf-48aa-87b9-3b35bdb1fe24"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:04.712516121Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1556,"client":"2","":"downstairs","session_id":"d1d5886e-edaf-48aa-87b9-3b35bdb1fe24"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:04.71252957Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1556,"session_id":"d1d5886e-edaf-48aa-87b9-3b35bdb1fe24"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:04.712542299Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1556,"client":"0","":"downstairs","session_id":"d1d5886e-edaf-48aa-87b9-3b35bdb1fe24"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:04.712559509Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1556,"client":"0","":"downstairs","session_id":"d1d5886e-edaf-48aa-87b9-3b35bdb1fe24"} {"msg":"[0]R flush_numbers[0..12]: [171, 165, 165, 171, 146, 171, 163, 165, 161, 163, 171, 171]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:04.712573968Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1556,"":"downstairs","":"downstairs","session_id":"d1d5886e-edaf-48aa-87b9-3b35bdb1fe24"} {"msg":"[0]R generation[0..12]: [92, 89, 89, 92, 77, 92, 87, 89, 85, 87, 92, 92]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:04.712587967Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1556,"":"downstairs","":"downstairs","session_id":"d1d5886e-edaf-48aa-87b9-3b35bdb1fe24"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:04.712602047Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1556,"":"downstairs","":"downstairs","session_id":"d1d5886e-edaf-48aa-87b9-3b35bdb1fe24"} {"msg":"[1]R flush_numbers[0..12]: [171, 165, 165, 171, 146, 171, 163, 165, 161, 163, 171, 171]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:04.712615906Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1556,"":"downstairs","":"downstairs","session_id":"d1d5886e-edaf-48aa-87b9-3b35bdb1fe24"} {"msg":"[1]R generation[0..12]: [92, 89, 89, 92, 77, 92, 87, 89, 85, 87, 92, 92]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:04.712629395Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1556,"":"downstairs","":"downstairs","session_id":"d1d5886e-edaf-48aa-87b9-3b35bdb1fe24"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:04.712643405Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1556,"":"downstairs","":"downstairs","session_id":"d1d5886e-edaf-48aa-87b9-3b35bdb1fe24"} {"msg":"[2]R flush_numbers[0..12]: [171, 165, 165, 171, 146, 171, 163, 165, 161, 163, 171, 171]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:04.712657304Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1556,"":"downstairs","":"downstairs","session_id":"d1d5886e-edaf-48aa-87b9-3b35bdb1fe24"} {"msg":"[2]R generation[0..12]: [92, 89, 89, 92, 77, 92, 87, 89, 85, 87, 92, 92]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:04.712670783Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1556,"":"downstairs","":"downstairs","session_id":"d1d5886e-edaf-48aa-87b9-3b35bdb1fe24"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:04.712684683Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1556,"":"downstairs","":"Wait for a query_work_queue command to finish before sending IO downstairs","session_id":"d1d5886e-edaf-48aa-87b9-3b35bdb1fe24"} {"msg":"Max found gen is 93","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:04.712706791Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1556,"":"downstairs","":"downstairs","session_id":"d1d5886e-edaf-48aa-87b9-3b35bdb1fe24"} {"msg":"Generation requested: 94 >= found:93","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:04.712720451Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1556,"":"downstairs","":"downstairs","session_id":"d1d5886e-edaf-48aa-87b9-3b35bdb1fe24"} {"msg":"Next flush: 172","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:04.71273352Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1556,"":"downstairs","":"downstairs","session_id":"d1d5886e-edaf-48aa-87b9-3b35bdb1fe24"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:04.712748179Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1556,"":"downstairs","":"downstairs","session_id":"d1d5886e-edaf-48aa-87b9-3b35bdb1fe24"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:04.712761539Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1556,"session_id":"d1d5886e-edaf-48aa-87b9-3b35bdb1fe24"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:04.712773688Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1556,"client":"0","":"downstairs","session_id":"d1d5886e-edaf-48aa-87b9-3b35bdb1fe24"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:04.712786998Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1556,"client":"1","":"downstairs","session_id":"d1d5886e-edaf-48aa-87b9-3b35bdb1fe24"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:04.712800197Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1556,"client":"2","":"downstairs","session_id":"d1d5886e-edaf-48aa-87b9-3b35bdb1fe24"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:04.712813646Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1556,"session_id":"d1d5886e-edaf-48aa-87b9-3b35bdb1fe24"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:04.712825936Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1556,"session_id":"d1d5886e-edaf-48aa-87b9-3b35bdb1fe24"} {"msg":"ae46ba37-ba44-4014-91ec-2df2d17521da is now active with session: d1d5886e-edaf-48aa-87b9-3b35bdb1fe24","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:04.712838645Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1556,"session_id":"d1d5886e-edaf-48aa-87b9-3b35bdb1fe24"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:04.712850764Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1556,"session_id":"d1d5886e-edaf-48aa-87b9-3b35bdb1fe24"} {"msg":"The guest has finished waiting for activation with:94","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:04.712863114Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1556} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 485 len 36864 data: 7 7 6 7 9 9 7 5 7 02/30 Read block 582 len 24576 03/30 Write block 161 len 12288 data: 4 3 3 04/30 Write block 400 len 20480 data: 8 7 6 6 6 05/30 Read block 430 len 36864 06/30 Read block 7 len 24576 07/30 Read block 417 len 40960 08/30 Write block 79 len 20480 data: 7 6 5 6 7 09/30 Read block 389 len 8192 10/30 Write block 436 len 20480 data: 13 11 11 10 11 11/30 Read block 460 len 16384 12/30 Read block 95 len 20480 13/30 Read block 587 len 36864 14/30 Read block 449 len 32768 15/30 Write block 222 len 28672 data: 9 8 8 8 9 11 9 16/30 Read block 205 len 40960 17/30 Write block 400 len 36864 data: 9 8 7 7 7 7 7 7 8 18/30 Write block 531 len 16384 data: 6 5 6 6 19/30 Write block 281 len 40960 data: 6 8 7 7 8 12 13 14 11 11 20/30 Write block 466 len 40960 data: 8 9 11 10 10 11 11 10 10 12 21/30 Write block 568 len 4096 data: 8 22/30 Read block 253 len 20480 23/30 Write block 92 len 20480 data: 5 5 4 5 4 24/30 Write block 530 len 8192 data: 6 7 25/30 Read block 137 len 16384 26/30 Read block 545 len 16384 27/30 Read block 565 len 24576 28/30 Read block 187 len 32768 29/30 Read block 109 len 8192 30/30 Read block 268 len 4096 ---------------------------------------------------------------- Crucible gen:94 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 9 Done Done Done false 1001 Acked Read 6 Done Done Done false 1002 Acked Write 3 Done Done Done false 1003 Acked Write 5 Done Done Done false 1004 Acked Read 9 Done Done Done false 1005 Acked Read 6 Done Done Done false 1006 Acked Read 10 Done Done Done false 1007 Acked Write 5 Done Done Done false 1008 Acked Read 2 Done Done Done false 1009 Acked Write 5 Done Done Done false 1010 Acked Read 4 Done Done Done false 1011 Acked Read 5 Done Done Done false 1012 Acked Read 9 Done Done Done false 1013 Acked Read 8 Done Done Done false 1014 Acked Write 7 Done Done Done false 1015 Acked Read 10 Done Done Done false 1016 Acked Write 9 Done Done Done false 1017 Acked Write 4 Done Done Done false 1018 Acked Write 10 Done Done Done false 1019 Acked Write 10 Done Done Done false 1020 Acked Write 1 Done Done Done false 1021 Acked Read 5 Done Done Done false 1022 Acked Write 5 Done Done Done false 1023 Acked Write 2 Done Done Done false 1024 Acked Read 4 Done Done Done false 1025 Acked Read 4 Done Done Done false 1026 Acked Read 6 Done Done Done false 1027 Acked Read 8 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Read 1 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 30 30 30 90 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 0 0 0 Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:26:04.733275286Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1556} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 92 92 92 171 171 171 F F F 1 020-039 89 89 89 165 165 165 F F F 2 040-059 89 89 89 165 165 165 F F F 3 060-079 92 92 92 171 171 171 T T T 4 080-099 77 77 77 146 146 146 T T T 5 100-119 92 92 92 171 171 171 F F F 6 120-139 87 87 87 163 163 163 F F F 7 140-159 89 89 89 165 165 165 F F F 8 160-179 85 85 85 161 161 161 T T T 9 180-199 87 87 87 163 163 163 F F F 10 200-219 92 92 92 171 171 171 F F F 11 220-239 92 92 92 171 171 171 T T T 12 240-259 92 92 92 171 171 171 F F F 13 260-279 85 85 85 161 161 161 F F F 14 280-299 90 90 90 169 169 169 T T T 15 300-319 90 90 90 169 169 169 F F F 16 320-339 90 90 90 166 166 166 F F F 17 340-359 92 92 92 171 171 171 F F F 18 360-379 89 89 89 165 165 165 F F F 19 380-399 90 90 90 168 168 168 F F F 20 400-419 91 91 91 170 170 170 T T T 21 420-439 91 91 91 170 170 170 T T T 22 440-459 89 89 89 165 165 165 T T T 23 460-479 92 92 92 171 171 171 T T T 24 480-499 89 89 89 165 165 165 T T T 25 500-519 92 92 92 171 171 171 F F F 26 520-539 92 92 92 171 171 171 T T T 27 540-559 90 90 90 166 166 166 F F F 28 560-579 89 89 89 165 165 165 T T T 29 580-599 92 92 92 171 171 171 F F F Max gen: 92, Max flush: 171 A Difference in extent metadata is expected here On loop 47 Verifying data now No change after verify Loop: 47 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 92 92 92 171 171 171 F F F 1 020-039 89 89 89 165 165 165 F F F 2 040-059 89 89 89 165 165 165 F F F 3 060-079 93 93 93 172 172 172 F F F 4 080-099 93 93 93 172 172 172 F F F 5 100-119 92 92 92 171 171 171 F F F 6 120-139 87 87 87 163 163 163 F F F 7 140-159 89 89 89 165 165 165 F F F 8 160-179 93 93 93 172 172 172 F F F 9 180-199 87 87 87 163 163 163 F F F 10 200-219 92 92 92 171 171 171 F F F 11 220-239 93 93 93 172 172 172 F F F 12 240-259 92 92 92 171 171 171 F F F 13 260-279 85 85 85 161 161 161 F F F 14 280-299 93 93 93 172 172 172 F F F 15 300-319 90 90 90 169 169 169 F F F 16 320-339 90 90 90 166 166 166 F F F 17 340-359 92 92 92 171 171 171 F F F 18 360-379 89 89 89 165 165 165 F F F 19 380-399 90 90 90 168 168 168 F F F 20 400-419 93 93 93 172 172 172 F F F 21 420-439 93 93 93 172 172 172 F F F 22 440-459 93 93 93 172 172 172 F F F 23 460-479 93 93 93 172 172 172 F F F 24 480-499 93 93 93 172 172 172 F F F 25 500-519 92 92 92 171 171 171 F F F 26 520-539 93 93 93 172 172 172 F F F 27 540-559 90 90 90 166 166 166 F F F 28 560-579 93 93 93 172 172 172 F F F 29 580-599 92 92 92 171 171 171 F F F Max gen: 93, Max flush: 172 Begin loop 48 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:09.159745057Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1564,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:09.20448517Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1564,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:09.205112919Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1564} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"0fd98d0d5d909611f8b408ccda2559d35fe117bb\",\n git_commit_timestamp: \"2024-11-08T20:05:44.000000000Z\",\n git_branch: \"mkeeter/drop-skipped-backpressure-guard\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:09.205163837Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1564} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:09.205187426Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1564} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:09.205214484Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1564} {"msg":"Crucible 10be2982-4c97-472c-8392-248709660122 has session id: 6517e707-bf0d-41ab-8df6-bc710d3abfed","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:09.205642083Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1564,"session_id":"6517e707-bf0d-41ab-8df6-bc710d3abfed"} {"msg":"Upstairs opts: Upstairs UUID: 10be2982-4c97-472c-8392-248709660122, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:09.205691531Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1564,"session_id":"6517e707-bf0d-41ab-8df6-bc710d3abfed"} {"msg":"Crucible stats registered with UUID: 10be2982-4c97-472c-8392-248709660122","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:09.20571717Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1564,"session_id":"6517e707-bf0d-41ab-8df6-bc710d3abfed"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:09.236875617Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1564,"task":"crutest"} {"msg":"The guest has requested activation with gen:96","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:09.236907455Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1564} {"msg":"10be2982-4c97-472c-8392-248709660122 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:09.236982002Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1564,"session_id":"6517e707-bf0d-41ab-8df6-bc710d3abfed"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:09.23701363Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1564,"client":"0","":"downstairs","session_id":"6517e707-bf0d-41ab-8df6-bc710d3abfed"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:09.237047459Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1564,"client":"0","":"downstairs","session_id":"6517e707-bf0d-41ab-8df6-bc710d3abfed"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:09.237067238Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1564,"client":"1","":"downstairs","session_id":"6517e707-bf0d-41ab-8df6-bc710d3abfed"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:09.237088217Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1564,"client":"1","":"downstairs","session_id":"6517e707-bf0d-41ab-8df6-bc710d3abfed"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:09.237112405Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1564,"client":"2","":"downstairs","session_id":"6517e707-bf0d-41ab-8df6-bc710d3abfed"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:09.237138224Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1564,"client":"2","":"downstairs","session_id":"6517e707-bf0d-41ab-8df6-bc710d3abfed"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:09.237170663Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1564,"":"io task","client":"2","":"downstairs","session_id":"6517e707-bf0d-41ab-8df6-bc710d3abfed"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:09.237204561Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1564,"":"io task","client":"0","":"downstairs","session_id":"6517e707-bf0d-41ab-8df6-bc710d3abfed"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:09.237240769Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1564,"":"io task","client":"1","":"downstairs","session_id":"6517e707-bf0d-41ab-8df6-bc710d3abfed"} {"msg":"ds_connection connected from Ok(127.0.0.1:63644)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:09.237274448Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1564,"":"io task","client":"0","":"downstairs","session_id":"6517e707-bf0d-41ab-8df6-bc710d3abfed"} {"msg":"ds_connection connected from Ok(127.0.0.1:38677)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:09.237309456Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1564,"":"io task","client":"2","":"downstairs","session_id":"6517e707-bf0d-41ab-8df6-bc710d3abfed"} {"msg":"ds_connection connected from Ok(127.0.0.1:58950)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:09.237442239Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1564,"":"io task","client":"1","":"downstairs","session_id":"6517e707-bf0d-41ab-8df6-bc710d3abfed"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:09.238012552Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1564,"client":"0","":"downstairs","session_id":"6517e707-bf0d-41ab-8df6-bc710d3abfed"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:09.238057109Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1564,"client":"0","":"downstairs","session_id":"6517e707-bf0d-41ab-8df6-bc710d3abfed"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:09.238091128Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1564,"client":"1","":"downstairs","session_id":"6517e707-bf0d-41ab-8df6-bc710d3abfed"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:09.238124316Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1564,"client":"1","":"downstairs","session_id":"6517e707-bf0d-41ab-8df6-bc710d3abfed"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:09.238157354Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1564,"client":"2","":"downstairs","session_id":"6517e707-bf0d-41ab-8df6-bc710d3abfed"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:09.238175233Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1564,"client":"2","":"downstairs","session_id":"6517e707-bf0d-41ab-8df6-bc710d3abfed"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:09.238686788Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1564,"client":"1","":"downstairs","session_id":"6517e707-bf0d-41ab-8df6-bc710d3abfed"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:09.238784984Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1564,"client":"2","":"downstairs","session_id":"6517e707-bf0d-41ab-8df6-bc710d3abfed"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:09.238806903Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1564,"client":"0","":"downstairs","session_id":"6517e707-bf0d-41ab-8df6-bc710d3abfed"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:09.239244441Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1564,"client":"1","":"downstairs","session_id":"6517e707-bf0d-41ab-8df6-bc710d3abfed"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:09.239283599Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1564,"client":"1","":"downstairs","session_id":"6517e707-bf0d-41ab-8df6-bc710d3abfed"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:09.239300558Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1564,"session_id":"6517e707-bf0d-41ab-8df6-bc710d3abfed"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:09.239369405Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1564,"client":"0","":"downstairs","session_id":"6517e707-bf0d-41ab-8df6-bc710d3abfed"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:09.239389194Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1564,"client":"0","":"downstairs","session_id":"6517e707-bf0d-41ab-8df6-bc710d3abfed"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:09.239406403Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1564,"session_id":"6517e707-bf0d-41ab-8df6-bc710d3abfed"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:09.239418693Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1564,"client":"2","":"downstairs","session_id":"6517e707-bf0d-41ab-8df6-bc710d3abfed"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:09.239432342Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1564,"client":"2","":"downstairs","session_id":"6517e707-bf0d-41ab-8df6-bc710d3abfed"} {"msg":"[0]R flush_numbers[0..12]: [171, 165, 165, 172, 172, 171, 163, 165, 172, 163, 171, 172]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:09.239447891Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1564,"":"downstairs","":"downstairs","session_id":"6517e707-bf0d-41ab-8df6-bc710d3abfed"} {"msg":"[0]R generation[0..12]: [92, 89, 89, 93, 93, 92, 87, 89, 93, 87, 92, 93]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:09.239461591Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1564,"":"downstairs","":"downstairs","session_id":"6517e707-bf0d-41ab-8df6-bc710d3abfed"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:09.23947565Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1564,"":"downstairs","":"downstairs","session_id":"6517e707-bf0d-41ab-8df6-bc710d3abfed"} {"msg":"[1]R flush_numbers[0..12]: [171, 165, 165, 172, 172, 171, 163, 165, 172, 163, 171, 172]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:09.239490919Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1564,"":"downstairs","":"downstairs","session_id":"6517e707-bf0d-41ab-8df6-bc710d3abfed"} {"msg":"[1]R generation[0..12]: [92, 89, 89, 93, 93, 92, 87, 89, 93, 87, 92, 93]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:09.239504399Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1564,"":"downstairs","":"downstairs","session_id":"6517e707-bf0d-41ab-8df6-bc710d3abfed"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:09.239518398Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1564,"":"downstairs","":"downstairs","session_id":"6517e707-bf0d-41ab-8df6-bc710d3abfed"} {"msg":"[2]R flush_numbers[0..12]: [171, 165, 165, 172, 172, 171, 163, 165, 172, 163, 171, 172]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:09.239532357Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1564,"":"downstairs","":"downstairs","session_id":"6517e707-bf0d-41ab-8df6-bc710d3abfed"} {"msg":"[2]R generation[0..12]: [92, 89, 89, 93, 93, 92, 87, 89, 93, 87, 92, 93]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:09.239546026Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1564,"":"downstairs","":"downstairs","session_id":"6517e707-bf0d-41ab-8df6-bc710d3abfed"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:09.239559926Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1564,"":"downstairs","":"downstairs","session_id":"6517e707-bf0d-41ab-8df6-bc710d3abfed"} {"msg":"Max found gen is 94","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:09.239572895Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1564,"":"downstairs","":"downstairs","session_id":"6517e707-bf0d-41ab-8df6-bc710d3abfed"} {"msg":"Generation requested: 96 >= found:94","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:09.239585944Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1564,"":"downstairs","":"downstairs","session_id":"6517e707-bf0d-41ab-8df6-bc710d3abfed"} {"msg":"Next flush: 173","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:09.239600384Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1564,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO 6517e707-bf0d-41ab-8df6-bc710d3abfed"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:09.239621543Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1564,"":"downstairs","":"downstairs","session_id":"6517e707-bf0d-41ab-8df6-bc710d3abfed"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:09.239634892Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1564,"session_id":"6517e707-bf0d-41ab-8df6-bc710d3abfed"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:09.239646832Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1564,"client":"0","":"downstairs","session_id":"6517e707-bf0d-41ab-8df6-bc710d3abfed"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:09.239660121Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1564,"client":"1","":"downstairs","session_id":"6517e707-bf0d-41ab-8df6-bc710d3abfed"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:09.23967338Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1564,"client":"2","":"downstairs","session_id":"6517e707-bf0d-41ab-8df6-bc710d3abfed"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:09.23968658Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1564,"session_id":"6517e707-bf0d-41ab-8df6-bc710d3abfed"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:09.239698809Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1564,"session_id":"6517e707-bf0d-41ab-8df6-bc710d3abfed"} {"msg":"10be2982-4c97-472c-8392-248709660122 is now active with session: 6517e707-bf0d-41ab-8df6-bc710d3abfed","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:09.239711629Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1564,"session_id":"6517e707-bf0d-41ab-8df6-bc710d3abfed"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:09.239723628Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1564,"session_id":"6517e707-bf0d-41ab-8df6-bc710d3abfed"} {"msg":"The guest has finished waiting for activation with:96","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:09.239735797Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1564} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 362 len 4096 data: 13 02/30 Write block 45 len 4096 data: 8 03/30 Write block 1 len 12288 data: 3 5 4 04/30 Read block 245 len 20480 05/30 Read block 19 len 20480 06/30 Read block 143 len 28672 07/30 Read block 355 len 36864 08/30 Read block 283 len 32768 09/30 Write block 135 len 28672 data: 10 10 11 10 8 6 7 10/30 Read block 104 len 16384 11/30 Read block 234 len 40960 12/30 Read block 175 len 12288 13/30 Read block 581 len 36864 14/30 Write block 215 len 32768 data: 7 8 7 8 7 6 9 10 15/30 Write block 531 len 28672 data: 8 6 7 7 8 8 9 16/30 Write block 590 len 28672 data: 7 6 7 7 5 6 5 17/30 Write block 546 len 12288 data: 10 7 9 18/30 Flush 19/30 Write block 48 len 28672 data: 4 4 5 5 4 5 8 20/30 Read block 478 len 4096 21/30 Read block 129 len 36864 22/30 Read block 338 len 16384 23/30 Write block 42 len 36864 data: 9 10 10 9 6 3 5 5 6 24/30 Write block 338 len 32768 data: 11 11 10 11 12 10 8 9 25/30 Read block 365 len 4096 26/30 Read block 461 len 28672 27/30 Write block 488 len 20480 data: 8 10 10 8 6 28/30 Write block 409 len 20480 data: 8 10 11 11 10 29/30 Write block 283 len 36864 data: 8 8 9 13 14 15 12 12 10 30/30 Read block 405 len 28672 ---------------------------------------------------------------- Crucible gen:96 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1018 Acked Write 7 Done Done Done false 1019 Acked Read 1 Done Done Done false 1020 Acked Read 9 Done Done Done false 1021 Acked Read 4 Done Done Done false 1022 Acked Write 9 Done Done Done false 1023 Acked Write 8 Done Done Done false 1024 Acked Read 1 Done Done Done false 1025 Acked Read 7 Done Done Done false 1026 Acked Write 5 Done Done Done false 1027 Acked Write 5 Done Done Done false 1028 Acked Write 9 Done Done Sent false 1029 Acked Read 7 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 2 3 Done 11 12 10 33 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1017 1017 1017 Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:26:09.263200841Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1564} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 96 96 96 173 173 173 F F F 1 020-039 89 89 89 165 165 165 F F F 2 040-059 96 96 96 173 173 173 T T T 3 060-079 93 93 93 172 172 172 F F F 4 080-099 93 93 93 172 172 172 F F F 5 100-119 92 92 92 171 171 171 F F F 6 120-139 96 96 96 173 173 173 F F F 7 140-159 96 96 96 173 173 173 F F F 8 160-179 93 93 93 172 172 172 F F F 9 180-199 87 87 87 163 163 163 F F F 10 200-219 96 96 96 173 173 173 F F F 11 220-239 96 96 96 173 173 173 F F F 12 240-259 92 92 92 171 171 171 F F F 13 260-279 85 85 85 161 161 161 F F F 14 280-299 93 93 93 172 172 172 T T T 15 300-319 90 90 90 169 169 169 F F F 16 320-339 90 90 90 166 166 166 T T T 17 340-359 92 92 92 171 171 171 T T T 18 360-379 96 96 96 173 173 173 F F F 19 380-399 90 90 90 168 168 168 F F F 20 400-419 93 93 93 172 172 172 T T T 21 420-439 93 93 93 172 172 172 F F F 22 440-459 93 93 93 172 172 172 F F F 23 460-479 93 93 93 172 172 172 F F F 24 480-499 93 93 93 172 172 172 T T T 25 500-519 92 92 92 171 171 171 F F F 26 520-539 96 96 96 173 173 173 F F F 27 540-559 96 96 96 173 173 173 F F F 28 560-579 93 93 93 172 172 172 F F F 29 580-599 96 96 96 173 173 173 F F F Max gen: 96, Max flush: 173 A Difference in extent metadata is expected here On loop 48 Verifying data now No change after verify Loop: 48 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 96 96 96 173 173 173 F F F 1 020-039 89 89 89 165 165 165 F F F 2 040-059 97 97 97 174 174 174 F F F 3 060-079 93 93 93 172 172 172 F F F 4 080-099 93 93 93 172 172 172 F F F 5 100-119 92 92 92 171 171 171 F F F 6 120-139 96 96 96 173 173 173 F F F 7 140-159 96 96 96 173 173 173 F F F 8 160-179 93 93 93 172 172 172 F F F 9 180-199 87 87 87 163 163 163 F F F 10 200-219 96 96 96 173 173 173 F F F 11 220-239 96 96 96 173 173 173 F F F 12 240-259 92 92 92 171 171 171 F F F 13 260-279 85 85 85 161 161 161 F F F 14 280-299 97 97 97 174 174 174 F F F 15 300-319 90 90 90 169 169 169 F F F 16 320-339 97 97 97 174 174 174 F F F 17 340-359 97 97 97 174 174 174 F F F 18 360-379 96 96 96 173 173 173 F F F 19 380-399 90 90 90 168 168 168 F F F 20 400-419 97 97 97 174 174 174 F F F 21 420-439 93 93 93 172 172 172 F F F 22 440-459 93 93 93 172 172 172 F F F 23 460-479 93 93 93 172 172 172 F F F 24 480-499 97 97 97 174 174 174 F F F 25 500-519 92 92 92 171 171 171 F F F 26 520-539 96 96 96 173 173 173 F F F 27 540-559 96 96 96 173 173 173 F F F 28 560-579 93 93 93 172 172 172 F F F 29 580-599 96 96 96 173 173 173 F F F Max gen: 97, Max flush: 174 Begin loop 49 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:23.60233603Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1572,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:26:23.645045452Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1572,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:23.645442513Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1572,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:23.677731774Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1572,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:23.678304417Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1572} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"0fd98d0d5d909611f8b408ccda2559d35fe117bb\",\n git_commit_timestamp: \"2024-11-08T20:05:44.000000000Z\",\n git_branch: \"mkeeter/drop-skipped-backpressure-guard\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:23.678341615Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1572} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:23.678356124Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1572} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:23.678370463Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1572} {"msg":"Crucible 68eeb35a-c55d-4460-abcf-42864725df9e has session id: 327fa272-f2c8-4a3c-bf34-052be96c7365","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:23.678781093Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1572,"session_id":"327fa272-f2c8-4a3c-bf34-052be96c7365"} {"msg":"Upstairs opts: Upstairs UUID: 68eeb35a-c55d-4460-abcf-42864725df9e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:23.678826471Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1572,"session_id":"327fa272-f2c8-4a3c-bf34-052be96c7365"} {"msg":"Crucible stats registered with UUID: 68eeb35a-c55d-4460-abcf-42864725df9e","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:23.67885743Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1572,"session_id":"327fa272-f2c8-4a3c-bf34-052be96c7365"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:23.709922911Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1572,"task":"crutest"} {"msg":"The guest has requested activation with gen:98","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:23.70995387Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1572} {"msg":"68eeb35a-c55d-4460-abcf-42864725df9e active request set","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:23.710023576Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1572,"session_id":"327fa272-f2c8-4a3c-bf34-052be96c7365"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:23.710046755Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1572,"client":"0","":"downstairs","session_id":"327fa272-f2c8-4a3c-bf34-052be96c7365"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:23.710074084Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1572,"client":"0","":"downstairs","session_id":"327fa272-f2c8-4a3c-bf34-052be96c7365"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:23.710091523Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1572,"client":"1","":"downstairs","session_id":"327fa272-f2c8-4a3c-bf34-052be96c7365"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:23.710112552Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1572,"client":"1","":"downstairs","session_id":"327fa272-f2c8-4a3c-bf34-052be96c7365"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:23.710136861Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1572,"client":"2","":"downstairs","session_id":"327fa272-f2c8-4a3c-bf34-052be96c7365"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:23.71015473Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1572,"client":"2","":"downstairs","session_id":"327fa272-f2c8-4a3c-bf34-052be96c7365"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:23.710171029Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1572,"":"io task","client":"0","":"downstairs","session_id":"327fa272-f2c8-4a3c-bf34-052be96c7365"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:23.710190048Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1572,"":"io task","client":"2","":"downstairs","session_id":"327fa272-f2c8-4a3c-bf34-052be96c7365"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:23.710214297Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1572,"":"io task","client":"1","":"downstairs","session_id":"327fa272-f2c8-4a3c-bf34-052be96c7365"} {"msg":"ds_connection connected from Ok(127.0.0.1:33295)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:23.710610548Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1572,"":"io task","client":"2","":"downstairs","session_id":"327fa272-f2c8-4a3c-bf34-052be96c7365"} {"msg":"ds_connection connected from Ok(127.0.0.1:52144)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:23.710645506Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1572,"":"io task","client":"0","":"downstairs","session_id":"327fa272-f2c8-4a3c-bf34-052be96c7365"} {"msg":"ds_connection connected from Ok(127.0.0.1:36609)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:23.710665295Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1572,"":"io task","client":"1","":"downstairs","session_id":"327fa272-f2c8-4a3c-bf34-052be96c7365"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:23.711239417Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1572,"client":"2","":"downstairs","session_id":"327fa272-f2c8-4a3c-bf34-052be96c7365"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:23.711279415Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1572,"client":"2","":"downstairs","session_id":"327fa272-f2c8-4a3c-bf34-052be96c7365"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:23.711310924Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1572,"client":"1","":"downstairs","session_id":"327fa272-f2c8-4a3c-bf34-052be96c7365"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:23.711341852Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1572,"client":"1","":"downstairs","session_id":"327fa272-f2c8-4a3c-bf34-052be96c7365"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:23.711370241Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1572,"client":"0","":"downstairs","session_id":"327fa272-f2c8-4a3c-bf34-052be96c7365"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:23.711399199Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1572,"client":"0","":"downstairs","session_id":"327fa272-f2c8-4a3c-bf34-052be96c7365"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:23.711871976Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1572,"client":"2","":"downstairs","session_id":"327fa272-f2c8-4a3c-bf34-052be96c7365"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:23.711956122Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1572,"client":"1","":"downstairs","session_id":"327fa272-f2c8-4a3c-bf34-052be96c7365"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:23.71198917Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1572,"client":"0","":"downstairs","session_id":"327fa272-f2c8-4a3c-bf34-052be96c7365"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:23.712490776Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1572,"client":"1","":"downstairs","session_id":"327fa272-f2c8-4a3c-bf34-052be96c7365"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:23.712522354Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1572,"client":"1","":"downstairs","session_id":"327fa272-f2c8-4a3c-bf34-052be96c7365"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:23.712538504Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1572,"session_id":"327fa272-f2c8-4a3c-bf34-052be96c7365"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:23.712551743Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1572,"client":"0","":"downstairs","session_id":"327fa272-f2c8-4a3c-bf34-052be96c7365"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:23.712565442Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1572,"client":"0","":"downstairs","session_id":"327fa272-f2c8-4a3c-bf34-052be96c7365"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:23.712582131Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1572,"session_id":"327fa272-f2c8-4a3c-bf34-052be96c7365"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:23.712594591Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1572,"client":"2","":"downstairs","session_id":"327fa272-f2c8-4a3c-bf34-052be96c7365"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:23.7126081Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1572,"client":"2","":"downstairs","session_id":"327fa272-f2c8-4a3c-bf34-052be96c7365"} {"msg":"[0]R flush_numbers[0..12]: [173, 165, 174, 172, 172, 171, 173, 173, 172, 163, 173, 173]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:23.712622169Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1572,"":"downstairs","":"downstairs","session_id":"327fa272-f2c8-4a3c-bf34-052be96c7365"} {"msg":"[0]R generation[0..12]: [96, 89, 97, 93, 93, 92, 96, 96, 93, 87, 96, 96]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:23.712636049Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1572,"":"downstairs","":"downstairs","session_id":"327fa272-f2c8-4a3c-bf34-052be96c7365"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:23.712650158Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1572,"":"downstairs","":"downstairs","session_id":"327fa272-f2c8-4a3c-bf34-052be96c7365"} {"msg":"[1]R flush_numbers[0..12]: [173, 165, 174, 172, 172, 171, 173, 173, 172, 163, 173, 173]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:23.712663867Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1572,"":"downstairs","":"downstairs","session_id":"327fa272-f2c8-4a3c-bf34-052be96c7365"} {"msg":"[1]R generation[0..12]: [96, 89, 97, 93, 93, 92, 96, 96, 93, 87, 96, 96]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:23.712677387Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1572,"":"downstairs","":"downstairs","session_id":"327fa272-f2c8-4a3c-bf34-052be96c7365"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:23.712691206Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1572,"":"downstairs","":"downstairs","session_id":"327fa272-f2c8-4a3c-bf34-052be96c7365"} {"msg":"[2]R flush_numbers[0..12]: [173, 165, 174, 172, 172, 171, 173, 173, 172, 163, 173, 173]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:23.712705025Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1572,"":"downstairs","":"downstairs","session_id":"327fa272-f2c8-4a3c-bf34-052be96c7365"} {"msg":"[2]R generation[0..12]: [96, 89, 97, 93, 93, 92, 96, 96, 93, 87, 96, 96]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:23.712718555Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1572,"":"downstairs","":"downstairs","session_id":"327fa272-f2c8-4a3c-bf34-052be96c7365"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:23.712732604Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1572,"":"downstairs","":"downstairs","session_id":"327fa272-f2c8-4a3c-bf34-052be96c7365"} {"msg":"Max found gen is 98","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:23.712745573Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1572,"":"downstairs","":"downstairs","session_id":"327fa272-f2c8-4a3c-bf34-052be96c7365"} {"msg":"Generation requested: 98 >= found:98","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:23.712758663Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1572,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO 327fa272-f2c8-4a3c-bf34-052be96c7365"} {"msg":"Next flush: 175","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:23.712785891Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1572,"":"downstairs","":"downstairs","session_id":"327fa272-f2c8-4a3c-bf34-052be96c7365"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:23.712799081Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1572,"":"downstairs","":"downstairs","session_id":"327fa272-f2c8-4a3c-bf34-052be96c7365"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:23.71281218Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1572,"session_id":"327fa272-f2c8-4a3c-bf34-052be96c7365"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:23.712824279Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1572,"client":"0","":"downstairs","session_id":"327fa272-f2c8-4a3c-bf34-052be96c7365"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:23.712837559Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1572,"client":"1","":"downstairs","session_id":"327fa272-f2c8-4a3c-bf34-052be96c7365"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:23.712850858Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1572,"client":"2","":"downstairs","session_id":"327fa272-f2c8-4a3c-bf34-052be96c7365"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:23.712864217Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1572,"session_id":"327fa272-f2c8-4a3c-bf34-052be96c7365"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:23.712876577Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1572,"session_id":"327fa272-f2c8-4a3c-bf34-052be96c7365"} {"msg":"68eeb35a-c55d-4460-abcf-42864725df9e is now active with session: 327fa272-f2c8-4a3c-bf34-052be96c7365","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:23.712889396Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1572,"session_id":"327fa272-f2c8-4a3c-bf34-052be96c7365"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:23.712901326Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1572,"session_id":"327fa272-f2c8-4a3c-bf34-052be96c7365"} {"msg":"The guest has finished waiting for activation with:98","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:23.712913515Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1572} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 213 len 8192 data: 9 8 02/30 Read block 458 len 8192 03/30 Read block 88 len 8192 04/30 Read block 521 len 40960 05/30 Write block 459 len 40960 data: 6 5 4 3 4 3 5 9 10 12 06/30 Write block 265 len 24576 data: 11 10 7 7 8 8 07/30 Read block 116 len 8192 08/30 Read block 492 len 40960 09/30 Write block 580 len 24576 data: 8 8 8 11 12 15 10/30 Read block 277 len 24576 11/30 Read block 10 len 12288 12/30 Read block 191 len 8192 13/30 Read block 170 len 24576 14/30 Flush 15/30 Write block 373 len 8192 data: 7 7 16/30 Write block 51 len 32768 data: 6 5 6 9 8 7 7 6 17/30 Flush 18/30 Write block 250 len 4096 data: 14 19/30 Read block 469 len 8192 20/30 Flush 21/30 Write block 508 len 16384 data: 7 7 8 9 22/30 Read block 280 len 36864 23/30 Write block 139 len 32768 data: 9 7 8 6 7 8 8 7 24/30 Write block 422 len 40960 data: 8 8 7 8 7 5 5 10 11 10 25/30 Read block 564 len 36864 26/30 Read block 436 len 4096 27/30 Write block 84 len 24576 data: 6 8 8 8 8 6 28/30 Read block 140 len 4096 29/30 Write block 128 len 32768 data: 7 6 8 7 8 10 10 11 30/30 Read block 296 len 4096 ---------------------------------------------------------------- Crucible gen:98 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1020 Acked Write 4 Done Done Done false 1021 Acked Read 9 Done Done Done false 1022 Acked Write 8 Done Done Done false 1023 Acked Write 10 Done Done Done false 1024 Acked Read 9 Done Done Done false 1025 Acked Read 1 Done Done Done false 1026 Acked Write 6 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Write 8 Done Done Done false 1029 Acked Read 1 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 10 10 10 30 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1019 1019 1019 Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:26:23.738743173Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1572} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 96 96 96 173 173 173 F F F 1 020-039 89 89 89 165 165 165 F F F 2 040-059 98 98 98 176 176 176 F F F 3 060-079 93 93 93 172 172 172 F F F 4 080-099 93 93 93 172 172 172 T T T 5 100-119 92 92 92 171 171 171 F F F 6 120-139 96 96 96 173 173 173 T T T 7 140-159 96 96 96 173 173 173 T T T 8 160-179 93 93 93 172 172 172 F F F 9 180-199 87 87 87 163 163 163 F F F 10 200-219 98 98 98 175 175 175 F F F 11 220-239 96 96 96 173 173 173 F F F 12 240-259 98 98 98 177 177 177 F F F 13 260-279 98 98 98 175 175 175 F F F 14 280-299 97 97 97 174 174 174 F F F 15 300-319 90 90 90 169 169 169 F F F 16 320-339 97 97 97 174 174 174 F F F 17 340-359 97 97 97 174 174 174 F F F 18 360-379 98 98 98 176 176 176 F F F 19 380-399 90 90 90 168 168 168 F F F 20 400-419 97 97 97 174 174 174 F F F 21 420-439 93 93 93 172 172 172 T T T 22 440-459 98 98 98 175 175 175 F F F 23 460-479 98 98 98 175 175 175 F F F 24 480-499 97 97 97 174 174 174 F F F 25 500-519 92 92 92 171 171 171 T T T 26 520-539 96 96 96 173 173 173 F F F 27 540-559 96 96 96 173 173 173 F F F 28 560-579 93 93 93 172 172 172 F F F 29 580-599 98 98 98 175 175 175 F F F Max gen: 98, Max flush: 177 A Difference in extent metadata is expected here On loop 49 Verifying data now No change after verify Loop: 49 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 96 96 96 173 173 173 F F F 1 020-039 89 89 89 165 165 165 F F F 2 040-059 98 98 98 176 176 176 F F F 3 060-079 93 93 93 172 172 172 F F F 4 080-099 99 99 99 178 178 178 F F F 5 100-119 92 92 92 171 171 171 F F F 6 120-139 99 99 99 178 178 178 F F F 7 140-159 99 99 99 178 178 178 F F F 8 160-179 93 93 93 172 172 172 F F F 9 180-199 87 87 87 163 163 163 F F F 10 200-219 98 98 98 175 175 175 F F F 11 220-239 96 96 96 173 173 173 F F F 12 240-259 98 98 98 177 177 177 F F F 13 260-279 98 98 98 175 175 175 F F F 14 280-299 97 97 97 174 174 174 F F F 15 300-319 90 90 90 169 169 169 F F F 16 320-339 97 97 97 174 174 174 F F F 17 340-359 97 97 97 174 174 174 F F F 18 360-379 98 98 98 176 176 176 F F F 19 380-399 90 90 90 168 168 168 F F F 20 400-419 97 97 97 174 174 174 F F F 21 420-439 99 99 99 178 178 178 F F F 22 440-459 98 98 98 175 175 175 F F F 23 460-479 98 98 98 175 175 175 F F F 24 480-499 97 97 97 174 174 174 F F F 25 500-519 99 99 99 178 178 178 F F F 26 520-539 96 96 96 173 173 173 F F F 27 540-559 96 96 96 173 173 173 F F F 28 560-579 93 93 93 172 172 172 F F F 29 580-599 98 98 98 175 175 175 F F F Max gen: 99, Max flush: 178 Begin loop 50 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:28.083871359Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1580,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:26:28.126579232Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1580,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:28.126980732Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1580,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:28.158721481Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1580,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:28.159321172Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1580} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"0fd98d0d5d909611f8b408ccda2559d35fe117bb\",\n git_commit_timestamp: \"2024-11-08T20:05:44.000000000Z\",\n git_branch: \"mkeeter/drop-skipped-backpressure-guard\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:28.159367019Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1580} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:28.159384428Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1580} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:28.159402017Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1580} {"msg":"Crucible 5ce8ac78-9354-482d-944e-a1ef5eb16512 has session id: 4802fab8-2523-433f-b83f-6e96ca356640","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:28.159890444Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1580,"session_id":"4802fab8-2523-433f-b83f-6e96ca356640"} {"msg":"Upstairs opts: Upstairs UUID: 5ce8ac78-9354-482d-944e-a1ef5eb16512, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:28.159938471Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1580,"session_id":"4802fab8-2523-433f-b83f-6e96ca356640"} {"msg":"Crucible stats registered with UUID: 5ce8ac78-9354-482d-944e-a1ef5eb16512","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:28.15996054Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1580,"session_id":"4802fab8-2523-433f-b83f-6e96ca356640"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:28.190922647Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1580,"task":"crutest"} {"msg":"The guest has requested activation with gen:100","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:28.190954475Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1580} {"msg":"5ce8ac78-9354-482d-944e-a1ef5eb16512 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:28.191004963Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1580,"session_id":"4802fab8-2523-433f-b83f-6e96ca356640"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:28.191025432Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1580,"client":"0","":"downstairs","session_id":"4802fab8-2523-433f-b83f-6e96ca356640"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:28.19105315Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1580,"client":"0","":"downstairs","session_id":"4802fab8-2523-433f-b83f-6e96ca356640"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:28.19107095Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1580,"client":"1","":"downstairs","session_id":"4802fab8-2523-433f-b83f-6e96ca356640"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:28.191088219Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1580,"client":"1","":"downstairs","session_id":"4802fab8-2523-433f-b83f-6e96ca356640"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:28.191109358Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1580,"client":"2","":"downstairs","session_id":"4802fab8-2523-433f-b83f-6e96ca356640"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:28.191142566Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1580,"client":"2","":"downstairs","session_id":"4802fab8-2523-433f-b83f-6e96ca356640"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:28.191173295Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1580,"":"io task","client":"2","":"downstairs","session_id":"4802fab8-2523-433f-b83f-6e96ca356640"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:28.191199833Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1580,"":"io task","client":"0","":"downstairs","session_id":"4802fab8-2523-433f-b83f-6e96ca356640"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:28.191223092Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1580,"":"io task","client":"1","":"downstairs","session_id":"4802fab8-2523-433f-b83f-6e96ca356640"} {"msg":"ds_connection connected from Ok(127.0.0.1:49106)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:28.191297449Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1580,"":"io task","client":"2","":"downstairs","session_id":"4802fab8-2523-433f-b83f-6e96ca356640"} {"msg":"ds_connection connected from Ok(127.0.0.1:39564)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:28.191340836Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1580,"":"io task","client":"0","":"downstairs","session_id":"4802fab8-2523-433f-b83f-6e96ca356640"} {"msg":"ds_connection connected from Ok(127.0.0.1:39667)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:28.191371945Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1580,"":"io task","client":"1","":"downstairs","session_id":"4802fab8-2523-433f-b83f-6e96ca356640"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:28.192075971Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1580,"client":"2","":"downstairs","session_id":"4802fab8-2523-433f-b83f-6e96ca356640"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:28.192104759Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1580,"client":"2","":"downstairs","session_id":"4802fab8-2523-433f-b83f-6e96ca356640"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:28.192235453Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1580,"client":"1","":"downstairs","session_id":"4802fab8-2523-433f-b83f-6e96ca356640"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:28.192264391Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1580,"client":"1","":"downstairs","session_id":"4802fab8-2523-433f-b83f-6e96ca356640"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:28.19229248Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1580,"client":"0","":"downstairs","session_id":"4802fab8-2523-433f-b83f-6e96ca356640"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:28.192317689Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1580,"client":"0","":"downstairs","session_id":"4802fab8-2523-433f-b83f-6e96ca356640"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:28.192769577Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1580,"client":"2","":"downstairs","session_id":"4802fab8-2523-433f-b83f-6e96ca356640"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:28.192801015Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1580,"client":"1","":"downstairs","session_id":"4802fab8-2523-433f-b83f-6e96ca356640"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:28.192938658Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1580,"client":"0","":"downstairs","session_id":"4802fab8-2523-433f-b83f-6e96ca356640"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:28.193382027Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1580,"client":"1","":"downstairs","session_id":"4802fab8-2523-433f-b83f-6e96ca356640"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:28.193416525Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1580,"client":"1","":"downstairs","session_id":"4802fab8-2523-433f-b83f-6e96ca356640"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:28.193432064Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1580,"session_id":"4802fab8-2523-433f-b83f-6e96ca356640"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:28.193444854Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1580,"client":"2","":"downstairs","session_id":"4802fab8-2523-433f-b83f-6e96ca356640"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:28.193458493Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1580,"client":"2","":"downstairs","session_id":"4802fab8-2523-433f-b83f-6e96ca356640"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:28.193472022Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1580,"session_id":"4802fab8-2523-433f-b83f-6e96ca356640"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:28.193484342Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1580,"client":"0","":"downstairs","session_id":"4802fab8-2523-433f-b83f-6e96ca356640"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:28.193501441Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1580,"client":"0","":"downstairs","session_id":"4802fab8-2523-433f-b83f-6e96ca356640"} {"msg":"[0]R flush_numbers[0..12]: [173, 165, 176, 172, 178, 171, 178, 178, 172, 163, 175, 173]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:28.19351612Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1580,"":"downstairs","":"downstairs","session_id":"4802fab8-2523-433f-b83f-6e96ca356640"} {"msg":"[0]R generation[0..12]: [96, 89, 98, 93, 99, 92, 99, 99, 93, 87, 98, 96]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:28.1935301Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1580,"":"downstairs","":"downstairs","session_id":"4802fab8-2523-433f-b83f-6e96ca356640"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:28.193544219Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1580,"":"downstairs","":"downstairs","session_id":"4802fab8-2523-433f-b83f-6e96ca356640"} {"msg":"[1]R flush_numbers[0..12]: [173, 165, 176, 172, 178, 171, 178, 178, 172, 163, 175, 173]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:28.193558338Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1580,"":"downstairs","":"downstairs","session_id":"4802fab8-2523-433f-b83f-6e96ca356640"} {"msg":"[1]R generation[0..12]: [96, 89, 98, 93, 99, 92, 99, 99, 93, 87, 98, 96]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:28.193571947Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1580,"":"downstairs","":"downstairs","session_id":"4802fab8-2523-433f-b83f-6e96ca356640"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:28.193586017Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1580,"":"downstairs","":"downstairs","session_id":"4802fab8-2523-433f-b83f-6e96ca356640"} {"msg":"[2]R flush_numbers[0..12]: [173, 165, 176, 172, 178, 171, 178, 178, 172, 163, 175, 173]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:28.193600046Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1580,"":"downstairs","":"downstairs","session_id":"4802fab8-2523-433f-b83f-6e96ca356640"} {"msg":"[2]R generation[0..12]: [96, 89, 98, 93, 99, 92, 99, 99, 93, 87, 98, 96]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:28.193613705Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1580,"":"downstairs","":"downstairs","session_id":"4802fab8-2523-433f-b83f-6e96ca356640"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:28.193627765Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1580,"":"downstairs","":"downstairs","session_id":"4802fab8-2523-433f-b83f-6e96ca356640"} {"msg":"Max found gen is 100","v":0,"name":"crucible"Wait for a query_work_queue command to finish before sending IO ,"level":30,"time":"2024-11-08T20:26:28.193651483Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1580,"":"downstairs","":"downstairs","session_id":"4802fab8-2523-433f-b83f-6e96ca356640"} {"msg":"Generation requested: 100 >= found:100","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:28.193664913Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1580,"":"downstairs","":"downstairs","session_id":"4802fab8-2523-433f-b83f-6e96ca356640"} {"msg":"Next flush: 179","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:28.193677972Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1580,"":"downstairs","":"downstairs","session_id":"4802fab8-2523-433f-b83f-6e96ca356640"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:28.193692961Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1580,"":"downstairs","":"downstairs","session_id":"4802fab8-2523-433f-b83f-6e96ca356640"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:28.193706371Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1580,"session_id":"4802fab8-2523-433f-b83f-6e96ca356640"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:28.19371862Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1580,"client":"0","":"downstairs","session_id":"4802fab8-2523-433f-b83f-6e96ca356640"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:28.19373181Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1580,"client":"1","":"downstairs","session_id":"4802fab8-2523-433f-b83f-6e96ca356640"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:28.193744969Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1580,"client":"2","":"downstairs","session_id":"4802fab8-2523-433f-b83f-6e96ca356640"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:28.193758308Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1580,"session_id":"4802fab8-2523-433f-b83f-6e96ca356640"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:28.193770528Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1580,"session_id":"4802fab8-2523-433f-b83f-6e96ca356640"} {"msg":"5ce8ac78-9354-482d-944e-a1ef5eb16512 is now active with session: 4802fab8-2523-433f-b83f-6e96ca356640","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:28.193783257Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1580,"session_id":"4802fab8-2523-433f-b83f-6e96ca356640"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:28.193795196Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1580,"session_id":"4802fab8-2523-433f-b83f-6e96ca356640"} {"msg":"The guest has finished waiting for activation with:100","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:28.193807526Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1580} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 406 len 24576 data: 8 8 9 9 11 12 03/30 Read block 321 len 36864 04/30 Read block 127 len 32768 05/30 Read block 524 len 28672 06/30 Write block 442 len 4096 data: 14 07/30 Read block 577 len 16384 08/30 Write block 353 len 8192 data: 13 11 09/30 Read block 385 len 4096 10/30 Read block 13 len 12288 11/30 Read block 381 len 28672 12/30 Write block 264 len 20480 data: 9 12 11 8 8 13/30 Write block 89 len 36864 data: 7 5 5 6 6 5 6 5 3 14/30 Write block 508 len 4096 data: 8 15/30 Write block 60 len 8192 data: 6 6 16/30 Read block 254 len 24576 17/30 Read block 65 len 36864 18/30 Read block 528 len 24576 19/30 Write block 449 len 36864 data: 8 7 7 9 8 8 5 6 7 20/30 Read block 95 len 36864 21/30 Read block 377 len 40960 22/30 Flush 23/30 Write block 414 len 24576 data: 9 8 7 7 9 9 24/30 Write block 409 len 8192 data: 10 12 25/30 Flush 26/30 Write block 461 len 20480 data: 5 4 5 4 6 27/30 Read block 565 len 40960 28/30 Read block 215 len 24576 29/30 Read block 58 len 12288 30/30 Read block 568 len 4096 ---------------------------------------------------------------- Crucible gen:100 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 5 Done Done Done false 1026 Acked Read 10 Done Done Done false 1027 Acked Read 6 Done Done Done false 1028 Acked Read 3 Done Done Done false 1029 Acked Read 1 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 5 5 4 14 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:26:28.21868496Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1580} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 96 96 96 173 173 173 F F F 1 020-039 89 89 89 165 165 165 F F F 2 040-059 98 98 98 176 176 176 F F F 3 060-079 100 100 100 180 180 180 F F F 4 080-099 100 100 100 180 180 180 F F F 5 100-119 92 92 92 171 171 171 F F F 6 120-139 99 99 99 178 178 178 F F F 7 140-159 99 99 99 178 178 178 F F F 8 160-179 93 93 93 172 172 172 F F F 9 180-199 87 87 87 163 163 163 F F F 10 200-219 98 98 98 175 175 175 F F F 11 220-239 96 96 96 173 173 173 F F F 12 240-259 98 98 98 177 177 177 F F F 13 260-279 100 100 100 180 180 180 F F F 14 280-299 97 97 97 174 174 174 F F F 15 300-319 90 90 90 169 169 169 F F F 16 320-339 97 97 97 174 174 174 F F F 17 340-359 100 100 100 180 180 180 F F F 18 360-379 98 98 98 176 176 176 F F F 19 380-399 90 90 90 168 168 168 F F F 20 400-419 100 100 100 181 181 181 F F F 21 420-439 99 99 99 178 178 178 F F F 22 440-459 100 100 100 180 180 180 F F F 23 460-479 98 98 98 175 175 175 T T T 24 480-499 97 97 97 174 174 174 F F F 25 500-519 100 100 100 180 180 180 F F F 26 520-539 96 96 96 173 173 173 F F F 27 540-559 96 96 96 173 173 173 F F F 28 560-579 93 93 93 172 172 172 F F F 29 580-599 98 98 98 175 175 175 F F F Max gen: 100, Max flush: 181 A Difference in extent metadata is expected here On loop 50 Verifying data now No change after verify Loop: 50 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 96 96 96 173 173 173 F F F 1 020-039 89 89 89 165 165 165 F F F 2 040-059 98 98 98 176 176 176 F F F 3 060-079 100 100 100 180 180 180 F F F 4 080-099 100 100 100 180 180 180 F F F 5 100-119 92 92 92 171 171 171 F F F 6 120-139 99 99 99 178 178 178 F F F 7 140-159 99 99 99 178 178 178 F F F 8 160-179 93 93 93 172 172 172 F F F 9 180-199 87 87 87 163 163 163 F F F 10 200-219 98 98 98 175 175 175 F F F 11 220-239 96 96 96 173 173 173 F F F 12 240-259 98 98 98 177 177 177 F F F 13 260-279 100 100 100 180 180 180 F F F 14 280-299 97 97 97 174 174 174 F F F 15 300-319 90 90 90 169 169 169 F F F 16 320-339 97 97 97 174 174 174 F F F 17 340-359 100 100 100 180 180 180 F F F 18 360-379 98 98 98 176 176 176 F F F 19 380-399 90 90 90 168 168 168 F F F 20 400-419 100 100 100 181 181 181 F F F 21 420-439 99 99 99 178 178 178 F F F 22 440-459 100 100 100 180 180 180 F F F 23 460-479 101 101 101 182 182 182 F F F 24 480-499 97 97 97 174 174 174 F F F 25 500-519 100 100 100 180 180 180 F F F 26 520-539 96 96 96 173 173 173 F F F 27 540-559 96 96 96 173 173 173 F F F 28 560-579 93 93 93 172 172 172 F F F 29 580-599 98 98 98 175 175 175 F F F Max gen: 101, Max flush: 182 Begin loop 51 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:32.509606886Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1588,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:32.554194387Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1588,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:32.554795557Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1588} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"0fd98d0d5d909611f8b408ccda2559d35fe117bb\",\n git_commit_timestamp: \"2024-11-08T20:05:44.000000000Z\",\n git_branch: \"mkeeter/drop-skipped-backpressure-guard\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:32.554831215Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1588} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:32.554845745Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1588} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:32.554860804Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1588} {"msg":"Crucible f66a4db7-1404-4409-ab09-f41d60cebbe5 has session id: cc4a548d-3967-481e-91e3-485c5218cc8b","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:32.555312722Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1588,"session_id":"cc4a548d-3967-481e-91e3-485c5218cc8b"} {"msg":"Upstairs opts: Upstairs UUID: f66a4db7-1404-4409-ab09-f41d60cebbe5, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:32.55535801Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1588,"session_id":"cc4a548d-3967-481e-91e3-485c5218cc8b"} {"msg":"Crucible stats registered with UUID: f66a4db7-1404-4409-ab09-f41d60cebbe5","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:32.555382999Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1588,"session_id":"cc4a548d-3967-481e-91e3-485c5218cc8b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:32.586966785Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1588,"task":"crutest"} {"msg":"The guest has requested activation with gen:102","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:32.587017393Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1588} {"msg":"f66a4db7-1404-4409-ab09-f41d60cebbe5 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:32.587041571Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1588,"session_id":"cc4a548d-3967-481e-91e3-485c5218cc8b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:32.587055491Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1588,"client":"0","":"downstairs","session_id":"cc4a548d-3967-481e-91e3-485c5218cc8b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:32.587081929Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1588,"client":"0","":"downstairs","session_id":"cc4a548d-3967-481e-91e3-485c5218cc8b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:32.587096909Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1588,"client":"1","":"downstairs","session_id":"cc4a548d-3967-481e-91e3-485c5218cc8b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:32.587110448Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1588,"client":"1","":"downstairs","session_id":"cc4a548d-3967-481e-91e3-485c5218cc8b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:32.587123697Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1588,"client":"2","":"downstairs","session_id":"cc4a548d-3967-481e-91e3-485c5218cc8b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:32.587137727Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1588,"client":"2","":"downstairs","session_id":"cc4a548d-3967-481e-91e3-485c5218cc8b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:32.587150876Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1588,"":"io task","client":"2","":"downstairs","session_id":"cc4a548d-3967-481e-91e3-485c5218cc8b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:32.587164835Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1588,"":"io task","client":"0","":"downstairs","session_id":"cc4a548d-3967-481e-91e3-485c5218cc8b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:32.587178485Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1588,"":"io task","client":"1","":"downstairs","session_id":"cc4a548d-3967-481e-91e3-485c5218cc8b"} {"msg":"ds_connection connected from Ok(127.0.0.1:45027)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:32.587258971Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1588,"":"io task","client":"2","":"downstairs","session_id":"cc4a548d-3967-481e-91e3-485c5218cc8b"} {"msg":"ds_connection connected from Ok(127.0.0.1:53848)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:32.587313058Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1588,"":"io task","client":"0","":"downstairs","session_id":"cc4a548d-3967-481e-91e3-485c5218cc8b"} {"msg":"ds_connection connected from Ok(127.0.0.1:61407)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:32.587390144Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1588,"":"io task","client":"1","":"downstairs","session_id":"cc4a548d-3967-481e-91e3-485c5218cc8b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:32.588056662Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1588,"client":"0","":"downstairs","session_id":"cc4a548d-3967-481e-91e3-485c5218cc8b"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:32.58809492Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1588,"client":"0","":"downstairs","session_id":"cc4a548d-3967-481e-91e3-485c5218cc8b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:32.588113629Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1588,"client":"2","":"downstairs","session_id":"cc4a548d-3967-481e-91e3-485c5218cc8b"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:32.588128138Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1588,"client":"2","":"downstairs","session_id":"cc4a548d-3967-481e-91e3-485c5218cc8b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:32.588265821Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1588,"client":"1","":"downstairs","session_id":"cc4a548d-3967-481e-91e3-485c5218cc8b"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:32.58829272Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1588,"client":"1","":"downstairs","session_id":"cc4a548d-3967-481e-91e3-485c5218cc8b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:32.588754888Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1588,"client":"0","":"downstairs","session_id":"cc4a548d-3967-481e-91e3-485c5218cc8b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:32.588786206Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1588,"client":"2","":"downstairs","session_id":"cc4a548d-3967-481e-91e3-485c5218cc8b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:32.588817045Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1588,"client":"1","":"downstairs","session_id":"cc4a548d-3967-481e-91e3-485c5218cc8b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:32.589308401Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1588,"client":"2","":"downstairs","session_id":"cc4a548d-3967-481e-91e3-485c5218cc8b"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:32.589338379Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1588,"client":"2","":"downstairs","session_id":"cc4a548d-3967-481e-91e3-485c5218cc8b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:32.589353438Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1588,"session_id":"cc4a548d-3967-481e-91e3-485c5218cc8b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:32.589396856Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1588,"client":"1","":"downstairs","session_id":"cc4a548d-3967-481e-91e3-485c5218cc8b"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:32.589414245Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1588,"client":"1","":"downstairs","session_id":"cc4a548d-3967-481e-91e3-485c5218cc8b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:32.589427665Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1588,"session_id":"cc4a548d-3967-481e-91e3-485c5218cc8b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:32.589439994Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1588,"client":"0","":"downstairs","session_id":"cc4a548d-3967-481e-91e3-485c5218cc8b"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:32.589453263Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1588,"client":"0","":"downstairs","session_id":"cc4a548d-3967-481e-91e3-485c5218cc8b"} {"msg":"[0]R flush_numbers[0..12]: [173, 165, 176, 180, 180, 171, 178, 178, 172, 163, 175, 173]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:32.589466903Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1588,"":"downstairs","":"downstairs","session_id":"cc4a548d-3967-481e-91e3-485c5218cc8b"} {"msg":"[0]R generation[0..12]: [96, 89, 98, 100, 100, 92, 99, 99, 93, 87, 98, 96]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:32.589480732Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1588,"":"downstairs","":"downstairs","session_id":"cc4a548d-3967-481e-91e3-485c5218cc8b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:32.589494841Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1588,"":"downstairs","":"downstairs","session_id":"cc4a548d-3967-481e-91e3-485c5218cc8b"} {"msg":"[1]R flush_numbers[0..12]: [173, 165, 176, 180, 180, 171, 178, 178, 172, 163, 175, 173]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:32.58951168Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1588,"":"downstairs","":"downstairs","session_id":"cc4a548d-3967-481e-91e3-485c5218cc8b"} {"msg":"[1]R generation[0..12]: [96, 89, 98, 100, 100, 92, 99, 99, 93, 87, 98, 96]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:32.58952557Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1588,"":"downstairs","":"downstairs","session_id":"cc4a548d-3967-481e-91e3-485c5218cc8b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:32.589539479Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1588,"":"downstairs","":"downstairs","session_id":"cc4a548d-3967-481e-91e3-485c5218cc8b"} {"msg":"[2]R flush_numbers[0..12]: [173, 165, 176, 180, 180, 171, 178, 178, 172, 163, 175, 173]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:32.589553438Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1588,"":"downstairs","":"downstairs","session_id":"cc4a548d-3967-481e-91e3-485c5218cc8b"} {"msg":"[2]R generation[0..12]: [96, 89, 98, 100, 100, 92, 99, 99, 93, 87, 98, 96]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:32.589567068Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1588,"":"downstairs","":"downstairs","session_id":"cc4a548d-3967-481e-91e3-485c5218cc8b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:32.589580857Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1588,"":"downstairs","":"downstairs","session_id":"cc4a548d-3967-481e-91e3-485c5218cc8b"} {"msg":"Max found gen is 102","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:32.589593856Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1588,"":"downstairs","":"downstairs","session_id":"cc4a548d-3967-481e-91e3-485c5218cc8b"} {"msg":"Generation requested: 102 >= found:102","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:32.589607076Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1588,"":"downstairs","":"downstairs","session_id":"cc4a548d-3967-481e-91e3-485c5218cc8b"} {"msg":"Next flush: 183","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:32.589620015Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1588,"":"downstairs","":"downstairs","session_id":"cc4a548d-3967-481e-91e3-485c5218cc8b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:32.589633095Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":Wait for a query_work_queue command to finish before sending IO 1588,"":"downstairs","":"downstairs","session_id":"cc4a548d-3967-481e-91e3-485c5218cc8b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:32.589655524Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1588,"session_id":"cc4a548d-3967-481e-91e3-485c5218cc8b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:32.589667513Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1588,"client":"0","":"downstairs","session_id":"cc4a548d-3967-481e-91e3-485c5218cc8b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:32.589692742Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1588,"client":"1","":"downstairs","session_id":"cc4a548d-3967-481e-91e3-485c5218cc8b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:32.589706711Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1588,"client":"2","":"downstairs","session_id":"cc4a548d-3967-481e-91e3-485c5218cc8b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:32.58972242Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1588,"session_id":"cc4a548d-3967-481e-91e3-485c5218cc8b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:32.5897352Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1588,"session_id":"cc4a548d-3967-481e-91e3-485c5218cc8b"} {"msg":"f66a4db7-1404-4409-ab09-f41d60cebbe5 is now active with session: cc4a548d-3967-481e-91e3-485c5218cc8b","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:32.589748009Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1588,"session_id":"cc4a548d-3967-481e-91e3-485c5218cc8b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:32.589760059Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1588,"session_id":"cc4a548d-3967-481e-91e3-485c5218cc8b"} {"msg":"The guest has finished waiting for activation with:102","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:32.589772328Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1588} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 240 len 12288 data: 10 9 11 02/30 Flush 03/30 Write block 3 len 40960 data: 5 5 6 6 6 7 7 7 9 13 04/30 Read block 188 len 12288 05/30 Flush 06/30 Write block 126 len 12288 data: 8 7 8 07/30 Read block 405 len 32768 08/30 Write block 177 len 12288 data: 6 5 6 09/30 Read block 489 len 4096 10/30 Read block 138 len 32768 11/30 Read block 12 len 16384 12/30 Write block 88 len 4096 data: 9 13/30 Flush 14/30 Flush 15/30 Write block 467 len 40960 data: 11 13 11 11 12 12 11 11 13 11 16/30 Flush 17/30 Write block 12 len 24576 data: 14 12 10 12 12 11 18/30 Read block 159 len 28672 19/30 Read block 263 len 12288 20/30 Read block 342 len 36864 21/30 Flush 22/30 Flush 23/30 Write block 182 len 12288 data: 6 8 7 24/30 Write block 471 len 28672 data: 13 13 12 12 14 12 10 25/30 Read block 208 len 12288 26/30 Write block 445 len 12288 data: 9 11 10 27/30 Read block 392 len 4096 28/30 Write block 110 len 4096 data: 7 29/30 Read block 217 len 4096 30/30 Write block 476 len 28672 data: 13 11 11 11 9 10 9 ---------------------------------------------------------------- Crucible gen:102 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 3 Done Done Done false 1023 Acked Write 7 Done Done Done false 1024 Acked Read 3 Done Done Done false 1025 Acked Write 3 Done Done Done false 1026 Acked Read 1 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Read 1 Done Done Done false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 7 7 7 21 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1021 1021 1021 Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:26:32.619338953Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1588} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 102 102 102 188 188 188 F F F 1 020-039 89 89 89 165 165 165 F F F 2 040-059 98 98 98 176 176 176 F F F 3 060-079 100 100 100 180 180 180 F F F 4 080-099 102 102 102 185 185 185 F F F 5 100-119 92 92 92 171 171 171 T T T 6 120-139 102 102 102 185 185 185 F F F 7 140-159 99 99 99 178 178 178 F F F 8 160-179 102 102 102 185 185 185 F F F 9 180-199 87 87 87 163 163 163 T T T 10 200-219 98 98 98 175 175 175 F F F 11 220-239 96 96 96 173 173 173 F F F 12 240-259 102 102 102 183 183 183 F F F 13 260-279 100 100 100 180 180 180 F F F 14 280-299 97 97 97 174 174 174 F F F 15 300-319 90 90 90 169 169 169 F F F 16 320-339 97 97 97 174 174 174 F F F 17 340-359 100 100 100 180 180 180 F F F 18 360-379 98 98 98 176 176 176 F F F 19 380-399 90 90 90 168 168 168 F F F 20 400-419 100 100 100 181 181 181 F F F 21 420-439 99 99 99 178 178 178 F F F 22 440-459 100 100 100 180 180 180 T T T 23 460-479 102 102 102 187 187 187 T T T 24 480-499 97 97 97 174 174 174 T T T 25 500-519 100 100 100 180 180 180 F F F 26 520-539 96 96 96 173 173 173 F F F 27 540-559 96 96 96 173 173 173 F F F 28 560-579 93 93 93 172 172 172 F F F 29 580-599 98 98 98 175 175 175 F F F Max gen: 102, Max flush: 188 A Difference in extent metadata is expected here On loop 51 Verifying data now No change after verify Loop: 51 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 102 102 102 188 188 188 F F F 1 020-039 89 89 89 165 165 165 F F F 2 040-059 98 98 98 176 176 176 F F F 3 060-079 100 100 100 180 180 180 F F F 4 080-099 102 102 102 185 185 185 F F F 5 100-119 103 103 103 189 189 189 F F F 6 120-139 102 102 102 185 185 185 F F F 7 140-159 99 99 99 178 178 178 F F F 8 160-179 102 102 102 185 185 185 F F F 9 180-199 103 103 103 189 189 189 F F F 10 200-219 98 98 98 175 175 175 F F F 11 220-239 96 96 96 173 173 173 F F F 12 240-259 102 102 102 183 183 183 F F F 13 260-279 100 100 100 180 180 180 F F F 14 280-299 97 97 97 174 174 174 F F F 15 300-319 90 90 90 169 169 169 F F F 16 320-339 97 97 97 174 174 174 F F F 17 340-359 100 100 100 180 180 180 F F F 18 360-379 98 98 98 176 176 176 F F F 19 380-399 90 90 90 168 168 168 F F F 20 400-419 100 100 100 181 181 181 F F F 21 420-439 99 99 99 178 178 178 F F F 22 440-459 103 103 103 189 189 189 F F F 23 460-479 103 103 103 189 189 189 F F F 24 480-499 103 103 103 189 189 189 F F F 25 500-519 100 100 100 180 180 180 F F F 26 520-539 96 96 96 173 173 173 F F F 27 540-559 96 96 96 173 173 173 F F F 28 560-579 93 93 93 172 172 172 F F F 29 580-599 98 98 98 175 175 175 F F F Max gen: 103, Max flush: 189 Begin loop 52 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:36.940109588Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1596,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:36.984168305Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1596,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:36.984918898Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1596} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"0fd98d0d5d909611f8b408ccda2559d35fe117bb\",\n git_commit_timestamp: \"2024-11-08T20:05:44.000000000Z\",\n git_branch: \"mkeeter/drop-skipped-backpressure-guard\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:36.984962456Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1596} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:36.984977375Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1596} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:36.984991625Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1596} {"msg":"Crucible 3e8c943f-896c-4b97-91d3-b6e4fc98e660 has session id: 2af3f087-29e3-49d5-a922-551ee9878143","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:36.985428124Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1596,"session_id":"2af3f087-29e3-49d5-a922-551ee9878143"} {"msg":"Upstairs opts: Upstairs UUID: 3e8c943f-896c-4b97-91d3-b6e4fc98e660, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:36.985474241Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1596,"session_id":"2af3f087-29e3-49d5-a922-551ee9878143"} {"msg":"Crucible stats registered with UUID: 3e8c943f-896c-4b97-91d3-b6e4fc98e660","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:36.98550019Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1596,"session_id":"2af3f087-29e3-49d5-a922-551ee9878143"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:37.017494746Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1596,"task":"crutest"} {"msg":"The guest has requested activation with gen:104","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:37.017528015Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1596} {"msg":"3e8c943f-896c-4b97-91d3-b6e4fc98e660 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:37.017562013Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1596,"session_id":"2af3f087-29e3-49d5-a922-551ee9878143"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:37.017578602Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1596,"client":"0","":"downstairs","session_id":"2af3f087-29e3-49d5-a922-551ee9878143"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:37.017607251Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1596,"client":"0","":"downstairs","session_id":"2af3f087-29e3-49d5-a922-551ee9878143"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:37.01762512Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1596,"client":"1","":"downstairs","session_id":"2af3f087-29e3-49d5-a922-551ee9878143"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:37.017642289Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1596,"client":"1","":"downstairs","session_id":"2af3f087-29e3-49d5-a922-551ee9878143"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:37.017704906Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1596,"client":"2","":"downstairs","session_id":"2af3f087-29e3-49d5-a922-551ee9878143"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:37.017746614Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1596,"client":"2","":"downstairs","session_id":"2af3f087-29e3-49d5-a922-551ee9878143"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:37.017769083Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1596,"":"io task","client":"2","":"downstairs","session_id":"2af3f087-29e3-49d5-a922-551ee9878143"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:37.017784382Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1596,"":"io task","client":"0","":"downstairs","session_id":"2af3f087-29e3-49d5-a922-551ee9878143"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:37.017798551Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1596,"":"io task","client":"1","":"downstairs","session_id":"2af3f087-29e3-49d5-a922-551ee9878143"} {"msg":"ds_connection connected from Ok(127.0.0.1:50010)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:37.018069328Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1596,"":"io task","client":"2","":"downstairs","session_id":"2af3f087-29e3-49d5-a922-551ee9878143"} {"msg":"ds_connection connected from Ok(127.0.0.1:64951)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:37.018154444Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1596,"":"io task","client":"1","":"downstairs","session_id":"2af3f087-29e3-49d5-a922-551ee9878143"} {"msg":"ds_connection connected from Ok(127.0.0.1:35456)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:37.018186032Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1596,"":"io task","client":"0","":"downstairs","session_id":"2af3f087-29e3-49d5-a922-551ee9878143"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:37.018484728Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1596,"client":"2","":"downstairs","session_id":"2af3f087-29e3-49d5-a922-551ee9878143"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:37.018521666Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1596,"client":"2","":"downstairs","session_id":"2af3f087-29e3-49d5-a922-551ee9878143"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:37.018551205Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1596,"client":"0","":"downstairs","session_id":"2af3f087-29e3-49d5-a922-551ee9878143"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:37.018580063Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1596,"client":"0","":"downstairs","session_id":"2af3f087-29e3-49d5-a922-551ee9878143"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:37.018688958Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1596,"client":"1","":"downstairs","session_id":"2af3f087-29e3-49d5-a922-551ee9878143"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:37.018725466Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1596,"client":"1","":"downstairs","session_id":"2af3f087-29e3-49d5-a922-551ee9878143"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:37.019076249Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1596,"client":"0","":"downstairs","session_id":"2af3f087-29e3-49d5-a922-551ee9878143"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:37.01926699Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1596,"client":"2","":"downstairs","session_id":"2af3f087-29e3-49d5-a922-551ee9878143"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:37.01946101Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1596,"client":"1","":"downstairs","session_id":"2af3f087-29e3-49d5-a922-551ee9878143"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:37.019614813Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1596,"client":"0","":"downstairs","session_id":"2af3f087-29e3-49d5-a922-551ee9878143"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:37.019636781Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1596,"client":"0","":"downstairs","session_id":"2af3f087-29e3-49d5-a922-551ee9878143"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:37.01966158Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1596,"session_id":"2af3f087-29e3-49d5-a922-551ee9878143"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:37.019807213Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1596,"client":"2","":"downstairs","session_id":"2af3f087-29e3-49d5-a922-551ee9878143"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:37.019831972Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1596,"client":"2","":"downstairs","session_id":"2af3f087-29e3-49d5-a922-551ee9878143"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:37.019846191Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1596,"session_id":"2af3f087-29e3-49d5-a922-551ee9878143"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:37.020019623Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1596,"client":"1","":"downstairs","session_id":"2af3f087-29e3-49d5-a922-551ee9878143"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:37.020039012Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1596,"client":"1","":"downstairs","session_id":"2af3f087-29e3-49d5-a922-551ee9878143"} {"msg":"[0]R flush_numbers[0..12]: [188, 165, 176, 180, 185, 189, 185, 178, 185, 189, 175, 173]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:37.020054481Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1596,"":"downstairs","":"downstairs","session_id":"2af3f087-29e3-49d5-a922-551ee9878143"} {"msg":"[0]R generation[0..12]: [102, 89, 98, 100, 102, 103, 102, 99, 102, 103, 98, 96]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:37.020068731Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1596,"":"downstairs","":"downstairs","session_id":"2af3f087-29e3-49d5-a922-551ee9878143"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:37.02008314Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1596,"":"downstairs","":"downstairs","session_id":"2af3f087-29e3-49d5-a922-551ee9878143"} {"msg":"[1]R flush_numbers[0..12]: [188, 165, 176, 180, 185, 189, 185, 178, 185, 189, 175, 173]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:37.020097049Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1596,"":"downstairs","":"downstairs","session_id":"2af3f087-29e3-49d5-a922-551ee9878143"} {"msg":"[1]R generation[0..12]: [102, 89, 98, 100, 102, 103, 102, 99, 102, 103, 98, 96]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:37.020110758Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1596,"":"downstairs","":"downstairs","session_id":"2af3f087-29e3-49d5-a922-551ee9878143"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:37.020124688Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1596,"":"downstairs","":"downstairs","session_id":"2af3f087-29e3-49d5-a922-551ee9878143"} {"msg":"[2]R flush_numbers[0..12]: [188, 165, 176, 180, 185, 189, 185, 178, 185, 189, 175, 173]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:37.020141507Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1596,"":"downstairs","":"downstairs","session_id":"2af3f087-29e3-49d5-a922-551ee9878143"} {"msg":"[2]R generation[0..12]: [102, 89, 98, 100, 102, 103, 102, 99, 102, 103, 98, 96]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:37.020155266Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1596,"":"downstairs","":"downstairs","session_id":"2af3f087-29e3-49d5-a922-551ee9878143"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:37.020169206Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1596,"":"downstairs","":"downstairs","session_id":"2af3f087-29e3-49d5-a922-551ee9878143"} {"msg":"Max found gen is 104","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:37.020182205Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1596,"":"downstairs","":"downstairs","session_id":"2af3f087-29e3-49d5-a922-551ee9878143"} {"msg":"Generation requested: 104 >= found:104","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:37.020195494Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1596,"":"downstairs","":"downstairs","session_id":"2af3f087-29e3-49d5-a922-551ee9878143"} {"msg":"Next flush: 190","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:37.020208393Z","Wait for a query_work_queue command to finish before sending IO hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1596,"":"downstairs","":"downstairs","session_id":"2af3f087-29e3-49d5-a922-551ee9878143"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:37.020228903Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1596,"":"downstairs","":"downstairs","session_id":"2af3f087-29e3-49d5-a922-551ee9878143"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:37.020242492Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1596,"session_id":"2af3f087-29e3-49d5-a922-551ee9878143"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:37.020254781Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1596,"client":"0","":"downstairs","session_id":"2af3f087-29e3-49d5-a922-551ee9878143"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:37.020268401Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1596,"client":"1","":"downstairs","session_id":"2af3f087-29e3-49d5-a922-551ee9878143"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:37.02028172Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1596,"client":"2","":"downstairs","session_id":"2af3f087-29e3-49d5-a922-551ee9878143"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:37.020295049Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1596,"session_id":"2af3f087-29e3-49d5-a922-551ee9878143"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:37.020307299Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1596,"session_id":"2af3f087-29e3-49d5-a922-551ee9878143"} {"msg":"3e8c943f-896c-4b97-91d3-b6e4fc98e660 is now active with session: 2af3f087-29e3-49d5-a922-551ee9878143","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:37.020320128Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1596,"session_id":"2af3f087-29e3-49d5-a922-551ee9878143"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:37.020332127Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1596,"session_id":"2af3f087-29e3-49d5-a922-551ee9878143"} {"msg":"The guest has finished waiting for activation with:104","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:37.020346067Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1596} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 118 len 4096 data: 6 02/30 Write block 502 len 20480 data: 9 7 8 7 6 03/30 Read block 194 len 24576 04/30 Read block 586 len 40960 05/30 Read block 560 len 20480 06/30 Read block 591 len 8192 07/30 Write block 528 len 36864 data: 8 7 7 9 7 8 8 9 9 08/30 Write block 561 len 24576 data: 5 5 5 6 8 7 09/30 Read block 418 len 8192 10/30 Read block 503 len 16384 11/30 Write block 352 len 24576 data: 10 14 12 13 13 13 12/30 Read block 76 len 12288 13/30 Write block 446 len 28672 data: 12 11 9 9 8 8 10 14/30 Read block 210 len 4096 15/30 Read block 141 len 28672 16/30 Write block 432 len 32768 data: 10 9 9 12 14 12 12 11 17/30 Write block 34 len 36864 data: 9 7 7 8 10 11 11 8 10 18/30 Read block 483 len 40960 19/30 Read block 246 len 16384 20/30 Read block 305 len 24576 21/30 Read block 492 len 28672 22/30 Read block 165 len 16384 23/30 Write block 78 len 16384 data: 8 8 7 6 24/30 Read block 525 len 12288 25/30 Read block 587 len 16384 26/30 Write block 359 len 12288 data: 14 14 15 27/30 Write block 273 len 8192 data: 8 10 28/30 Write block 478 len 20480 data: 12 12 10 11 10 29/30 Read block 174 len 8192 30/30 Read block 490 len 40960 ---------------------------------------------------------------- Crucible gen:104 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Done Done Done false 1001 Acked Write 5 Done Done Done false 1002 Acked Read 6 Done Done Done false 1003 Acked Read 10 Done Done Done false 1004 Acked Read 5 Done Done Done false 1005 Acked Read 2 Done Done Done false 1006 Acked Write 9 Done Done Done false 1007 Acked Write 6 Done Done Done false 1008 Acked Read 2 Done Done Done false 1009 Acked Read 4 Done Done Done false 1010 Acked Write 6 Done Done Done false 1011 Acked Read 3 Done Done Done false 1012 Acked Write 7 Done Done Done false 1013 Acked Read 1 Done Done Done false 1014 Acked Read 7 Done Done Done false 1015 Acked Write 8 Done Done Done false 1016 Acked Write 9 Done Done Done false 1017 Acked Read 10 Done Done Done false 1018 Acked Read 4 Done Done Done false 1019 Acked Read 6 Done Done Done false 1020 Acked Read 7 Done Done Done false 1021 Acked Read 4 Done Done Done false 1022 Acked Write 4 Done Done Done false 1023 Acked Read 3 Done Done Done false 1024 Acked Read 4 Done Done Done false 1025 Acked Write 3 Done Done Done false 1026 Acked Write 2 Done Done Done false 1027 Acked Write 5 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Read 10 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 30 30 29 89 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 0 0 0 Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:26:37.040377668Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1596} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 102 102 102 188 188 188 F F F 1 020-039 89 89 89 165 165 165 T T T 2 040-059 98 98 98 176 176 176 T T T 3 060-079 100 100 100 180 180 180 T T T 4 080-099 102 102 102 185 185 185 T T T 5 100-119 103 103 103 189 189 189 T T T 6 120-139 102 102 102 185 185 185 F F F 7 140-159 99 99 99 178 178 178 F F F 8 160-179 102 102 102 185 185 185 F F F 9 180-199 103 103 103 189 189 189 F F F 10 200-219 98 98 98 175 175 175 F F F 11 220-239 96 96 96 173 173 173 F F F 12 240-259 102 102 102 183 183 183 F F F 13 260-279 100 100 100 180 180 180 T T T 14 280-299 97 97 97 174 174 174 F F F 15 300-319 90 90 90 169 169 169 F F F 16 320-339 97 97 97 174 174 174 F F F 17 340-359 100 100 100 180 180 180 T T T 18 360-379 98 98 98 176 176 176 T T T 19 380-399 90 90 90 168 168 168 F F F 20 400-419 100 100 100 181 181 181 F F F 21 420-439 99 99 99 178 178 178 T T T 22 440-459 103 103 103 189 189 189 T T T 23 460-479 103 103 103 189 189 189 T T T 24 480-499 103 103 103 189 189 189 T T T 25 500-519 100 100 100 180 180 180 T T T 26 520-539 96 96 96 173 173 173 T T T 27 540-559 96 96 96 173 173 173 F F F 28 560-579 93 93 93 172 172 172 T T T 29 580-599 98 98 98 175 175 175 F F F Max gen: 103, Max flush: 189 A Difference in extent metadata is expected here On loop 52 Verifying data now No change after verify Loop: 52 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 102 102 102 188 188 188 F F F 1 020-039 104 104 104 190 190 190 F F F 2 040-059 104 104 104 190 190 190 F F F 3 060-079 104 104 104 190 190 190 F F F 4 080-099 104 104 104 190 190 190 F F F 5 100-119 104 104 104 190 190 190 F F F 6 120-139 102 102 102 185 185 185 F F F 7 140-159 99 99 99 178 178 178 F F F 8 160-179 102 102 102 185 185 185 F F F 9 180-199 103 103 103 189 189 189 F F F 10 200-219 98 98 98 175 175 175 F F F 11 220-239 96 96 96 173 173 173 F F F 12 240-259 102 102 102 183 183 183 F F F 13 260-279 104 104 104 190 190 190 F F F 14 280-299 97 97 97 174 174 174 F F F 15 300-319 90 90 90 169 169 169 F F F 16 320-339 97 97 97 174 174 174 F F F 17 340-359 104 104 104 190 190 190 F F F 18 360-379 104 104 104 190 190 190 F F F 19 380-399 90 90 90 168 168 168 F F F 20 400-419 100 100 100 181 181 181 F F F 21 420-439 104 104 104 190 190 190 F F F 22 440-459 104 104 104 190 190 190 F F F 23 460-479 104 104 104 190 190 190 F F F 24 480-499 104 104 104 190 190 190 F F F 25 500-519 104 104 104 190 190 190 F F F 26 520-539 104 104 104 190 190 190 F F F 27 540-559 96 96 96 173 173 173 F F F 28 560-579 104 104 104 190 190 190 F F F 29 580-599 98 98 98 175 175 175 F F F Max gen: 104, Max flush: 190 Begin loop 53 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:41.502247308Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1604,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:26:41.545036617Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1604,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:41.545444237Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1604,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:41.577579796Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1604,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:41.578135Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1604} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"0fd98d0d5d909611f8b408ccda2559d35fe117bb\",\n git_commit_timestamp: \"2024-11-08T20:05:44.000000000Z\",\n git_branch: \"mkeeter/drop-skipped-backpressure-guard\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:41.578172567Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1604} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:41.578187597Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1604} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:41.578201796Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1604} {"msg":"Crucible 58d2acd2-283a-426a-b2f1-1b8f66f31e10 has session id: a621bb4e-aa6c-4fb7-9354-fc1a289f7121","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:41.578685282Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1604,"session_id":"a621bb4e-aa6c-4fb7-9354-fc1a289f7121"} {"msg":"Upstairs opts: Upstairs UUID: 58d2acd2-283a-426a-b2f1-1b8f66f31e10, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:41.57874109Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1604,"session_id":"a621bb4e-aa6c-4fb7-9354-fc1a289f7121"} {"msg":"Crucible stats registered with UUID: 58d2acd2-283a-426a-b2f1-1b8f66f31e10","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:41.578772898Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1604,"session_id":"a621bb4e-aa6c-4fb7-9354-fc1a289f7121"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:41.609617001Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1604,"task":"crutest"} {"msg":"The guest has requested activation with gen:106","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:41.609663979Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1604} {"msg":"58d2acd2-283a-426a-b2f1-1b8f66f31e10 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:41.609700357Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1604,"session_id":"a621bb4e-aa6c-4fb7-9354-fc1a289f7121"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:41.609730995Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1604,"client":"0","":"downstairs","session_id":"a621bb4e-aa6c-4fb7-9354-fc1a289f7121"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:41.609778393Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1604,"client":"0","":"downstairs","session_id":"a621bb4e-aa6c-4fb7-9354-fc1a289f7121"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:41.609810921Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1604,"client":"1","":"downstairs","session_id":"a621bb4e-aa6c-4fb7-9354-fc1a289f7121"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:41.60983836Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1604,"client":"1","":"downstairs","session_id":"a621bb4e-aa6c-4fb7-9354-fc1a289f7121"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:41.609865549Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1604,"client":"2","":"downstairs","session_id":"a621bb4e-aa6c-4fb7-9354-fc1a289f7121"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:41.609892197Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1604,"client":"2","":"downstairs","session_id":"a621bb4e-aa6c-4fb7-9354-fc1a289f7121"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:41.609920126Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1604,"":"io task","client":"2","":"downstairs","session_id":"a621bb4e-aa6c-4fb7-9354-fc1a289f7121"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:41.609948245Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1604,"":"io task","client":"0","":"downstairs","session_id":"a621bb4e-aa6c-4fb7-9354-fc1a289f7121"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:41.609975223Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1604,"":"io task","client":"1","":"downstairs","session_id":"a621bb4e-aa6c-4fb7-9354-fc1a289f7121"} {"msg":"ds_connection connected from Ok(127.0.0.1:51743)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:41.610003472Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1604,"":"io task","client":"2","":"downstairs","session_id":"a621bb4e-aa6c-4fb7-9354-fc1a289f7121"} {"msg":"ds_connection connected from Ok(127.0.0.1:43915)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:41.610259709Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1604,"":"io task","client":"1","":"downstairs","session_id":"a621bb4e-aa6c-4fb7-9354-fc1a289f7121"} {"msg":"ds_connection connected from Ok(127.0.0.1:53259)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:41.610392653Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1604,"":"io task","client":"0","":"downstairs","session_id":"a621bb4e-aa6c-4fb7-9354-fc1a289f7121"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:41.610777524Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1604,"client":"1","":"downstairs","session_id":"a621bb4e-aa6c-4fb7-9354-fc1a289f7121"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:41.610823972Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1604,"client":"1","":"downstairs","session_id":"a621bb4e-aa6c-4fb7-9354-fc1a289f7121"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:41.61085496Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1604,"client":"2","":"downstairs","session_id":"a621bb4e-aa6c-4fb7-9354-fc1a289f7121"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:41.610885689Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1604,"client":"2","":"downstairs","session_id":"a621bb4e-aa6c-4fb7-9354-fc1a289f7121"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:41.610913347Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1604,"client":"0","":"downstairs","session_id":"a621bb4e-aa6c-4fb7-9354-fc1a289f7121"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:41.610947976Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1604,"client":"0","":"downstairs","session_id":"a621bb4e-aa6c-4fb7-9354-fc1a289f7121"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:41.611404473Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1604,"client":"1","":"downstairs","session_id":"a621bb4e-aa6c-4fb7-9354-fc1a289f7121"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:41.611543347Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1604,"client":"0","":"downstairs","session_id":"a621bb4e-aa6c-4fb7-9354-fc1a289f7121"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:41.611586585Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1604,"client":"2","":"downstairs","session_id":"a621bb4e-aa6c-4fb7-9354-fc1a289f7121"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:41.611977395Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1604,"client":"1","":"downstairs","session_id":"a621bb4e-aa6c-4fb7-9354-fc1a289f7121"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:41.612046422Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1604,"client":"1","":"downstairs","session_id":"a621bb4e-aa6c-4fb7-9354-fc1a289f7121"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:41.612072681Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1604,"session_id":"a621bb4e-aa6c-4fb7-9354-fc1a289f7121"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:41.61208719Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1604,"client":"2","":"downstairs","session_id":"a621bb4e-aa6c-4fb7-9354-fc1a289f7121"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:41.612101459Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1604,"client":"2","":"downstairs","session_id":"a621bb4e-aa6c-4fb7-9354-fc1a289f7121"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:41.612115069Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1604,"session_id":"a621bb4e-aa6c-4fb7-9354-fc1a289f7121"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:41.612184025Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1604,"client":"0","":"downstairs","session_id":"a621bb4e-aa6c-4fb7-9354-fc1a289f7121"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:41.612204114Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1604,"client":"0","":"downstairs","session_id":"a621bb4e-aa6c-4fb7-9354-fc1a289f7121"} {"msg":"[0]R flush_numbers[0..12]: [188, 190, 190, 190, 190, 190, 185, 178, 185, 189, 175, 173]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:41.612218534Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1604,"":"downstairs","":"downstairs","session_id":"a621bb4e-aa6c-4fb7-9354-fc1a289f7121"} {"msg":"[0]R generation[0..12]: [102, 104, 104, 104, 104, 104, 102, 99, 102, 103, 98, 96]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:41.612232613Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1604,"":"downstairs","":"downstairs","session_id":"a621bb4e-aa6c-4fb7-9354-fc1a289f7121"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:41.612246792Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1604,"":"downstairs","":"downstairs","session_id":"a621bb4e-aa6c-4fb7-9354-fc1a289f7121"} {"msg":"[1]R flush_numbers[0..12]: [188, 190, 190, 190, 190, 190, 185, 178, 185, 189, 175, 173]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:41.612264541Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1604,"":"downstairs","":"downstairs","session_id":"a621bb4e-aa6c-4fb7-9354-fc1a289f7121"} {"msg":"[1]R generation[0..12]: [102, 104, 104, 104, 104, 104, 102, 99, 102, 103, 98, 96]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:41.612278701Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1604,"":"downstairs","":"downstairs","session_id":"a621bb4e-aa6c-4fb7-9354-fc1a289f7121"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:41.61229276Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1604,"":"downstairs","":"downstairs","session_id":"a621bb4e-aa6c-4fb7-9354-fc1a289f7121"} {"msg":"[2]R flush_numbers[0..12]: [188, 190, 190, 190, 190, 190, 185, 178, 185, 189, 175, 173]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:41.612306399Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1604,"":"downstairs","":"downstairs","session_id":"a621bb4e-aa6c-4fb7-9354-fc1a289f7121"} {"msg":"[2]R generation[0..12]: [102, 104, 104, 104, 104, 104, 102, 99, 102, 103, 98, 96]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:41.612320209Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1604,"":"downstairs","":"downstairs","session_id":"a621bb4e-aa6c-4fb7-9354-fc1a289f7121"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:41.612334478Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1604,"":"downstairs","":"downstairs","session_id":"a621bb4e-aa6c-4fb7-9354-fc1a289f7121"} {"msg":"Max found gen is 105","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:41.612347767Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1604,"":"downstairs","":"downstairs","session_id":"a621bb4e-aa6c-4fb7-9354-fc1a289f7121"} {"msg":"Generation requested: 106 >= found:105","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:41.612361167Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1604,"":"downstairs","":"downstairs","session_id":"a621bb4e-aa6c-4fb7-9354-fc1a289f7121"} {"msg":"Next flush: 191","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:41.612374286Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1604Wait for a query_work_queue command to finish before sending IO ,"":"downstairs","":"downstairs","session_id":"a621bb4e-aa6c-4fb7-9354-fc1a289f7121"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:41.612396505Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1604,"":"downstairs","":"downstairs","session_id":"a621bb4e-aa6c-4fb7-9354-fc1a289f7121"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:41.612410074Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1604,"session_id":"a621bb4e-aa6c-4fb7-9354-fc1a289f7121"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:41.612422294Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1604,"client":"0","":"downstairs","session_id":"a621bb4e-aa6c-4fb7-9354-fc1a289f7121"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:41.612435783Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1604,"client":"1","":"downstairs","session_id":"a621bb4e-aa6c-4fb7-9354-fc1a289f7121"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:41.612449102Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1604,"client":"2","":"downstairs","session_id":"a621bb4e-aa6c-4fb7-9354-fc1a289f7121"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:41.612462672Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1604,"session_id":"a621bb4e-aa6c-4fb7-9354-fc1a289f7121"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:41.612478451Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1604,"session_id":"a621bb4e-aa6c-4fb7-9354-fc1a289f7121"} {"msg":"58d2acd2-283a-426a-b2f1-1b8f66f31e10 is now active with session: a621bb4e-aa6c-4fb7-9354-fc1a289f7121","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:41.61249152Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1604,"session_id":"a621bb4e-aa6c-4fb7-9354-fc1a289f7121"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:41.6125036Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1604,"session_id":"a621bb4e-aa6c-4fb7-9354-fc1a289f7121"} {"msg":"The guest has finished waiting for activation with:106","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:41.612515989Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1604} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 339 len 36864 data: 12 11 12 13 11 9 10 12 10 02/30 Write block 518 len 24576 data: 8 9 7 7 6 7 03/30 Read block 342 len 8192 04/30 Read block 569 len 8192 05/30 Read block 3 len 28672 06/30 Read block 581 len 32768 07/30 Write block 273 len 32768 data: 9 11 10 10 9 7 7 8 08/30 Read block 305 len 4096 09/30 Write block 103 len 32768 data: 8 7 6 6 9 8 9 8 10/30 Read block 357 len 4096 11/30 Read block 457 len 28672 12/30 Flush 13/30 Write block 35 len 36864 data: 8 8 9 11 12 12 9 11 11 14/30 Write block 276 len 20480 data: 11 10 8 8 9 15/30 Read block 505 len 8192 16/30 Flush 17/30 Write block 219 len 36864 data: 8 7 10 11 9 9 9 10 12 18/30 Write block 383 len 32768 data: 6 6 4 4 4 5 5 7 19/30 Read block 84 len 20480 20/30 Read block 62 len 8192 21/30 Read block 459 len 32768 22/30 Read block 282 len 20480 23/30 Write block 124 len 12288 data: 11 8 9 24/30 Read block 56 len 4096 25/30 Write block 309 len 28672 data: 6 7 8 8 10 7 10 26/30 Write block 193 len 36864 data: 8 8 8 9 9 9 9 8 7 27/30 Read block 311 len 40960 28/30 Read block 219 len 8192 29/30 Write block 186 len 4096 data: 11 30/30 Write block 437 len 12288 data: 13 13 12 ---------------------------------------------------------------- Crucible gen:106 GIO:true work queues: Upstairs:0 downstairs:14 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1016 Acked Write 9 Done Done Done false 1017 Acked Write 8 Done Done Done false 1018 Acked Read 5 Done Done Done false 1019 Acked Read 2 Done Done Done false 1020 Acked Read 8 Done Done Done false 1021 Acked Read 5 Done Done Done false 1022 Acked Write 3 Done Done Done false 1023 Acked Read 1 Done Done Done false 1024 Acked Write 7 Done Done Done false 1025 Acked Write 9 Done Done Done false 1026 Acked Read 10 Done Done Done false 1027 Acked Read 2 Done Done Done false 1028 Acked Write 1 Done Done Done false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 13 13 13 39 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1015 1015 1015 Downstairs last five completed: 1015 1014 1013 1012 1011 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:26:41.635573332Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1604} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 102 102 102 188 188 188 F F F 1 020-039 106 106 106 192 192 192 F F F 2 040-059 106 106 106 192 192 192 F F F 3 060-079 104 104 104 190 190 190 F F F 4 080-099 104 104 104 190 190 190 F F F 5 100-119 106 106 106 191 191 191 F F F 6 120-139 102 102 102 185 185 185 T T T 7 140-159 99 99 99 178 178 178 F F F 8 160-179 102 102 102 185 185 185 F F F 9 180-199 103 103 103 189 189 189 T T T 10 200-219 98 98 98 175 175 175 T T T 11 220-239 96 96 96 173 173 173 T T T 12 240-259 102 102 102 183 183 183 F F F 13 260-279 106 106 106 192 192 192 F F F 14 280-299 106 106 106 192 192 192 F F F 15 300-319 90 90 90 169 169 169 T T T 16 320-339 106 106 106 191 191 191 F F F 17 340-359 106 106 106 191 191 191 F F F 18 360-379 104 104 104 190 190 190 F F F 19 380-399 90 90 90 168 168 168 T T T 20 400-419 100 100 100 181 181 181 F F F 21 420-439 104 104 104 190 190 190 T T T 22 440-459 104 104 104 190 190 190 F F F 23 460-479 104 104 104 190 190 190 F F F 24 480-499 104 104 104 190 190 190 F F F 25 500-519 106 106 106 191 191 191 F F F 26 520-539 106 106 106 191 191 191 F F F 27 540-559 96 96 96 173 173 173 F F F 28 560-579 104 104 104 190 190 190 F F F 29 580-599 98 98 98 175 175 175 F F F Max gen: 106, Max flush: 192 A Difference in extent metadata is expected here On loop 53 Verifying data now No change after verify Loop: 53 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 102 102 102 188 188 188 F F F 1 020-039 106 106 106 192 192 192 F F F 2 040-059 106 106 106 192 192 192 F F F 3 060-079 104 104 104 190 190 190 F F F 4 080-099 104 104 104 190 190 190 F F F 5 100-119 106 106 106 191 191 191 F F F 6 120-139 107 107 107 193 193 193 F F F 7 140-159 99 99 99 178 178 178 F F F 8 160-179 102 102 102 185 185 185 F F F 9 180-199 107 107 107 193 193 193 F F F 10 200-219 107 107 107 193 193 193 F F F 11 220-239 107 107 107 193 193 193 F F F 12 240-259 102 102 102 183 183 183 F F F 13 260-279 106 106 106 192 192 192 F F F 14 280-299 106 106 106 192 192 192 F F F 15 300-319 107 107 107 193 193 193 F F F 16 320-339 106 106 106 191 191 191 F F F 17 340-359 106 106 106 191 191 191 F F F 18 360-379 104 104 104 190 190 190 F F F 19 380-399 107 107 107 193 193 193 F F F 20 400-419 100 100 100 181 181 181 F F F 21 420-439 107 107 107 193 193 193 F F F 22 440-459 104 104 104 190 190 190 F F F 23 460-479 104 104 104 190 190 190 F F F 24 480-499 104 104 104 190 190 190 F F F 25 500-519 106 106 106 191 191 191 F F F 26 520-539 106 106 106 191 191 191 F F F 27 540-559 96 96 96 173 173 173 F F F 28 560-579 104 104 104 190 190 190 F F F 29 580-599 98 98 98 175 175 175 F F F Max gen: 107, Max flush: 193 Begin loop 54 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:46.012453265Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1612,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:46.057799919Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1612,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:46.058439468Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1612} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"0fd98d0d5d909611f8b408ccda2559d35fe117bb\",\n git_commit_timestamp: \"2024-11-08T20:05:44.000000000Z\",\n git_branch: \"mkeeter/drop-skipped-backpressure-guard\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:46.058477576Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1612} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:46.058492735Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1612} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:46.058507285Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1612} {"msg":"Crucible 24d163af-0571-4859-afcd-7a3e75a0070f has session id: eb2003d9-792c-44e5-90be-2b75d7e9299d","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:46.058911525Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1612,"session_id":"eb2003d9-792c-44e5-90be-2b75d7e9299d"} {"msg":"Upstairs opts: Upstairs UUID: 24d163af-0571-4859-afcd-7a3e75a0070f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:46.058953323Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1612,"session_id":"eb2003d9-792c-44e5-90be-2b75d7e9299d"} {"msg":"Crucible stats registered with UUID: 24d163af-0571-4859-afcd-7a3e75a0070f","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:46.058970702Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1612,"session_id":"eb2003d9-792c-44e5-90be-2b75d7e9299d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:46.090195406Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1612,"task":"crutest"} {"msg":"The guest has requested activation with gen:108","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:46.090227395Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1612} {"msg":"24d163af-0571-4859-afcd-7a3e75a0070f active request set","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:46.090276962Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1612,"session_id":"eb2003d9-792c-44e5-90be-2b75d7e9299d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:46.090296941Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1612,"client":"0","":"downstairs","session_id":"eb2003d9-792c-44e5-90be-2b75d7e9299d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:46.09032513Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1612,"client":"0","":"downstairs","session_id":"eb2003d9-792c-44e5-90be-2b75d7e9299d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:46.090356878Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1612,"client":"1","":"downstairs","session_id":"eb2003d9-792c-44e5-90be-2b75d7e9299d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:46.090395736Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1612,"client":"1","":"downstairs","session_id":"eb2003d9-792c-44e5-90be-2b75d7e9299d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:46.090432594Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1612,"client":"2","":"downstairs","session_id":"eb2003d9-792c-44e5-90be-2b75d7e9299d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:46.090468153Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1612,"client":"2","":"downstairs","session_id":"eb2003d9-792c-44e5-90be-2b75d7e9299d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:46.090502591Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1612,"":"io task","client":"2","":"downstairs","session_id":"eb2003d9-792c-44e5-90be-2b75d7e9299d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:46.090538589Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1612,"":"io task","client":"0","":"downstairs","session_id":"eb2003d9-792c-44e5-90be-2b75d7e9299d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:46.090575227Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1612,"":"io task","client":"1","":"downstairs","session_id":"eb2003d9-792c-44e5-90be-2b75d7e9299d"} {"msg":"ds_connection connected from Ok(127.0.0.1:51490)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:46.090688462Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1612,"":"io task","client":"2","":"downstairs","session_id":"eb2003d9-792c-44e5-90be-2b75d7e9299d"} {"msg":"ds_connection connected from Ok(127.0.0.1:38741)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:46.09073017Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1612,"":"io task","client":"0","":"downstairs","session_id":"eb2003d9-792c-44e5-90be-2b75d7e9299d"} {"msg":"ds_connection connected from Ok(127.0.0.1:42907)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:46.090761578Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1612,"":"io task","client":"1","":"downstairs","session_id":"eb2003d9-792c-44e5-90be-2b75d7e9299d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:46.091214756Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1612,"client":"0","":"downstairs","session_id":"eb2003d9-792c-44e5-90be-2b75d7e9299d"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:46.091244825Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1612,"client":"0","":"downstairs","session_id":"eb2003d9-792c-44e5-90be-2b75d7e9299d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:46.091401667Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1612,"client":"1","":"downstairs","session_id":"eb2003d9-792c-44e5-90be-2b75d7e9299d"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:46.091439865Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1612,"client":"1","":"downstairs","session_id":"eb2003d9-792c-44e5-90be-2b75d7e9299d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:46.091468014Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1612,"client":"2","":"downstairs","session_id":"eb2003d9-792c-44e5-90be-2b75d7e9299d"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:46.091483633Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1612,"client":"2","":"downstairs","session_id":"eb2003d9-792c-44e5-90be-2b75d7e9299d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:46.091994488Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1612,"client":"2","":"downstairs","session_id":"eb2003d9-792c-44e5-90be-2b75d7e9299d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:46.092043256Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1612,"client":"0","":"downstairs","session_id":"eb2003d9-792c-44e5-90be-2b75d7e9299d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:46.092150461Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1612,"client":"1","":"downstairs","session_id":"eb2003d9-792c-44e5-90be-2b75d7e9299d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:46.092608088Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1612,"client":"2","":"downstairs","session_id":"eb2003d9-792c-44e5-90be-2b75d7e9299d"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:46.092646946Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1612,"client":"2","":"downstairs","session_id":"eb2003d9-792c-44e5-90be-2b75d7e9299d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:46.092663855Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1612,"session_id":"eb2003d9-792c-44e5-90be-2b75d7e9299d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:46.092713683Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1612,"client":"1","":"downstairs","session_id":"eb2003d9-792c-44e5-90be-2b75d7e9299d"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:46.092742001Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1612,"client":"1","":"downstairs","session_id":"eb2003d9-792c-44e5-90be-2b75d7e9299d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:46.092757381Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1612,"session_id":"eb2003d9-792c-44e5-90be-2b75d7e9299d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:46.09278149Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1612,"client":"0","":"downstairs","session_id":"eb2003d9-792c-44e5-90be-2b75d7e9299d"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:46.092796089Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1612,"client":"0","":"downstairs","session_id":"eb2003d9-792c-44e5-90be-2b75d7e9299d"} {"msg":"[0]R flush_numbers[0..12]: [188, 192, 192, 190, 190, 191, 193, 178, 185, 193, 193, 193]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:46.092810398Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1612,"":"downstairs","":"downstairs","session_id":"eb2003d9-792c-44e5-90be-2b75d7e9299d"} {"msg":"[0]R generation[0..12]: [102, 106, 106, 104, 104, 106, 107, 99, 102, 107, 107, 107]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:46.092824597Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1612,"":"downstairs","":"downstairs","session_id":"eb2003d9-792c-44e5-90be-2b75d7e9299d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:46.092838657Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1612,"":"downstairs","":"downstairs","session_id":"eb2003d9-792c-44e5-90be-2b75d7e9299d"} {"msg":"[1]R flush_numbers[0..12]: [188, 192, 192, 190, 190, 191, 193, 178, 185, 193, 193, 193]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:46.092852616Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1612,"":"downstairs","":"downstairs","session_id":"eb2003d9-792c-44e5-90be-2b75d7e9299d"} {"msg":"[1]R generation[0..12]: [102, 106, 106, 104, 104, 106, 107, 99, 102, 107, 107, 107]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:46.092870005Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1612,"":"downstairs","":"downstairs","session_id":"eb2003d9-792c-44e5-90be-2b75d7e9299d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:46.092884535Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1612,"":"downstairs","":"downstairs","session_id":"eb2003d9-792c-44e5-90be-2b75d7e9299d"} {"msg":"[2]R flush_numbers[0..12]: [188, 192, 192, 190, 190, 191, 193, 178, 185, 193, 193, 193]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:46.092898284Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1612,"":"downstairs","":"downstairs","session_id":"eb2003d9-792c-44e5-90be-2b75d7e9299d"} {"msg":"[2]R generation[0..12]: [102, 106, 106, 104, 104, 106, 107, 99, 102, 107, 107, 107]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:46.092912123Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1612,"":"downstairs","":"downstairs","session_id":"eb2003d9-792c-44e5-90be-2b75d7e9299d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:46.092926023Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1612,"":"downstairs","":"downstairs","session_id":"eb2003d9-792c-44e5-90be-2b75d7e9299d"} {"msg":"Max found gen is 108","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:46.092939112Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1612,"":"downstairs","":"downstairs","session_id":"eb2003d9-792c-44e5-90be-2b75d7e9299d"} {"msg":"Generation requested: 108 >= found:108","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-08T20:26:46.092952341Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1612,"":"downstairs","":"downstairs","session_id":"eb2003d9-792c-44e5-90be-2b75d7e9299d"} {"msg":"Next flush: 194","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:46.09297413Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1612,"":"downstairs","":"downstairs","session_id":"eb2003d9-792c-44e5-90be-2b75d7e9299d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:46.092987069Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1612,"":"downstairs","":"downstairs","session_id":"eb2003d9-792c-44e5-90be-2b75d7e9299d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:46.093000509Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1612,"session_id":"eb2003d9-792c-44e5-90be-2b75d7e9299d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:46.093012688Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1612,"client":"0","":"downstairs","session_id":"eb2003d9-792c-44e5-90be-2b75d7e9299d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:46.093025988Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1612,"client":"1","":"downstairs","session_id":"eb2003d9-792c-44e5-90be-2b75d7e9299d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:46.093039087Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1612,"client":"2","":"downstairs","session_id":"eb2003d9-792c-44e5-90be-2b75d7e9299d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:46.093052406Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1612,"session_id":"eb2003d9-792c-44e5-90be-2b75d7e9299d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:46.093064796Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1612,"session_id":"eb2003d9-792c-44e5-90be-2b75d7e9299d"} {"msg":"24d163af-0571-4859-afcd-7a3e75a0070f is now active with session: eb2003d9-792c-44e5-90be-2b75d7e9299d","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:46.093079395Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1612,"session_id":"eb2003d9-792c-44e5-90be-2b75d7e9299d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:46.093091624Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1612,"session_id":"eb2003d9-792c-44e5-90be-2b75d7e9299d"} {"msg":"The guest has finished waiting for activation with:108","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:46.093104084Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1612} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 217 len 20480 data: 8 9 9 8 11 02/30 Read block 290 len 28672 03/30 Write block 326 len 4096 data: 8 04/30 Read block 375 len 32768 05/30 Write block 294 len 8192 data: 11 9 06/30 Read block 527 len 28672 07/30 Write block 268 len 40960 data: 9 9 9 9 9 10 12 11 12 11 08/30 Read block 271 len 16384 09/30 Read block 320 len 40960 10/30 Read block 222 len 12288 11/30 Read block 81 len 12288 12/30 Write block 314 len 28672 data: 8 11 9 7 9 11 9 13/30 Read block 433 len 24576 14/30 Write block 358 len 28672 data: 14 15 15 16 14 12 11 15/30 Flush 16/30 Write block 135 len 8192 data: 12 11 17/30 Read block 269 len 36864 18/30 Flush 19/30 Write block 416 len 8192 data: 8 8 20/30 Read block 560 len 32768 21/30 Write block 501 len 12288 data: 10 10 8 22/30 Read block 119 len 40960 23/30 Read block 494 len 36864 24/30 Read block 486 len 28672 25/30 Read block 455 len 24576 26/30 Write block 188 len 8192 data: 9 8 27/30 Read block 216 len 16384 28/30 Read block 100 len 32768 29/30 Read block 378 len 12288 30/30 Write block 477 len 20480 data: 12 13 13 11 12 ---------------------------------------------------------------- Crucible gen:108 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1018 Acked Write 2 Done Done Done false 1019 Acked Read 8 Done Done Done false 1020 Acked Write 3 Done Done Done false 1021 Acked Read 10 Done Done Done false 1022 Acked Read 9 Done Done Done false 1023 Acked Read 7 Done Done Done false 1024 Acked Read 6 Done Done Done false 1025 Acked Write 2 Done Done Done false 1026 Acked Read 4 Done Done Done false 1027 Acked Read 8 Done Done Done false 1028 Acked Read 3 Done Done Done false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 11 11 11 33 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1017 1017 1017 Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:26:46.118653365Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1612} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 102 102 102 188 188 188 F F F 1 020-039 106 106 106 192 192 192 F F F 2 040-059 106 106 106 192 192 192 F F F 3 060-079 104 104 104 190 190 190 F F F 4 080-099 104 104 104 190 190 190 F F F 5 100-119 106 106 106 191 191 191 F F F 6 120-139 108 108 108 195 195 195 F F F 7 140-159 99 99 99 178 178 178 F F F 8 160-179 102 102 102 185 185 185 F F F 9 180-199 107 107 107 193 193 193 T T T 10 200-219 108 108 108 194 194 194 F F F 11 220-239 108 108 108 194 194 194 F F F 12 240-259 102 102 102 183 183 183 F F F 13 260-279 108 108 108 194 194 194 F F F 14 280-299 108 108 108 194 194 194 F F F 15 300-319 108 108 108 194 194 194 F F F 16 320-339 108 108 108 194 194 194 F F F 17 340-359 108 108 108 194 194 194 F F F 18 360-379 108 108 108 194 194 194 F F F 19 380-399 107 107 107 193 193 193 F F F 20 400-419 100 100 100 181 181 181 T T T 21 420-439 107 107 107 193 193 193 F F F 22 440-459 104 104 104 190 190 190 F F F 23 460-479 104 104 104 190 190 190 T T T 24 480-499 104 104 104 190 190 190 T T T 25 500-519 106 106 106 191 191 191 T T T 26 520-539 106 106 106 191 191 191 F F F 27 540-559 96 96 96 173 173 173 F F F 28 560-579 104 104 104 190 190 190 F F F 29 580-599 98 98 98 175 175 175 F F F Max gen: 108, Max flush: 195 A Difference in extent metadata is expected here On loop 54 Verifying data now No change after verify Loop: 54 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 102 102 102 188 188 188 F F F 1 020-039 106 106 106 192 192 192 F F F 2 040-059 106 106 106 192 192 192 F F F 3 060-079 104 104 104 190 190 190 F F F 4 080-099 104 104 104 190 190 190 F F F 5 100-119 106 106 106 191 191 191 F F F 6 120-139 108 108 108 195 195 195 F F F 7 140-159 99 99 99 178 178 178 F F F 8 160-179 102 102 102 185 185 185 F F F 9 180-199 109 109 109 196 196 196 F F F 10 200-219 108 108 108 194 194 194 F F F 11 220-239 108 108 108 194 194 194 F F F 12 240-259 102 102 102 183 183 183 F F F 13 260-279 108 108 108 194 194 194 F F F 14 280-299 108 108 108 194 194 194 F F F 15 300-319 108 108 108 194 194 194 F F F 16 320-339 108 108 108 194 194 194 F F F 17 340-359 108 108 108 194 194 194 F F F 18 360-379 108 108 108 194 194 194 F F F 19 380-399 107 107 107 193 193 193 F F F 20 400-419 109 109 109 196 196 196 F F F 21 420-439 107 107 107 193 193 193 F F F 22 440-459 104 104 104 190 190 190 F F F 23 460-479 109 109 109 196 196 196 F F F 24 480-499 109 109 109 196 196 196 F F F 25 500-519 109 109 109 196 196 196 F F F 26 520-539 106 106 106 191 191 191 F F F 27 540-559 96 96 96 173 173 173 F F F 28 560-579 104 104 104 190 190 190 F F F 29 580-599 98 98 98 175 175 175 F F F Max gen: 109, Max flush: 196 Begin loop 55 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:50.435317782Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1620,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:26:50.478579428Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1620,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:50.478977319Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1620,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:50.511354266Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1620,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:50.511915349Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1620} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"0fd98d0d5d909611f8b408ccda2559d35fe117bb\",\n git_commit_timestamp: \"2024-11-08T20:05:44.000000000Z\",\n git_branch: \"mkeeter/drop-skipped-backpressure-guard\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:50.511951627Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1620} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:50.511965636Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1620} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:50.511979946Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1620} {"msg":"Crucible bdedfca7-8779-4abf-ba17-1f38ae48fff3 has session id: 81629451-e34a-4271-878c-57573b0e2766","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:50.512423614Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1620,"session_id":"81629451-e34a-4271-878c-57573b0e2766"} {"msg":"Upstairs opts: Upstairs UUID: bdedfca7-8779-4abf-ba17-1f38ae48fff3, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:50.512484051Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1620,"session_id":"81629451-e34a-4271-878c-57573b0e2766"} {"msg":"Crucible stats registered with UUID: bdedfca7-8779-4abf-ba17-1f38ae48fff3","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:50.51250616Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1620,"session_id":"81629451-e34a-4271-878c-57573b0e2766"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:50.545733556Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1620,"task":"crutest"} {"msg":"The guest has requested activation with gen:110","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:50.545779254Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1620} {"msg":"bdedfca7-8779-4abf-ba17-1f38ae48fff3 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:50.545810272Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1620,"session_id":"81629451-e34a-4271-878c-57573b0e2766"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:50.545844251Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1620,"client":"0","":"downstairs","session_id":"81629451-e34a-4271-878c-57573b0e2766"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:50.545888478Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1620,"client":"0","":"downstairs","session_id":"81629451-e34a-4271-878c-57573b0e2766"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:50.545922447Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1620,"client":"1","":"downstairs","session_id":"81629451-e34a-4271-878c-57573b0e2766"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:50.545957915Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1620,"client":"1","":"downstairs","session_id":"81629451-e34a-4271-878c-57573b0e2766"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:50.545999233Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1620,"client":"2","":"downstairs","session_id":"81629451-e34a-4271-878c-57573b0e2766"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:50.546033361Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1620,"client":"2","":"downstairs","session_id":"81629451-e34a-4271-878c-57573b0e2766"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:50.54606625Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1620,"":"io task","client":"2","":"downstairs","session_id":"81629451-e34a-4271-878c-57573b0e2766"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:50.546101508Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1620,"":"io task","client":"0","":"downstairs","session_id":"81629451-e34a-4271-878c-57573b0e2766"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:50.546134537Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1620,"":"io task","client":"1","":"downstairs","session_id":"81629451-e34a-4271-878c-57573b0e2766"} {"msg":"ds_connection connected from Ok(127.0.0.1:36502)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:50.546166775Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1620,"":"io task","client":"0","":"downstairs","session_id":"81629451-e34a-4271-878c-57573b0e2766"} {"msg":"ds_connection connected from Ok(127.0.0.1:61457)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:50.546200923Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1620,"":"io task","client":"1","":"downstairs","session_id":"81629451-e34a-4271-878c-57573b0e2766"} {"msg":"ds_connection connected from Ok(127.0.0.1:41318)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:50.546306598Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1620,"":"io task","client":"2","":"downstairs","session_id":"81629451-e34a-4271-878c-57573b0e2766"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:50.546895679Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1620,"client":"2","":"downstairs","session_id":"81629451-e34a-4271-878c-57573b0e2766"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:50.546936327Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1620,"client":"2","":"downstairs","session_id":"81629451-e34a-4271-878c-57573b0e2766"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:50.546965836Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1620,"client":"1","":"downstairs","session_id":"81629451-e34a-4271-878c-57573b0e2766"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:50.546991775Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1620,"client":"1","":"downstairs","session_id":"81629451-e34a-4271-878c-57573b0e2766"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:50.547048462Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1620,"client":"0","":"downstairs","session_id":"81629451-e34a-4271-878c-57573b0e2766"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:50.547065691Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1620,"client":"0","":"downstairs","session_id":"81629451-e34a-4271-878c-57573b0e2766"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:50.547587095Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1620,"client":"1","":"downstairs","session_id":"81629451-e34a-4271-878c-57573b0e2766"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:50.547618994Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1620,"client":"2","":"downstairs","session_id":"81629451-e34a-4271-878c-57573b0e2766"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:50.547917609Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1620,"client":"0","":"downstairs","session_id":"81629451-e34a-4271-878c-57573b0e2766"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:50.548175577Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1620,"client":"1","":"downstairs","session_id":"81629451-e34a-4271-878c-57573b0e2766"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:50.548200616Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1620,"client":"1","":"downstairs","session_id":"81629451-e34a-4271-878c-57573b0e2766"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:50.548214805Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1620,"session_id":"81629451-e34a-4271-878c-57573b0e2766"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:50.548266812Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1620,"client":"2","":"downstairs","session_id":"81629451-e34a-4271-878c-57573b0e2766"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:50.548284291Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1620,"client":"2","":"downstairs","session_id":"81629451-e34a-4271-878c-57573b0e2766"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:50.548297751Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1620,"session_id":"81629451-e34a-4271-878c-57573b0e2766"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:50.548400046Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1620,"client":"0","":"downstairs","session_id":"81629451-e34a-4271-878c-57573b0e2766"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:50.548418965Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1620,"client":"0","":"downstairs","session_id":"81629451-e34a-4271-878c-57573b0e2766"} {"msg":"[0]R flush_numbers[0..12]: [188, 192, 192, 190, 190, 191, 195, 178, 185, 196, 194, 194]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:50.548434204Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1620,"":"downstairs","":"downstairs","session_id":"81629451-e34a-4271-878c-57573b0e2766"} {"msg":"[0]R generation[0..12]: [102, 106, 106, 104, 104, 106, 108, 99, 102, 109, 108, 108]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:50.548448123Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1620,"":"downstairs","":"downstairs","session_id":"81629451-e34a-4271-878c-57573b0e2766"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:50.548480872Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1620,"":"downstairs","":"downstairs","session_id":"81629451-e34a-4271-878c-57573b0e2766"} {"msg":"[1]R flush_numbers[0..12]: [188, 192, 192, 190, 190, 191, 195, 178, 185, 196, 194, 194]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:50.548496201Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1620,"":"downstairs","":"downstairs","session_id":"81629451-e34a-4271-878c-57573b0e2766"} {"msg":"[1]R generation[0..12]: [102, 106, 106, 104, 104, 106, 108, 99, 102, 109, 108, 108]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:50.54851Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1620,"":"downstairs","":"downstairs","session_id":"81629451-e34a-4271-878c-57573b0e2766"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:50.548566507Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1620,"":"downstairs","":"downstairs","session_id":"81629451-e34a-4271-878c-57573b0e2766"} {"msg":"[2]R flush_numbers[0..12]: [188, 192, 192, 190, 190, 191, 195, 178, 185, 196, 194, 194]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:50.548587486Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1620,"":"downstairs","":"downstairs","session_id":"81629451-e34a-4271-878c-57573b0e2766"} {"msg":"[2]R generation[0..12]: [102, 106, 106, 104, 104, 106, 108, 99, 102, 109, 108, 108]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:50.548629795Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1620,"":"downstairs","":"downstairs","session_id":"81629451-e34a-4271-878c-57573b0e2766"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-08T20:26:50.548644164Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1620,"":"downstairs","":"downstairs","session_id":"81629451-e34a-4271-878c-57573b0e2766"} {"msg":"Max found gen is 110","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:50.548665093Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1620,"":"downstairs","":"downstairs","session_id":"81629451-e34a-4271-878c-57573b0e2766"} {"msg":"Generation requested: 110 >= found:110","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:50.548678302Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1620,"":"downstairs","":"downstairs","session_id":"81629451-e34a-4271-878c-57573b0e2766"} {"msg":"Next flush: 197","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:50.548691281Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1620,"":"downstairs","":"downstairs","session_id":"81629451-e34a-4271-878c-57573b0e2766"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:50.548704231Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1620,"":"downstairs","":"downstairs","session_id":"81629451-e34a-4271-878c-57573b0e2766"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:50.54871761Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1620,"session_id":"81629451-e34a-4271-878c-57573b0e2766"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:50.54872975Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1620,"client":"0","":"downstairs","session_id":"81629451-e34a-4271-878c-57573b0e2766"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:50.548743119Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1620,"client":"1","":"downstairs","session_id":"81629451-e34a-4271-878c-57573b0e2766"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:50.548756328Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1620,"client":"2","":"downstairs","session_id":"81629451-e34a-4271-878c-57573b0e2766"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:50.548769518Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1620,"session_id":"81629451-e34a-4271-878c-57573b0e2766"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:50.548781807Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1620,"session_id":"81629451-e34a-4271-878c-57573b0e2766"} {"msg":"bdedfca7-8779-4abf-ba17-1f38ae48fff3 is now active with session: 81629451-e34a-4271-878c-57573b0e2766","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:50.548794626Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1620,"session_id":"81629451-e34a-4271-878c-57573b0e2766"} {"msg":"Set Active after no reconciliation","v"Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 :0,"name":"crucible","level":30,"time":"2024-11-08T20:26:50.548813495Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1620,"session_id":"81629451-e34a-4271-878c-57573b0e2766"} {"msg":"The guest has finished waiting for activation with:110","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:50.548826175Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1620} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 519 len 4096 data: 10 02/30 Read block 432 len 32768 03/30 Read block 318 len 8192 04/30 Read block 537 len 28672 05/30 Read block 404 len 36864 06/30 Write block 67 len 4096 data: 6 07/30 Read block 50 len 24576 08/30 Read block 298 len 28672 09/30 Flush 10/30 Write block 402 len 8192 data: 8 8 11/30 Read block 228 len 32768 12/30 Read block 474 len 20480 13/30 Write block 330 len 36864 data: 5 5 6 8 9 10 9 10 12 14/30 Write block 574 len 4096 data: 11 15/30 Read block 551 len 32768 16/30 Write block 83 len 40960 data: 8 7 9 9 9 10 8 6 6 7 17/30 Read block 221 len 4096 18/30 Read block 351 len 4096 19/30 Read block 226 len 4096 20/30 Write block 328 len 16384 data: 6 6 6 6 21/30 Write block 110 len 32768 data: 9 6 5 6 7 6 7 7 22/30 Read block 546 len 12288 23/30 Read block 112 len 40960 24/30 Read block 478 len 20480 25/30 Read block 338 len 24576 26/30 Read block 212 len 32768 27/30 Read block 113 len 28672 28/30 Write block 244 len 28672 data: 9 8 8 12 12 13 15 29/30 Read block 316 len 20480 30/30 Read block 529 len 36864 ---------------------------------------------------------------- Crucible gen:110 GIO:true work queues: Upstairs:0 downstairs:21 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1009 Acked Write 2 Done Done Done false 1010 Acked Read 8 Done Done Done false 1011 Acked Read 5 Done Done Done false 1012 Acked Write 9 Done Done Done false 1013 Acked Write 1 Done Done Done false 1014 Acked Read 8 Done Done Done false 1015 Acked Write 10 Done Done Done false 1016 Acked Read 1 Done Done Done false 1017 Acked Read 1 Done Done Done false 1018 Acked Read 1 Done Done Done false 1019 Acked Write 4 Done Done Done false 1020 Acked Write 8 Done Done Done false 1021 Acked Read 3 Done Done Done false 1022 Acked Read 10 Done Done Done false 1023 Acked Read 5 Done Done Done false 1024 Acked Read 6 Done Done Done false 1025 Acked Read 8 Done Done Done false 1026 Acked Read 7 Done Done Done false 1027 Acked Write 7 Done Done Done false 1028 Acked Read 5 Done Done Done false 1029 Acked Read 9 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 21 21 21 63 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1008 1008 1008 Downstairs last five completed: 1008 1007 1006 1005 1004 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:26:50.570340553Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1620} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 102 102 102 188 188 188 F F F 1 020-039 106 106 106 192 192 192 F F F 2 040-059 106 106 106 192 192 192 F F F 3 060-079 110 110 110 197 197 197 F F F 4 080-099 104 104 104 190 190 190 T T T 5 100-119 106 106 106 191 191 191 T T T 6 120-139 108 108 108 195 195 195 F F F 7 140-159 99 99 99 178 178 178 F F F 8 160-179 102 102 102 185 185 185 F F F 9 180-199 109 109 109 196 196 196 F F F 10 200-219 108 108 108 194 194 194 F F F 11 220-239 108 108 108 194 194 194 F F F 12 240-259 102 102 102 183 183 183 T T T 13 260-279 108 108 108 194 194 194 F F F 14 280-299 108 108 108 194 194 194 F F F 15 300-319 108 108 108 194 194 194 F F F 16 320-339 108 108 108 194 194 194 T T T 17 340-359 108 108 108 194 194 194 F F F 18 360-379 108 108 108 194 194 194 F F F 19 380-399 107 107 107 193 193 193 F F F 20 400-419 109 109 109 196 196 196 T T T 21 420-439 107 107 107 193 193 193 F F F 22 440-459 104 104 104 190 190 190 F F F 23 460-479 109 109 109 196 196 196 F F F 24 480-499 109 109 109 196 196 196 F F F 25 500-519 110 110 110 197 197 197 F F F 26 520-539 106 106 106 191 191 191 F F F 27 540-559 96 96 96 173 173 173 F F F 28 560-579 104 104 104 190 190 190 T T T 29 580-599 98 98 98 175 175 175 F F F Max gen: 110, Max flush: 197 A Difference in extent metadata is expected here On loop 55 Verifying data now No change after verify Loop: 55 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 102 102 102 188 188 188 F F F 1 020-039 106 106 106 192 192 192 F F F 2 040-059 106 106 106 192 192 192 F F F 3 060-079 110 110 110 197 197 197 F F F 4 080-099 111 111 111 198 198 198 F F F 5 100-119 111 111 111 198 198 198 F F F 6 120-139 108 108 108 195 195 195 F F F 7 140-159 99 99 99 178 178 178 F F F 8 160-179 102 102 102 185 185 185 F F F 9 180-199 109 109 109 196 196 196 F F F 10 200-219 108 108 108 194 194 194 F F F 11 220-239 108 108 108 194 194 194 F F F 12 240-259 111 111 111 198 198 198 F F F 13 260-279 108 108 108 194 194 194 F F F 14 280-299 108 108 108 194 194 194 F F F 15 300-319 108 108 108 194 194 194 F F F 16 320-339 111 111 111 198 198 198 F F F 17 340-359 108 108 108 194 194 194 F F F 18 360-379 108 108 108 194 194 194 F F F 19 380-399 107 107 107 193 193 193 F F F 20 400-419 111 111 111 198 198 198 F F F 21 420-439 107 107 107 193 193 193 F F F 22 440-459 104 104 104 190 190 190 F F F 23 460-479 109 109 109 196 196 196 F F F 24 480-499 109 109 109 196 196 196 F F F 25 500-519 110 110 110 197 197 197 F F F 26 520-539 106 106 106 191 191 191 F F F 27 540-559 96 96 96 173 173 173 F F F 28 560-579 111 111 111 198 198 198 F F F 29 580-599 98 98 98 175 175 175 F F F Max gen: 111, Max flush: 198 Begin loop 56 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:54.935200665Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1628,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:54.979233753Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1628,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:54.979865792Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1628} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"0fd98d0d5d909611f8b408ccda2559d35fe117bb\",\n git_commit_timestamp: \"2024-11-08T20:05:44.000000000Z\",\n git_branch: \"mkeeter/drop-skipped-backpressure-guard\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:54.97990652Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1628} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:54.979921209Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1628} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:54.979935319Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1628} {"msg":"Crucible de4a5152-9867-4294-9f0e-47afc883643c has session id: 7b581fa6-afdc-41fc-808d-393961021f4d","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:54.980381487Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1628,"session_id":"7b581fa6-afdc-41fc-808d-393961021f4d"} {"msg":"Upstairs opts: Upstairs UUID: de4a5152-9867-4294-9f0e-47afc883643c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:54.980428365Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1628,"session_id":"7b581fa6-afdc-41fc-808d-393961021f4d"} {"msg":"Crucible stats registered with UUID: de4a5152-9867-4294-9f0e-47afc883643c","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:54.980453854Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1628,"session_id":"7b581fa6-afdc-41fc-808d-393961021f4d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:55.011772473Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1628,"task":"crutest"} {"msg":"The guest has requested activation with gen:112","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:55.011806061Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1628} {"msg":"de4a5152-9867-4294-9f0e-47afc883643c active request set","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:55.01184066Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1628,"session_id":"7b581fa6-afdc-41fc-808d-393961021f4d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:55.011858388Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1628,"client":"0","":"downstairs","session_id":"7b581fa6-afdc-41fc-808d-393961021f4d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:55.011886707Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1628,"client":"0","":"downstairs","session_id":"7b581fa6-afdc-41fc-808d-393961021f4d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:55.011904646Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1628,"client":"1","":"downstairs","session_id":"7b581fa6-afdc-41fc-808d-393961021f4d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:55.011921505Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1628,"client":"1","":"downstairs","session_id":"7b581fa6-afdc-41fc-808d-393961021f4d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:55.011937865Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1628,"client":"2","":"downstairs","session_id":"7b581fa6-afdc-41fc-808d-393961021f4d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:55.011997522Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1628,"client":"2","":"downstairs","session_id":"7b581fa6-afdc-41fc-808d-393961021f4d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:55.01202711Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1628,"":"io task","client":"2","":"downstairs","session_id":"7b581fa6-afdc-41fc-808d-393961021f4d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:55.01204395Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1628,"":"io task","client":"0","":"downstairs","session_id":"7b581fa6-afdc-41fc-808d-393961021f4d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:55.012058599Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1628,"":"io task","client":"1","":"downstairs","session_id":"7b581fa6-afdc-41fc-808d-393961021f4d"} {"msg":"ds_connection connected from Ok(127.0.0.1:61975)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:55.012199352Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1628,"":"io task","client":"2","":"downstairs","session_id":"7b581fa6-afdc-41fc-808d-393961021f4d"} {"msg":"ds_connection connected from Ok(127.0.0.1:60579)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:55.012250779Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1628,"":"io task","client":"0","":"downstairs","session_id":"7b581fa6-afdc-41fc-808d-393961021f4d"} {"msg":"ds_connection connected from Ok(127.0.0.1:63777)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:55.012373294Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1628,"":"io task","client":"1","":"downstairs","session_id":"7b581fa6-afdc-41fc-808d-393961021f4d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:55.012934606Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1628,"client":"0","":"downstairs","session_id":"7b581fa6-afdc-41fc-808d-393961021f4d"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:55.012967334Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1628,"client":"0","":"downstairs","session_id":"7b581fa6-afdc-41fc-808d-393961021f4d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:55.012995093Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1628,"client":"1","":"downstairs","session_id":"7b581fa6-afdc-41fc-808d-393961021f4d"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:55.013023272Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1628,"client":"1","":"downstairs","session_id":"7b581fa6-afdc-41fc-808d-393961021f4d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:55.013076269Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1628,"client":"2","":"downstairs","session_id":"7b581fa6-afdc-41fc-808d-393961021f4d"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:55.013110887Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1628,"client":"2","":"downstairs","session_id":"7b581fa6-afdc-41fc-808d-393961021f4d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:55.013625812Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1628,"client":"1","":"downstairs","session_id":"7b581fa6-afdc-41fc-808d-393961021f4d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:55.01366889Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1628,"client":"0","":"downstairs","session_id":"7b581fa6-afdc-41fc-808d-393961021f4d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:55.013939367Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1628,"client":"2","":"downstairs","session_id":"7b581fa6-afdc-41fc-808d-393961021f4d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:55.014174455Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1628,"client":"1","":"downstairs","session_id":"7b581fa6-afdc-41fc-808d-393961021f4d"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:55.014214404Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1628,"client":"1","":"downstairs","session_id":"7b581fa6-afdc-41fc-808d-393961021f4d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:55.014241152Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1628,"session_id":"7b581fa6-afdc-41fc-808d-393961021f4d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:55.014263601Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1628,"client":"0","":"downstairs","session_id":"7b581fa6-afdc-41fc-808d-393961021f4d"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:55.01428801Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1628,"client":"0","":"downstairs","session_id":"7b581fa6-afdc-41fc-808d-393961021f4d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:55.014313059Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1628,"session_id":"7b581fa6-afdc-41fc-808d-393961021f4d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:55.01449977Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1628,"client":"2","":"downstairs","session_id":"7b581fa6-afdc-41fc-808d-393961021f4d"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:55.014530548Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1628,"client":"2","":"downstairs","session_id":"7b581fa6-afdc-41fc-808d-393961021f4d"} {"msg":"[0]R flush_numbers[0..12]: [188, 192, 192, 197, 198, 198, 195, 178, 185, 196, 194, 194]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:55.014547197Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1628,"":"downstairs","":"downstairs","session_id":"7b581fa6-afdc-41fc-808d-393961021f4d"} {"msg":"[0]R generation[0..12]: [102, 106, 106, 110, 111, 111, 108, 99, 102, 109, 108, 108]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:55.014562496Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1628,"":"downstairs","":"downstairs","session_id":"7b581fa6-afdc-41fc-808d-393961021f4d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:55.014577056Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1628,"":"downstairs","":"downstairs","session_id":"7b581fa6-afdc-41fc-808d-393961021f4d"} {"msg":"[1]R flush_numbers[0..12]: [188, 192, 192, 197, 198, 198, 195, 178, 185, 196, 194, 194]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:55.014590875Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1628,"":"downstairs","":"downstairs","session_id":"7b581fa6-afdc-41fc-808d-393961021f4d"} {"msg":"[1]R generation[0..12]: [102, 106, 106, 110, 111, 111, 108, 99, 102, 109, 108, 108]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:55.014604794Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1628,"":"downstairs","":"downstairs","session_id":"7b581fa6-afdc-41fc-808d-393961021f4d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:55.014618684Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1628,"":"downstairs","":"downstairs","session_id":"7b581fa6-afdc-41fc-808d-393961021f4d"} {"msg":"[2]R flush_numbers[0..12]: [188, 192, 192, 197, 198, 198, 195, 178, 185, 196, 194, 194]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:55.014635363Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1628,"":"downstairs","":"downstairs","session_id":"7b581fa6-afdc-41fc-808d-393961021f4d"} {"msg":"[2]R generation[0..12]: [102, 106, 106, 110, 111, 111, 108, 99, 102, 109, 108, 108]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:55.014649712Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1628,"":"downstairs","":"downstairs","session_id":"7b581fa6-afdc-41fc-808d-393961021f4d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:55.014664082Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1628,"":"downstairs","":"downstairs","session_id":"7b581fa6-afdc-41fc-808d-393961021f4d"} {"msg":"Max found gen is 112","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:55.014677301Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1628,"":"downstairs","":"downstairs","session_id":"7b581fa6-afdc-41fc-808d-393961021f4d"} {"msg":"Generation requested: 112 >= found:112","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-08T20:26:55.01469074Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1628,"":"downstairs","":"downstairs","session_id":"7b581fa6-afdc-41fc-808d-393961021f4d"} {"msg":"Next flush: 199","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:55.014711619Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1628,"":"downstairs","":"downstairs","session_id":"7b581fa6-afdc-41fc-808d-393961021f4d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:55.014724799Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1628,"":"downstairs","":"downstairs","session_id":"7b581fa6-afdc-41fc-808d-393961021f4d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:55.014738228Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1628,"session_id":"7b581fa6-afdc-41fc-808d-393961021f4d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:55.014750297Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1628,"client":"0","":"downstairs","session_id":"7b581fa6-afdc-41fc-808d-393961021f4d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:55.014763737Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1628,"client":"1","":"downstairs","session_id":"7b581fa6-afdc-41fc-808d-393961021f4d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:55.014777216Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1628,"client":"2","":"downstairs","session_id":"7b581fa6-afdc-41fc-808d-393961021f4d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:55.014790655Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1628,"session_id":"7b581fa6-afdc-41fc-808d-393961021f4d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:55.014803165Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1628,"session_id":"7b581fa6-afdc-41fc-808d-393961021f4d"} {"msg":"de4a5152-9867-4294-9f0e-47afc883643c is now active with session: 7b581fa6-afdc-41fc-808d-393961021f4d","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:55.014815994Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1628,"session_id":"7b581fa6-afdc-41fc-808d-393961021f4d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:55.014828074Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1628,"session_id":"7b581fa6-afdc-41fc-808d-393961021f4d"} {"msg":"The guest has finished waiting for activation with:112","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:55.014841973Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1628} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 232 len 4096 data: 7 02/30 Write block 482 len 32768 data: 11 11 10 8 8 7 9 11 03/30 Write block 5 len 28672 data: 7 7 7 8 8 8 10 04/30 Write block 176 len 4096 data: 4 05/30 Read block 422 len 32768 06/30 Write block 542 len 28672 data: 11 11 13 9 11 8 10 07/30 Read block 325 len 28672 08/30 Read block 557 len 12288 09/30 Read block 60 len 12288 10/30 Read block 363 len 36864 11/30 Read block 159 len 4096 12/30 Flush 13/30 Write block 348 len 36864 data: 8 11 10 10 11 15 13 14 14 14/30 Read block 226 len 20480 15/30 Read block 155 len 8192 16/30 Flush 17/30 Write block 336 len 12288 data: 10 11 13 18/30 Write block 299 len 4096 data: 12 19/30 Read block 445 len 32768 20/30 Write block 545 len 24576 data: 10 12 9 11 8 7 21/30 Write block 370 len 24576 data: 8 9 11 8 8 7 22/30 Read block 78 len 16384 23/30 Write block 351 len 20480 data: 11 12 16 14 15 24/30 Read block 83 len 32768 25/30 Write block 93 len 4096 data: 7 26/30 Read block 268 len 36864 27/30 Read block 481 len 20480 28/30 Read block 151 len 40960 29/30 Read block 80 len 16384 30/30 Read block 371 len 20480 ---------------------------------------------------------------- Crucible gen:112 GIO:true work queues: Upstairs:0 downstairs:14 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1016 Acked Write 3 Done Done Done false 1017 Acked Write 1 Done Done Done false 1018 Acked Read 8 Done Done Done false 1019 Acked Write 6 Done Done Done false 1020 Acked Write 6 Done Done Done false 1021 Acked Read 4 Done Done Done false 1022 Acked Write 5 Done Done Done false 1023 Acked Read 8 Done Done Done false 1024 Acked Write 1 Done Done Done false 1025 Acked Read 9 Done Done Done false 1026 Acked Read 5 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Read 4 Done Done Done false 1029 Acked Read 5 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 1 2 Done 13 14 13 40 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1015 1015 1015 Downstairs last five completed: 1015 1014 1013 1012 1011 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:26:55.03923655Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1628} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 112 112 112 199 199 199 F F F 1 020-039 106 106 106 192 192 192 F F F 2 040-059 106 106 106 192 192 192 F F F 3 060-079 110 110 110 197 197 197 F F F 4 080-099 111 111 111 198 198 198 T T T 5 100-119 111 111 111 198 198 198 F F F 6 120-139 108 108 108 195 195 195 F F F 7 140-159 99 99 99 178 178 178 F F F 8 160-179 112 112 112 199 199 199 F F F 9 180-199 109 109 109 196 196 196 F F F 10 200-219 108 108 108 194 194 194 F F F 11 220-239 112 112 112 199 199 199 F F F 12 240-259 111 111 111 198 198 198 F F F 13 260-279 108 108 108 194 194 194 F F F 14 280-299 108 108 108 194 194 194 T T T 15 300-319 108 108 108 194 194 194 F F F 16 320-339 111 111 111 198 198 198 T T T 17 340-359 112 112 112 200 200 200 T T T 18 360-379 108 108 108 194 194 194 T T T 19 380-399 107 107 107 193 193 193 F F F 20 400-419 111 111 111 198 198 198 F F F 21 420-439 107 107 107 193 193 193 F F F 22 440-459 104 104 104 190 190 190 F F F 23 460-479 109 109 109 196 196 196 F F F 24 480-499 112 112 112 199 199 199 F F F 25 500-519 110 110 110 197 197 197 F F F 26 520-539 106 106 106 191 191 191 F F F 27 540-559 112 112 112 199 199 199 T T T 28 560-579 111 111 111 198 198 198 F F F 29 580-599 98 98 98 175 175 175 F F F Max gen: 112, Max flush: 200 A Difference in extent metadata is expected here On loop 56 Verifying data now No change after verify Loop: 56 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 112 112 112 199 199 199 F F F 1 020-039 106 106 106 192 192 192 F F F 2 040-059 106 106 106 192 192 192 F F F 3 060-079 110 110 110 197 197 197 F F F 4 080-099 113 113 113 201 201 201 F F F 5 100-119 111 111 111 198 198 198 F F F 6 120-139 108 108 108 195 195 195 F F F 7 140-159 99 99 99 178 178 178 F F F 8 160-179 112 112 112 199 199 199 F F F 9 180-199 109 109 109 196 196 196 F F F 10 200-219 108 108 108 194 194 194 F F F 11 220-239 112 112 112 199 199 199 F F F 12 240-259 111 111 111 198 198 198 F F F 13 260-279 108 108 108 194 194 194 F F F 14 280-299 113 113 113 201 201 201 F F F 15 300-319 108 108 108 194 194 194 F F F 16 320-339 113 113 113 201 201 201 F F F 17 340-359 113 113 113 201 201 201 F F F 18 360-379 113 113 113 201 201 201 F F F 19 380-399 107 107 107 193 193 193 F F F 20 400-419 111 111 111 198 198 198 F F F 21 420-439 107 107 107 193 193 193 F F F 22 440-459 104 104 104 190 190 190 F F F 23 460-479 109 109 109 196 196 196 F F F 24 480-499 112 112 112 199 199 199 F F F 25 500-519 110 110 110 197 197 197 F F F 26 520-539 106 106 106 191 191 191 F F F 27 540-559 113 113 113 201 201 201 F F F 28 560-579 111 111 111 198 198 198 F F F 29 580-599 98 98 98 175 175 175 F F F Max gen: 113, Max flush: 201 Begin loop 57 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:59.379064936Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1636,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:59.423541902Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1636,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:59.42419078Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1636} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"0fd98d0d5d909611f8b408ccda2559d35fe117bb\",\n git_commit_timestamp: \"2024-11-08T20:05:44.000000000Z\",\n git_branch: \"mkeeter/drop-skipped-backpressure-guard\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:59.424228218Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1636} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:59.424246757Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1636} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:59.424261927Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1636} {"msg":"Crucible f92744f4-9ca9-455b-a54e-d481273940cd has session id: a0d9f403-1a16-408b-b631-5b09c9ef41aa","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:59.424690416Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1636,"session_id":"a0d9f403-1a16-408b-b631-5b09c9ef41aa"} {"msg":"Upstairs opts: Upstairs UUID: f92744f4-9ca9-455b-a54e-d481273940cd, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:59.424736533Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1636,"session_id":"a0d9f403-1a16-408b-b631-5b09c9ef41aa"} {"msg":"Crucible stats registered with UUID: f92744f4-9ca9-455b-a54e-d481273940cd","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:59.424764142Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1636,"session_id":"a0d9f403-1a16-408b-b631-5b09c9ef41aa"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:59.456619385Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1636,"task":"crutest"} {"msg":"The guest has requested activation with gen:114","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:59.456651534Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1636} {"msg":"f92744f4-9ca9-455b-a54e-d481273940cd active request set","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:59.45673532Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1636,"session_id":"a0d9f403-1a16-408b-b631-5b09c9ef41aa"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:59.456758268Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1636,"client":"0","":"downstairs","session_id":"a0d9f403-1a16-408b-b631-5b09c9ef41aa"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:59.456785687Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1636,"client":"0","":"downstairs","session_id":"a0d9f403-1a16-408b-b631-5b09c9ef41aa"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:59.456803686Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1636,"client":"1","":"downstairs","session_id":"a0d9f403-1a16-408b-b631-5b09c9ef41aa"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:59.456821065Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1636,"client":"1","":"downstairs","session_id":"a0d9f403-1a16-408b-b631-5b09c9ef41aa"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:59.456837915Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1636,"client":"2","":"downstairs","session_id":"a0d9f403-1a16-408b-b631-5b09c9ef41aa"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:59.456854834Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1636,"client":"2","":"downstairs","session_id":"a0d9f403-1a16-408b-b631-5b09c9ef41aa"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:59.456881142Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1636,"":"io task","client":"2","":"downstairs","session_id":"a0d9f403-1a16-408b-b631-5b09c9ef41aa"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:59.45691894Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1636,"":"io task","client":"0","":"downstairs","session_id":"a0d9f403-1a16-408b-b631-5b09c9ef41aa"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:59.456954549Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1636,"":"io task","client":"1","":"downstairs","session_id":"a0d9f403-1a16-408b-b631-5b09c9ef41aa"} {"msg":"ds_connection connected from Ok(127.0.0.1:52115)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:59.456989137Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1636,"":"io task","client":"0","":"downstairs","session_id":"a0d9f403-1a16-408b-b631-5b09c9ef41aa"} {"msg":"ds_connection connected from Ok(127.0.0.1:43937)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:59.457023775Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1636,"":"io task","client":"2","":"downstairs","session_id":"a0d9f403-1a16-408b-b631-5b09c9ef41aa"} {"msg":"ds_connection connected from Ok(127.0.0.1:44479)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:59.457059214Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1636,"":"io task","client":"1","":"downstairs","session_id":"a0d9f403-1a16-408b-b631-5b09c9ef41aa"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:59.457818237Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1636,"client":"2","":"downstairs","session_id":"a0d9f403-1a16-408b-b631-5b09c9ef41aa"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:59.457858365Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1636,"client":"2","":"downstairs","session_id":"a0d9f403-1a16-408b-b631-5b09c9ef41aa"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:59.457888503Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1636,"client":"0","":"downstairs","session_id":"a0d9f403-1a16-408b-b631-5b09c9ef41aa"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:59.457918132Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1636,"client":"0","":"downstairs","session_id":"a0d9f403-1a16-408b-b631-5b09c9ef41aa"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:59.457944461Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1636,"client":"1","":"downstairs","session_id":"a0d9f403-1a16-408b-b631-5b09c9ef41aa"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:59.457973799Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1636,"client":"1","":"downstairs","session_id":"a0d9f403-1a16-408b-b631-5b09c9ef41aa"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:59.458494064Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1636,"client":"1","":"downstairs","session_id":"a0d9f403-1a16-408b-b631-5b09c9ef41aa"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:59.458526692Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1636,"client":"2","":"downstairs","session_id":"a0d9f403-1a16-408b-b631-5b09c9ef41aa"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:59.45855584Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1636,"client":"0","":"downstairs","session_id":"a0d9f403-1a16-408b-b631-5b09c9ef41aa"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:59.459149162Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1636,"client":"1","":"downstairs","session_id":"a0d9f403-1a16-408b-b631-5b09c9ef41aa"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:59.4591853Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1636,"client":"1","":"downstairs","session_id":"a0d9f403-1a16-408b-b631-5b09c9ef41aa"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:59.459201439Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1636,"session_id":"a0d9f403-1a16-408b-b631-5b09c9ef41aa"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:59.459214478Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1636,"client":"2","":"downstairs","session_id":"a0d9f403-1a16-408b-b631-5b09c9ef41aa"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:59.459227968Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1636,"client":"2","":"downstairs","session_id":"a0d9f403-1a16-408b-b631-5b09c9ef41aa"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:59.459241107Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1636,"session_id":"a0d9f403-1a16-408b-b631-5b09c9ef41aa"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:59.459253346Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1636,"client":"0","":"downstairs","session_id":"a0d9f403-1a16-408b-b631-5b09c9ef41aa"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:59.459266406Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1636,"client":"0","":"downstairs","session_id":"a0d9f403-1a16-408b-b631-5b09c9ef41aa"} {"msg":"[0]R flush_numbers[0..12]: [199, 192, 192, 197, 201, 198, 195, 178, 199, 196, 194, 199]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:59.459280105Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1636,"":"downstairs","":"downstairs","session_id":"a0d9f403-1a16-408b-b631-5b09c9ef41aa"} {"msg":"[0]R generation[0..12]: [112, 106, 106, 110, 113, 111, 108, 99, 112, 109, 108, 112]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:59.459293815Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1636,"":"downstairs","":"downstairs","session_id":"a0d9f403-1a16-408b-b631-5b09c9ef41aa"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:59.459307814Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1636,"":"downstairs","":"downstairs","session_id":"a0d9f403-1a16-408b-b631-5b09c9ef41aa"} {"msg":"[1]R flush_numbers[0..12]: [199, 192, 192, 197, 201, 198, 195, 178, 199, 196, 194, 199]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:59.459321513Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1636,"":"downstairs","":"downstairs","session_id":"a0d9f403-1a16-408b-b631-5b09c9ef41aa"} {"msg":"[1]R generation[0..12]: [112, 106, 106, 110, 113, 111, 108, 99, 112, 109, 108, 112]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:59.459335242Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1636,"":"downstairs","":"downstairs","session_id":"a0d9f403-1a16-408b-b631-5b09c9ef41aa"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:59.459349192Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1636,"":"downstairs","":"downstairs","session_id":"a0d9f403-1a16-408b-b631-5b09c9ef41aa"} {"msg":"[2]R flush_numbers[0..12]: [199, 192, 192, 197, 201, 198, 195, 178, 199, 196, 194, 199]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:59.459363021Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1636,"":"downstairs","":"downstairs","session_id":"a0d9f403-1a16-408b-b631-5b09c9ef41aa"} {"msg":"[2]R generation[0..12]: [112, 106, 106, 110, 113, 111, 108, 99, 112, 109, 108, 112]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:59.45937692Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1636,"":"downstairs","":"downstairs","session_id":"a0d9f403-1a16-408b-b631-5b09c9ef41aa"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:59.45939334Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1636,"":"downstairs","":"downstairs","session_id":"a0d9f403-1a16-408b-b631-5b09c9ef41aa"} {"msg":"Max found gen is 114","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:59.459406539Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1636,"":"downstairs","":"downstairs","session_id":"a0d9f403-1a16-408b-b631-5b09c9ef41aa"} {"msg":"Generation requested: 114 >= found:114","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:59.459419658Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1636,"":"downstairs","":"downstairs","session_id":"a0d9f403-1a16-408b-b631-5b09c9ef41aa"} {"msg":"Next flush: 202","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-08T20:26:59.459432768Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1636,"":"downstairs","":"downstairs","session_id":"a0d9f403-1a16-408b-b631-5b09c9ef41aa"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:59.459456216Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1636,"":"downstairs","":"downstairs","session_id":"a0d9f403-1a16-408b-b631-5b09c9ef41aa"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:59.459469346Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1636,"session_id":"a0d9f403-1a16-408b-b631-5b09c9ef41aa"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:59.459481325Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1636,"client":"0","":"downstairs","session_id":"a0d9f403-1a16-408b-b631-5b09c9ef41aa"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:59.459494515Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1636,"client":"1","":"downstairs","session_id":"a0d9f403-1a16-408b-b631-5b09c9ef41aa"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:59.459507654Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1636,"client":"2","":"downstairs","session_id":"a0d9f403-1a16-408b-b631-5b09c9ef41aa"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:59.459520844Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1636,"session_id":"a0d9f403-1a16-408b-b631-5b09c9ef41aa"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:59.459533253Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1636,"session_id":"a0d9f403-1a16-408b-b631-5b09c9ef41aa"} {"msg":"f92744f4-9ca9-455b-a54e-d481273940cd is now active with session: a0d9f403-1a16-408b-b631-5b09c9ef41aa","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:59.459546142Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1636,"session_id":"a0d9f403-1a16-408b-b631-5b09c9ef41aa"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:59.459558072Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1636,"session_id":"a0d9f403-1a16-408b-b631-5b09c9ef41aa"} {"msg":"The guest has finished waiting for activation with:114","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:26:59.459570221Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1636} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 561 len 8192 data: 6 6 02/30 Write block 469 len 20480 data: 12 12 14 14 13 03/30 Read block 234 len 20480 04/30 Write block 568 len 28672 data: 9 9 7 10 12 13 12 05/30 Write block 207 len 8192 data: 9 7 06/30 Write block 435 len 36864 data: 13 15 14 14 13 12 14 15 10 07/30 Write block 311 len 24576 data: 9 9 11 9 12 10 08/30 Read block 188 len 36864 09/30 Read block 85 len 36864 10/30 Write block 477 len 8192 data: 13 14 11/30 Read block 66 len 4096 12/30 Flush 13/30 Write block 112 len 40960 data: 6 7 8 7 8 8 7 9 9 10 14/30 Read block 7 len 36864 15/30 Flush 16/30 Write block 456 len 28672 data: 7 8 6 7 6 6 5 17/30 Write block 168 len 8192 data: 10 10 18/30 Read block 445 len 40960 19/30 Write block 477 len 12288 data: 14 15 14 20/30 Read block 266 len 28672 21/30 Read block 490 len 40960 22/30 Read block 399 len 36864 23/30 Flush 24/30 Write block 181 len 8192 data: 6 7 25/30 Write block 595 len 8192 data: 7 6 26/30 Write block 531 len 24576 data: 10 8 9 9 10 10 27/30 Read block 286 len 8192 28/30 Write block 115 len 16384 data: 8 9 9 8 29/30 Write block 504 len 20480 data: 9 8 7 6 9 30/30 Read block 328 len 32768 ---------------------------------------------------------------- Crucible gen:114 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 2 Done Done Done false 1024 Acked Write 2 Done Done Done false 1025 Acked Write 6 Done Done Done false 1026 Acked Read 2 Done Done Done false 1027 Acked Write 4 Sent Done Sent false 1028 Acked Write 5 Sent Done Sent false 1029 Acked Read 8 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 0 3 6 Done 4 7 4 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:26:59.485459936Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1636} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 112 112 112 199 199 199 F F F 1 020-039 106 106 106 192 192 192 F F F 2 040-059 106 106 106 192 192 192 F F F 3 060-079 110 110 110 197 197 197 F F F 4 080-099 113 113 113 201 201 201 F F F 5 100-119 114 114 114 203 203 203 T T T 6 120-139 114 114 114 203 203 203 F F F 7 140-159 99 99 99 178 178 178 F F F 8 160-179 114 114 114 204 204 204 F F F 9 180-199 109 109 109 196 196 196 T T T 10 200-219 114 114 114 202 202 202 F F F 11 220-239 112 112 112 199 199 199 F F F 12 240-259 111 111 111 198 198 198 F F F 13 260-279 108 108 108 194 194 194 F F F 14 280-299 113 113 113 201 201 201 F F F 15 300-319 114 114 114 202 202 202 F F F 16 320-339 113 113 113 201 201 201 F F F 17 340-359 113 113 113 201 201 201 F F F 18 360-379 113 113 113 201 201 201 F F F 19 380-399 107 107 107 193 193 193 F F F 20 400-419 111 111 111 198 198 198 F F F 21 420-439 114 114 114 202 202 202 F F F 22 440-459 114 114 114 204 204 204 F F F 23 460-479 114 114 114 204 204 204 F F F 24 480-499 112 112 112 199 199 199 F F F 25 500-519 110 110 110 197 197 197 T T T 26 520-539 106 106 106 191 191 191 T T T 27 540-559 113 113 113 201 201 201 F F F 28 560-579 114 114 114 202 202 202 F F F 29 580-599 98 98 98 175 175 175 T T T Max gen: 114, Max flush: 204 A Difference in extent metadata is expected here On loop 57 Verifying data now No change after verify Loop: 57 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 112 112 112 199 199 199 F F F 1 020-039 106 106 106 192 192 192 F F F 2 040-059 106 106 106 192 192 192 F F F 3 060-079 110 110 110 197 197 197 F F F 4 080-099 113 113 113 201 201 201 F F F 5 100-119 115 115 115 205 205 205 F F F 6 120-139 114 114 114 203 203 203 F F F 7 140-159 99 99 99 178 178 178 F F F 8 160-179 114 114 114 204 204 204 F F F 9 180-199 115 115 115 205 205 205 F F F 10 200-219 114 114 114 202 202 202 F F F 11 220-239 112 112 112 199 199 199 F F F 12 240-259 111 111 111 198 198 198 F F F 13 260-279 108 108 108 194 194 194 F F F 14 280-299 113 113 113 201 201 201 F F F 15 300-319 114 114 114 202 202 202 F F F 16 320-339 113 113 113 201 201 201 F F F 17 340-359 113 113 113 201 201 201 F F F 18 360-379 113 113 113 201 201 201 F F F 19 380-399 107 107 107 193 193 193 F F F 20 400-419 111 111 111 198 198 198 F F F 21 420-439 114 114 114 202 202 202 F F F 22 440-459 114 114 114 204 204 204 F F F 23 460-479 114 114 114 204 204 204 F F F 24 480-499 112 112 112 199 199 199 F F F 25 500-519 115 115 115 205 205 205 F F F 26 520-539 115 115 115 205 205 205 F F F 27 540-559 113 113 113 201 201 201 F F F 28 560-579 114 114 114 202 202 202 F F F 29 580-599 115 115 115 205 205 205 F F F Max gen: 115, Max flush: 205 Begin loop 58 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:03.803610569Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1644,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:27:03.847073915Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1644,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:03.847466676Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1644,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:03.879893141Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1644,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:03.880444134Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1644} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"0fd98d0d5d909611f8b408ccda2559d35fe117bb\",\n git_commit_timestamp: \"2024-11-08T20:05:44.000000000Z\",\n git_branch: \"mkeeter/drop-skipped-backpressure-guard\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:03.880488322Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1644} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:03.880505871Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1644} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:03.8805241Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1644} {"msg":"Crucible b8fb576b-59cf-4185-9f93-c1d36889c880 has session id: 54c4ad36-c850-43f8-88c5-c144487de07a","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:03.880911541Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1644,"session_id":"54c4ad36-c850-43f8-88c5-c144487de07a"} {"msg":"Upstairs opts: Upstairs UUID: b8fb576b-59cf-4185-9f93-c1d36889c880, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:03.880956689Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1644,"session_id":"54c4ad36-c850-43f8-88c5-c144487de07a"} {"msg":"Crucible stats registered with UUID: b8fb576b-59cf-4185-9f93-c1d36889c880","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:03.880981668Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1644,"session_id":"54c4ad36-c850-43f8-88c5-c144487de07a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:03.912051639Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1644,"task":"crutest"} {"msg":"The guest has requested activation with gen:116","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:03.912084578Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1644} {"msg":"b8fb576b-59cf-4185-9f93-c1d36889c880 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:03.912200102Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1644,"session_id":"54c4ad36-c850-43f8-88c5-c144487de07a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:03.91224609Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1644,"client":"0","":"downstairs","session_id":"54c4ad36-c850-43f8-88c5-c144487de07a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:03.912293987Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1644,"client":"0","":"downstairs","session_id":"54c4ad36-c850-43f8-88c5-c144487de07a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:03.912330765Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1644,"client":"1","":"downstairs","session_id":"54c4ad36-c850-43f8-88c5-c144487de07a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:03.912401612Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1644,"client":"1","":"downstairs","session_id":"54c4ad36-c850-43f8-88c5-c144487de07a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:03.91244654Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1644,"client":"2","":"downstairs","session_id":"54c4ad36-c850-43f8-88c5-c144487de07a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:03.912483388Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1644,"client":"2","":"downstairs","session_id":"54c4ad36-c850-43f8-88c5-c144487de07a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:03.912517876Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1644,"":"io task","client":"2","":"downstairs","session_id":"54c4ad36-c850-43f8-88c5-c144487de07a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:03.912680739Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1644,"":"io task","client":"0","":"downstairs","session_id":"54c4ad36-c850-43f8-88c5-c144487de07a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:03.912717617Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1644,"":"io task","client":"1","":"downstairs","session_id":"54c4ad36-c850-43f8-88c5-c144487de07a"} {"msg":"ds_connection connected from Ok(127.0.0.1:36495)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:03.912751525Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1644,"":"io task","client":"2","":"downstairs","session_id":"54c4ad36-c850-43f8-88c5-c144487de07a"} {"msg":"ds_connection connected from Ok(127.0.0.1:32958)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:03.912778564Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1644,"":"io task","client":"0","":"downstairs","session_id":"54c4ad36-c850-43f8-88c5-c144487de07a"} {"msg":"ds_connection connected from Ok(127.0.0.1:54996)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:03.912803853Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1644,"":"io task","client":"1","":"downstairs","session_id":"54c4ad36-c850-43f8-88c5-c144487de07a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:03.913250481Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1644,"client":"1","":"downstairs","session_id":"54c4ad36-c850-43f8-88c5-c144487de07a"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:03.913279739Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1644,"client":"1","":"downstairs","session_id":"54c4ad36-c850-43f8-88c5-c144487de07a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:03.913405813Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1644,"client":"0","":"downstairs","session_id":"54c4ad36-c850-43f8-88c5-c144487de07a"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:03.913449141Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1644,"client":"0","":"downstairs","session_id":"54c4ad36-c850-43f8-88c5-c144487de07a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:03.913524197Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1644,"client":"2","":"downstairs","session_id":"54c4ad36-c850-43f8-88c5-c144487de07a"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:03.913552486Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1644,"client":"2","":"downstairs","session_id":"54c4ad36-c850-43f8-88c5-c144487de07a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:03.913939917Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1644,"client":"1","":"downstairs","session_id":"54c4ad36-c850-43f8-88c5-c144487de07a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:03.913982075Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1644,"client":"2","":"downstairs","session_id":"54c4ad36-c850-43f8-88c5-c144487de07a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:03.914249712Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1644,"client":"0","":"downstairs","session_id":"54c4ad36-c850-43f8-88c5-c144487de07a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:03.914512599Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1644,"client":"1","":"downstairs","session_id":"54c4ad36-c850-43f8-88c5-c144487de07a"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:03.914537708Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1644,"client":"1","":"downstairs","session_id":"54c4ad36-c850-43f8-88c5-c144487de07a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:03.914551817Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1644,"session_id":"54c4ad36-c850-43f8-88c5-c144487de07a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:03.914615514Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1644,"client":"2","":"downstairs","session_id":"54c4ad36-c850-43f8-88c5-c144487de07a"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:03.914636403Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1644,"client":"2","":"downstairs","session_id":"54c4ad36-c850-43f8-88c5-c144487de07a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:03.914649982Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1644,"session_id":"54c4ad36-c850-43f8-88c5-c144487de07a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:03.91468732Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1644,"client":"0","":"downstairs","session_id":"54c4ad36-c850-43f8-88c5-c144487de07a"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:03.91470374Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1644,"client":"0","":"downstairs","session_id":"54c4ad36-c850-43f8-88c5-c144487de07a"} {"msg":"[0]R flush_numbers[0..12]: [199, 192, 192, 197, 201, 205, 203, 178, 204, 205, 202, 199]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:03.914728878Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1644,"":"downstairs","":"downstairs","session_id":"54c4ad36-c850-43f8-88c5-c144487de07a"} {"msg":"[0]R generation[0..12]: [112, 106, 106, 110, 113, 115, 114, 99, 114, 115, 114, 112]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:03.914743378Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1644,"":"downstairs","":"downstairs","session_id":"54c4ad36-c850-43f8-88c5-c144487de07a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:03.914757467Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1644,"":"downstairs","":"downstairs","session_id":"54c4ad36-c850-43f8-88c5-c144487de07a"} {"msg":"[1]R flush_numbers[0..12]: [199, 192, 192, 197, 201, 205, 203, 178, 204, 205, 202, 199]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:03.914771496Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1644,"":"downstairs","":"downstairs","session_id":"54c4ad36-c850-43f8-88c5-c144487de07a"} {"msg":"[1]R generation[0..12]: [112, 106, 106, 110, 113, 115, 114, 99, 114, 115, 114, 112]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:03.914785126Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1644,"":"downstairs","":"downstairs","session_id":"54c4ad36-c850-43f8-88c5-c144487de07a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:03.914799285Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1644,"":"downstairs","":"downstairs","session_id":"54c4ad36-c850-43f8-88c5-c144487de07a"} {"msg":"[2]R flush_numbers[0..12]: [199, 192, 192, 197, 201, 205, 203, 178, 204, 205, 202, 199]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:03.914813044Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1644,"":"downstairs","":"downstairs","session_id":"54c4ad36-c850-43f8-88c5-c144487de07a"} {"msg":"[2]R generation[0..12]: [112, 106, 106, 110, 113, 115, 114, 99, 114, 115, 114, 112]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:03.914826823Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1644,"":"downstairs","":"downstairs","session_id":"54c4ad36-c850-43f8-88c5-c144487de07a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:03.914840553Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1644,"":"downstairs","":"downstairs","session_id":"54c4ad36-c850-43f8-88c5-c144487de07a"} {"msg":"Max found gen is 116","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:03.914856482Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1644,"":"downstairs","":"downstairs","session_id":"54c4ad36-c850-43f8-88c5-c144487de07a"} {"msg":"Generation requested: 116 >= found:116","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:03.914869701Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1644,"":"downstairs","":"downstairs","session_id":"54c4ad36-c850-43f8-88c5-c144487de07a"} {"msg":"Next flush: 206","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:03.914882611Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1644,"":"downstairs","":"downstairs","session_id":"54c4ad36-c850-43f8-88c5-c144487de07a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:03.91489565Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1644,"":"downstairs","":"downstairs","session_id":"54c4ad36-c850-43f8-88c5-c144487de07a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:03.914908679Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1644,"session_id":"54c4ad36-c850-43f8-88c5-c144487de07a"} Wait for a query_work_queue command to finish before sending IO {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:03.914979356Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1644,"client":"0","":"downstairs","session_id":"54c4ad36-c850-43f8-88c5-c144487de07a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:03.914998255Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1644,"client":"1","":"downstairs","session_id":"54c4ad36-c850-43f8-88c5-c144487de07a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:03.915011865Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1644,"client":"2","":"downstairs","session_id":"54c4ad36-c850-43f8-88c5-c144487de07a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:03.915025384Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1644,"session_id":"54c4ad36-c850-43f8-88c5-c144487de07a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:03.915037743Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1644,"session_id":"54c4ad36-c850-43f8-88c5-c144487de07a"} {"msg":"b8fb576b-59cf-4185-9f93-c1d36889c880 is now active with session: 54c4ad36-c850-43f8-88c5-c144487de07a","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:03.915050523Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1644,"session_id":"54c4ad36-c850-43f8-88c5-c144487de07a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:03.915062642Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1644,"session_id":"54c4ad36-c850-43f8-88c5-c144487de07a"} {"msg":"The guest has finished waiting for activation with:116","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:03.915074751Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1644} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 402 len 36864 data: 9 9 8 8 9 9 10 11 13 02/30 Write block 578 len 36864 data: 7 6 9 9 9 12 13 16 12 03/30 Read block 108 len 20480 04/30 Flush 05/30 Flush 06/30 Write block 53 len 20480 data: 7 10 9 8 8 07/30 Flush 08/30 Write block 556 len 40960 data: 7 7 7 8 7 7 7 6 7 9 09/30 Read block 204 len 32768 10/30 Read block 414 len 32768 11/30 Read block 216 len 24576 12/30 Flush 13/30 Write block 268 len 12288 data: 10 10 10 14/30 Read block 504 len 4096 15/30 Write block 320 len 4096 data: 10 16/30 Read block 477 len 28672 17/30 Read block 342 len 12288 18/30 Read block 374 len 12288 19/30 Write block 246 len 40960 data: 9 13 13 14 16 12 15 14 13 11 20/30 Read block 562 len 28672 21/30 Read block 341 len 16384 22/30 Read block 544 len 16384 23/30 Read block 559 len 36864 24/30 Read block 529 len 16384 25/30 Write block 509 len 16384 data: 8 9 10 8 26/30 Flush 27/30 Write block 350 len 8192 data: 11 12 28/30 Read block 420 len 40960 29/30 Write block 101 len 8192 data: 6 7 30/30 Read block 339 len 16384 ---------------------------------------------------------------- Crucible gen:116 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 2 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Write 2 Done Done Done false 1029 Acked Read 4 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:27:03.943083582Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1644} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 112 112 112 199 199 199 F F F 1 020-039 106 106 106 192 192 192 F F F 2 040-059 116 116 116 208 208 208 F F F 3 060-079 110 110 110 197 197 197 F F F 4 080-099 113 113 113 201 201 201 F F F 5 100-119 115 115 115 205 205 205 T T T 6 120-139 114 114 114 203 203 203 F F F 7 140-159 99 99 99 178 178 178 F F F 8 160-179 114 114 114 204 204 204 F F F 9 180-199 115 115 115 205 205 205 F F F 10 200-219 114 114 114 202 202 202 F F F 11 220-239 112 112 112 199 199 199 F F F 12 240-259 116 116 116 210 210 210 F F F 13 260-279 116 116 116 210 210 210 F F F 14 280-299 113 113 113 201 201 201 F F F 15 300-319 114 114 114 202 202 202 F F F 16 320-339 116 116 116 210 210 210 F F F 17 340-359 113 113 113 201 201 201 T T T 18 360-379 113 113 113 201 201 201 F F F 19 380-399 107 107 107 193 193 193 F F F 20 400-419 116 116 116 206 206 206 F F F 21 420-439 114 114 114 202 202 202 F F F 22 440-459 114 114 114 204 204 204 F F F 23 460-479 114 114 114 204 204 204 F F F 24 480-499 112 112 112 199 199 199 F F F 25 500-519 116 116 116 210 210 210 F F F 26 520-539 115 115 115 205 205 205 F F F 27 540-559 116 116 116 209 209 209 F F F 28 560-579 116 116 116 209 209 209 F F F 29 580-599 116 116 116 206 206 206 F F F Max gen: 116, Max flush: 210 A Difference in extent metadata is expected here On loop 58 Verifying data now No change after verify Loop: 58 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 112 112 112 199 199 199 F F F 1 020-039 106 106 106 192 192 192 F F F 2 040-059 116 116 116 208 208 208 F F F 3 060-079 110 110 110 197 197 197 F F F 4 080-099 113 113 113 201 201 201 F F F 5 100-119 117 117 117 211 211 211 F F F 6 120-139 114 114 114 203 203 203 F F F 7 140-159 99 99 99 178 178 178 F F F 8 160-179 114 114 114 204 204 204 F F F 9 180-199 115 115 115 205 205 205 F F F 10 200-219 114 114 114 202 202 202 F F F 11 220-239 112 112 112 199 199 199 F F F 12 240-259 116 116 116 210 210 210 F F F 13 260-279 116 116 116 210 210 210 F F F 14 280-299 113 113 113 201 201 201 F F F 15 300-319 114 114 114 202 202 202 F F F 16 320-339 116 116 116 210 210 210 F F F 17 340-359 117 117 117 211 211 211 F F F 18 360-379 113 113 113 201 201 201 F F F 19 380-399 107 107 107 193 193 193 F F F 20 400-419 116 116 116 206 206 206 F F F 21 420-439 114 114 114 202 202 202 F F F 22 440-459 114 114 114 204 204 204 F F F 23 460-479 114 114 114 204 204 204 F F F 24 480-499 112 112 112 199 199 199 F F F 25 500-519 116 116 116 210 210 210 F F F 26 520-539 115 115 115 205 205 205 F F F 27 540-559 116 116 116 209 209 209 F F F 28 560-579 116 116 116 209 209 209 F F F 29 580-599 116 116 116 206 206 206 F F F Max gen: 117, Max flush: 211 Begin loop 59 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:08.253776321Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1652,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:27:08.297314893Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1652,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:08.297692995Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1652,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:08.330441874Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1652,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:08.330975458Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1652} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"0fd98d0d5d909611f8b408ccda2559d35fe117bb\",\n git_commit_timestamp: \"2024-11-08T20:05:44.000000000Z\",\n git_branch: \"mkeeter/drop-skipped-backpressure-guard\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:08.331010077Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1652} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:08.331023596Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1652} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:08.331037765Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1652} {"msg":"Crucible 2c35b57b-cb98-452c-acf9-82ff49b86ae7 has session id: 4a680a27-874b-492d-8de4-786c34347e6e","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:08.331608487Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1652,"session_id":"4a680a27-874b-492d-8de4-786c34347e6e"} {"msg":"Upstairs opts: Upstairs UUID: 2c35b57b-cb98-452c-acf9-82ff49b86ae7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:08.331653745Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1652,"session_id":"4a680a27-874b-492d-8de4-786c34347e6e"} {"msg":"Crucible stats registered with UUID: 2c35b57b-cb98-452c-acf9-82ff49b86ae7","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:08.331682644Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1652,"session_id":"4a680a27-874b-492d-8de4-786c34347e6e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:08.363349546Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1652,"task":"crutest"} {"msg":"The guest has requested activation with gen:118","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:08.363387804Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1652} {"msg":"2c35b57b-cb98-452c-acf9-82ff49b86ae7 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:08.363419043Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1652,"session_id":"4a680a27-874b-492d-8de4-786c34347e6e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:08.363441641Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1652,"client":"0","":"downstairs","session_id":"4a680a27-874b-492d-8de4-786c34347e6e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:08.363486979Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1652,"client":"0","":"downstairs","session_id":"4a680a27-874b-492d-8de4-786c34347e6e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:08.363524137Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1652,"client":"1","":"downstairs","session_id":"4a680a27-874b-492d-8de4-786c34347e6e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:08.363560236Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1652,"client":"1","":"downstairs","session_id":"4a680a27-874b-492d-8de4-786c34347e6e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:08.363642922Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1652,"client":"2","":"downstairs","session_id":"4a680a27-874b-492d-8de4-786c34347e6e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:08.36368302Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1652,"client":"2","":"downstairs","session_id":"4a680a27-874b-492d-8de4-786c34347e6e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:08.363720778Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1652,"":"io task","client":"2","":"downstairs","session_id":"4a680a27-874b-492d-8de4-786c34347e6e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:08.363752776Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1652,"":"io task","client":"0","":"downstairs","session_id":"4a680a27-874b-492d-8de4-786c34347e6e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:08.363782435Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1652,"":"io task","client":"1","":"downstairs","session_id":"4a680a27-874b-492d-8de4-786c34347e6e"} {"msg":"ds_connection connected from Ok(127.0.0.1:65193)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:08.36387774Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1652,"":"io task","client":"2","":"downstairs","session_id":"4a680a27-874b-492d-8de4-786c34347e6e"} {"msg":"ds_connection connected from Ok(127.0.0.1:65238)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:08.363898989Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1652,"":"io task","client":"1","":"downstairs","session_id":"4a680a27-874b-492d-8de4-786c34347e6e"} {"msg":"ds_connection connected from Ok(127.0.0.1:55095)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:08.363918188Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1652,"":"io task","client":"0","":"downstairs","session_id":"4a680a27-874b-492d-8de4-786c34347e6e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:08.364452192Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1652,"client":"2","":"downstairs","session_id":"4a680a27-874b-492d-8de4-786c34347e6e"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:08.36449043Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1652,"client":"2","":"downstairs","session_id":"4a680a27-874b-492d-8de4-786c34347e6e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:08.364538138Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1652,"client":"1","":"downstairs","session_id":"4a680a27-874b-492d-8de4-786c34347e6e"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:08.364569136Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1652,"client":"1","":"downstairs","session_id":"4a680a27-874b-492d-8de4-786c34347e6e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:08.364636863Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1652,"client":"0","":"downstairs","session_id":"4a680a27-874b-492d-8de4-786c34347e6e"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:08.364670081Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1652,"client":"0","":"downstairs","session_id":"4a680a27-874b-492d-8de4-786c34347e6e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:08.365153528Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1652,"client":"1","":"downstairs","session_id":"4a680a27-874b-492d-8de4-786c34347e6e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:08.365193746Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1652,"client":"2","":"downstairs","session_id":"4a680a27-874b-492d-8de4-786c34347e6e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:08.365223694Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1652,"client":"0","":"downstairs","session_id":"4a680a27-874b-492d-8de4-786c34347e6e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:08.365684122Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1652,"client":"1","":"downstairs","session_id":"4a680a27-874b-492d-8de4-786c34347e6e"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:08.365709521Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1652,"client":"1","":"downstairs","session_id":"4a680a27-874b-492d-8de4-786c34347e6e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:08.3657246Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1652,"session_id":"4a680a27-874b-492d-8de4-786c34347e6e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:08.365788867Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1652,"client":"0","":"downstairs","session_id":"4a680a27-874b-492d-8de4-786c34347e6e"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:08.365807956Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1652,"client":"0","":"downstairs","session_id":"4a680a27-874b-492d-8de4-786c34347e6e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:08.365821765Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1652,"session_id":"4a680a27-874b-492d-8de4-786c34347e6e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:08.365837315Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1652,"client":"2","":"downstairs","session_id":"4a680a27-874b-492d-8de4-786c34347e6e"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:08.365850904Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1652,"client":"2","":"downstairs","session_id":"4a680a27-874b-492d-8de4-786c34347e6e"} {"msg":"[0]R flush_numbers[0..12]: [199, 192, 208, 197, 201, 211, 203, 178, 204, 205, 202, 199]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:08.365864573Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1652,"":"downstairs","":"downstairs","session_id":"4a680a27-874b-492d-8de4-786c34347e6e"} {"msg":"[0]R generation[0..12]: [112, 106, 116, 110, 113, 117, 114, 99, 114, 115, 114, 112]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:08.365878582Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1652,"":"downstairs","":"downstairs","session_id":"4a680a27-874b-492d-8de4-786c34347e6e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:08.365892582Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1652,"":"downstairs","":"downstairs","session_id":"4a680a27-874b-492d-8de4-786c34347e6e"} {"msg":"[1]R flush_numbers[0..12]: [199, 192, 208, 197, 201, 211, 203, 178, 204, 205, 202, 199]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:08.365906481Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1652,"":"downstairs","":"downstairs","session_id":"4a680a27-874b-492d-8de4-786c34347e6e"} {"msg":"[1]R generation[0..12]: [112, 106, 116, 110, 113, 117, 114, 99, 114, 115, 114, 112]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:08.36592017Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1652,"":"downstairs","":"downstairs","session_id":"4a680a27-874b-492d-8de4-786c34347e6e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:08.36593425Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1652,"":"downstairs","":"downstairs","session_id":"4a680a27-874b-492d-8de4-786c34347e6e"} {"msg":"[2]R flush_numbers[0..12]: [199, 192, 208, 197, 201, 211, 203, 178, 204, 205, 202, 199]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:08.365947979Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1652,"":"downstairs","":"downstairs","session_id":"4a680a27-874b-492d-8de4-786c34347e6e"} {"msg":"[2]R generation[0..12]: [112, 106, 116, 110, 113, 117, 114, 99, 114, 115, 114, 112]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:08.365961778Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1652,"":"downstairs","":"downstairs","session_id":"4a680a27-874b-492d-8de4-786c34347e6e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:08.365975688Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1652,"":"downstairs","":"downstairs","session_id":"4a680a27-874b-492d-8de4-786c34347e6e"} {"msg":"Max found gen is 118","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:08.365988757Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1652,"":"downstairs","":"downstairs","session_id":"4a680a27-874b-492d-8de4-786c34347e6e"} {"msg":"Generation requested: 118 >= found:118","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:08.366001726Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1652,"":"downstairs","":"downstairs","session_id":"4a680a27-874b-492d-8de4-786c34347e6e"} {"msg":"Next flush: 212","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:08.366014786Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1652,"":"downstairs","":"downstairs","session_id":"4a680a27-874b-492d-8de4-786c34347e6e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:08.366029505Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1652,"":"downstairs","":"downstairs","session_id":"4a680a27-874b-492d-8de4-786c34347e6e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:08.366042644Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1652,"session_id":Wait for a query_work_queue command to finish before sending IO "4a680a27-874b-492d-8de4-786c34347e6e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:08.366063603Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1652,"client":"0","":"downstairs","session_id":"4a680a27-874b-492d-8de4-786c34347e6e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:08.366076963Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1652,"client":"1","":"downstairs","session_id":"4a680a27-874b-492d-8de4-786c34347e6e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:08.366090042Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1652,"client":"2","":"downstairs","session_id":"4a680a27-874b-492d-8de4-786c34347e6e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:08.366103271Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1652,"session_id":"4a680a27-874b-492d-8de4-786c34347e6e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:08.366116921Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1652,"session_id":"4a680a27-874b-492d-8de4-786c34347e6e"} {"msg":"2c35b57b-cb98-452c-acf9-82ff49b86ae7 is now active with session: 4a680a27-874b-492d-8de4-786c34347e6e","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:08.36612999Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1652,"session_id":"4a680a27-874b-492d-8de4-786c34347e6e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:08.36614198Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1652,"session_id":"4a680a27-874b-492d-8de4-786c34347e6e"} {"msg":"The guest has finished waiting for activation with:118","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:08.366154109Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1652} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 289 len 36864 data: 13 13 11 11 11 12 10 7 9 02/30 Read block 531 len 28672 03/30 Write block 592 len 4096 data: 8 04/30 Write block 69 len 12288 data: 5 7 6 05/30 Write block 471 len 12288 data: 15 15 14 06/30 Write block 269 len 24576 data: 11 11 10 10 11 13 07/30 Write block 476 len 28672 data: 14 15 16 15 12 13 12 08/30 Read block 372 len 40960 09/30 Flush 10/30 Write block 81 len 24576 data: 7 7 9 8 10 10 11/30 Write block 189 len 32768 data: 9 12 9 9 9 9 9 10 12/30 Flush 13/30 Write block 326 len 36864 data: 9 6 7 7 7 7 7 9 10 14/30 Read block 494 len 32768 15/30 Read block 575 len 8192 16/30 Write block 204 len 24576 data: 7 8 10 10 8 8 17/30 Read block 545 len 28672 18/30 Write block 312 len 12288 data: 10 12 10 19/30 Flush 20/30 Flush 21/30 Write block 131 len 20480 data: 8 9 11 11 13 22/30 Read block 391 len 36864 23/30 Read block 447 len 32768 24/30 Flush 25/30 Write block 89 len 8192 data: 9 7 26/30 Read block 119 len 4096 27/30 Write block 441 len 36864 data: 15 16 11 10 10 13 12 10 10 28/30 Write block 355 len 28672 data: 16 15 14 15 16 16 17 29/30 Read block 550 len 40960 30/30 Write block 146 len 16384 data: 8 9 8 9 ---------------------------------------------------------------- Crucible gen:118 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 2 Done Done Done false 1025 Acked Read 1 Done Done Done false 1026 Acked Write 9 Done Done Sent false 1027 Acked Write 7 Done Done Sent false 1028 Acked Read 10 Done Done Sent false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 4 6 Done 5 5 2 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:27:08.394379009Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1652} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 112 112 112 199 199 199 F F F 1 020-039 106 106 106 192 192 192 F F F 2 040-059 116 116 116 208 208 208 F F F 3 060-079 118 118 118 212 212 212 F F F 4 080-099 118 118 118 213 213 213 T T T 5 100-119 117 117 117 211 211 211 F F F 6 120-139 118 118 118 216 216 216 F F F 7 140-159 99 99 99 178 178 178 T T T 8 160-179 114 114 114 204 204 204 F F F 9 180-199 118 118 118 213 213 213 F F F 10 200-219 118 118 118 214 214 214 F F F 11 220-239 112 112 112 199 199 199 F F F 12 240-259 116 116 116 210 210 210 F F F 13 260-279 118 118 118 212 212 212 F F F 14 280-299 118 118 118 212 212 212 F F F 15 300-319 118 118 118 214 214 214 F F F 16 320-339 118 118 118 214 214 214 F F F 17 340-359 117 117 117 211 211 211 T T T 18 360-379 113 113 113 201 201 201 T T T 19 380-399 107 107 107 193 193 193 F F F 20 400-419 116 116 116 206 206 206 F F F 21 420-439 114 114 114 202 202 202 F F F 22 440-459 114 114 114 204 204 204 T T T 23 460-479 118 118 118 212 212 212 F F F 24 480-499 118 118 118 212 212 212 F F F 25 500-519 116 116 116 210 210 210 F F F 26 520-539 115 115 115 205 205 205 F F F 27 540-559 116 116 116 209 209 209 F F F 28 560-579 116 116 116 209 209 209 F F F 29 580-599 118 118 118 212 212 212 F F F Max gen: 118, Max flush: 216 A Difference in extent metadata is expected here On loop 59 Verifying data now No change after verify Loop: 59 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 112 112 112 199 199 199 F F F 1 020-039 106 106 106 192 192 192 F F F 2 040-059 116 116 116 208 208 208 F F F 3 060-079 118 118 118 212 212 212 F F F 4 080-099 119 119 119 217 217 217 F F F 5 100-119 117 117 117 211 211 211 F F F 6 120-139 118 118 118 216 216 216 F F F 7 140-159 119 119 119 217 217 217 F F F 8 160-179 114 114 114 204 204 204 F F F 9 180-199 118 118 118 213 213 213 F F F 10 200-219 118 118 118 214 214 214 F F F 11 220-239 112 112 112 199 199 199 F F F 12 240-259 116 116 116 210 210 210 F F F 13 260-279 118 118 118 212 212 212 F F F 14 280-299 118 118 118 212 212 212 F F F 15 300-319 118 118 118 214 214 214 F F F 16 320-339 118 118 118 214 214 214 F F F 17 340-359 119 119 119 217 217 217 F F F 18 360-379 119 119 119 217 217 217 F F F 19 380-399 107 107 107 193 193 193 F F F 20 400-419 116 116 116 206 206 206 F F F 21 420-439 114 114 114 202 202 202 F F F 22 440-459 119 119 119 217 217 217 F F F 23 460-479 118 118 118 212 212 212 F F F 24 480-499 118 118 118 212 212 212 F F F 25 500-519 116 116 116 210 210 210 F F F 26 520-539 115 115 115 205 205 205 F F F 27 540-559 116 116 116 209 209 209 F F F 28 560-579 116 116 116 209 209 209 F F F 29 580-599 118 118 118 212 212 212 F F F Max gen: 119, Max flush: 217 Begin loop 60 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:12.752418385Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1660,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:12.797341929Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1660,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:12.797943709Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1660} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"0fd98d0d5d909611f8b408ccda2559d35fe117bb\",\n git_commit_timestamp: \"2024-11-08T20:05:44.000000000Z\",\n git_branch: \"mkeeter/drop-skipped-backpressure-guard\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:12.797979888Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1660} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:12.797994197Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1660} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:12.798008486Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1660} {"msg":"Crucible 5365937f-6388-4db6-9f07-a98fdc977438 has session id: 19772f06-0a91-48a8-843e-0268f1ca227e","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:12.798536491Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1660,"session_id":"19772f06-0a91-48a8-843e-0268f1ca227e"} {"msg":"Upstairs opts: Upstairs UUID: 5365937f-6388-4db6-9f07-a98fdc977438, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:12.798583428Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1660,"session_id":"19772f06-0a91-48a8-843e-0268f1ca227e"} {"msg":"Crucible stats registered with UUID: 5365937f-6388-4db6-9f07-a98fdc977438","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:12.798608077Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1660,"session_id":"19772f06-0a91-48a8-843e-0268f1ca227e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:12.82984257Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1660,"task":"crutest"} {"msg":"The guest has requested activation with gen:120","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:12.829875589Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1660} {"msg":"5365937f-6388-4db6-9f07-a98fdc977438 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:12.829913667Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1660,"session_id":"19772f06-0a91-48a8-843e-0268f1ca227e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:12.829931076Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1660,"client":"0","":"downstairs","session_id":"19772f06-0a91-48a8-843e-0268f1ca227e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:12.829959795Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1660,"client":"0","":"downstairs","session_id":"19772f06-0a91-48a8-843e-0268f1ca227e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:12.829986683Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1660,"client":"1","":"downstairs","session_id":"19772f06-0a91-48a8-843e-0268f1ca227e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:12.830018492Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1660,"client":"1","":"downstairs","session_id":"19772f06-0a91-48a8-843e-0268f1ca227e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:12.83005267Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1660,"client":"2","":"downstairs","session_id":"19772f06-0a91-48a8-843e-0268f1ca227e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:12.830087929Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1660,"client":"2","":"downstairs","session_id":"19772f06-0a91-48a8-843e-0268f1ca227e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:12.830120617Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1660,"":"io task","client":"0","":"downstairs","session_id":"19772f06-0a91-48a8-843e-0268f1ca227e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:12.830156155Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1660,"":"io task","client":"2","":"downstairs","session_id":"19772f06-0a91-48a8-843e-0268f1ca227e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:12.830187493Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1660,"":"io task","client":"1","":"downstairs","session_id":"19772f06-0a91-48a8-843e-0268f1ca227e"} {"msg":"ds_connection connected from Ok(127.0.0.1:37083)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:12.830223972Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1660,"":"io task","client":"0","":"downstairs","session_id":"19772f06-0a91-48a8-843e-0268f1ca227e"} {"msg":"ds_connection connected from Ok(127.0.0.1:41794)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:12.83025567Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1660,"":"io task","client":"2","":"downstairs","session_id":"19772f06-0a91-48a8-843e-0268f1ca227e"} {"msg":"ds_connection connected from Ok(127.0.0.1:37394)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:12.830285979Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1660,"":"io task","client":"1","":"downstairs","session_id":"19772f06-0a91-48a8-843e-0268f1ca227e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:12.831002964Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1660,"client":"2","":"downstairs","session_id":"19772f06-0a91-48a8-843e-0268f1ca227e"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:12.831042432Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1660,"client":"2","":"downstairs","session_id":"19772f06-0a91-48a8-843e-0268f1ca227e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:12.831122988Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1660,"client":"0","":"downstairs","session_id":"19772f06-0a91-48a8-843e-0268f1ca227e"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:12.831201874Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1660,"client":"0","":"downstairs","session_id":"19772f06-0a91-48a8-843e-0268f1ca227e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:12.83128457Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1660,"client":"1","":"downstairs","session_id":"19772f06-0a91-48a8-843e-0268f1ca227e"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:12.831322318Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1660,"client":"1","":"downstairs","session_id":"19772f06-0a91-48a8-843e-0268f1ca227e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:12.831722989Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1660,"client":"0","":"downstairs","session_id":"19772f06-0a91-48a8-843e-0268f1ca227e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:12.831762716Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1660,"client":"2","":"downstairs","session_id":"19772f06-0a91-48a8-843e-0268f1ca227e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:12.832023214Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1660,"client":"1","":"downstairs","session_id":"19772f06-0a91-48a8-843e-0268f1ca227e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:12.832376357Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1660,"client":"0","":"downstairs","session_id":"19772f06-0a91-48a8-843e-0268f1ca227e"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:12.832418195Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1660,"client":"0","":"downstairs","session_id":"19772f06-0a91-48a8-843e-0268f1ca227e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:12.832447933Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1660,"session_id":"19772f06-0a91-48a8-843e-0268f1ca227e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:12.832470722Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1660,"client":"2","":"downstairs","session_id":"19772f06-0a91-48a8-843e-0268f1ca227e"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:12.832497851Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1660,"client":"2","":"downstairs","session_id":"19772f06-0a91-48a8-843e-0268f1ca227e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:12.83252203Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1660,"session_id":"19772f06-0a91-48a8-843e-0268f1ca227e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:12.832536699Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1660,"client":"1","":"downstairs","session_id":"19772f06-0a91-48a8-843e-0268f1ca227e"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:12.832550558Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1660,"client":"1","":"downstairs","session_id":"19772f06-0a91-48a8-843e-0268f1ca227e"} {"msg":"[0]R flush_numbers[0..12]: [199, 192, 208, 212, 217, 211, 216, 217, 204, 213, 214, 199]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:12.832593386Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1660,"":"downstairs","":"downstairs","session_id":"19772f06-0a91-48a8-843e-0268f1ca227e"} {"msg":"[0]R generation[0..12]: [112, 106, 116, 118, 119, 117, 118, 119, 114, 118, 118, 112]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:12.832615395Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1660,"":"downstairs","":"downstairs","session_id":"19772f06-0a91-48a8-843e-0268f1ca227e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:12.832630684Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1660,"":"downstairs","":"downstairs","session_id":"19772f06-0a91-48a8-843e-0268f1ca227e"} {"msg":"[1]R flush_numbers[0..12]: [199, 192, 208, 212, 217, 211, 216, 217, 204, 213, 214, 199]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:12.832644564Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1660,"":"downstairs","":"downstairs","session_id":"19772f06-0a91-48a8-843e-0268f1ca227e"} {"msg":"[1]R generation[0..12]: [112, 106, 116, 118, 119, 117, 118, 119, 114, 118, 118, 112]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:12.832658323Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1660,"":"downstairs","":"downstairs","session_id":"19772f06-0a91-48a8-843e-0268f1ca227e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:12.832672122Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1660,"":"downstairs","":"downstairs","session_id":"19772f06-0a91-48a8-843e-0268f1ca227e"} {"msg":"[2]R flush_numbers[0..12]: [199, 192, 208, 212, 217, 211, 216, 217, 204, 213, 214, 199]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:12.832685791Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1660,"":"downstairs","":"downstairs","session_id":"19772f06-0a91-48a8-843e-0268f1ca227e"} {"msg":"[2]R generation[0..12]: [112, 106, 116, 118, 119, 117, 118, 119, 114, 118, 118, 112]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:12.832699421Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1660,"":"downstairs","":"downstairs","session_id":"19772f06-0a91-48a8-843e-0268f1ca227e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:12.83271337Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1660,"":"downstairs","":"downstairs","session_id":"19772f06-0a91-48a8-843e-0268f1ca227e"} {"msg":"Max found gen is 120","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:12.83272638Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1660,"":"downstairs","":"downstairs","session_id":"19772f06-0a91-48a8-843e-0268f1ca227e"} {"msg":"Generation requested: 120 >= found:120","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:12.832739549Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1660,"":"downstairs","":"downstairs","session_id":"19772f06-0a91-48a8-843e-0268f1ca227e"} {"msg":"Next flush: 218","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:12.832752398Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1660,"":"downstairs","":"downstairs","session_id":"19772f06-0a91-48a8-843e-0268f1ca227e"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-08T20:27:12.832765498Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1660,"":"downstairs","":"downstairs","session_id":"19772f06-0a91-48a8-843e-0268f1ca227e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:12.832785567Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1660,"session_id":"19772f06-0a91-48a8-843e-0268f1ca227e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:12.832797556Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1660,"client":"0","":"downstairs","session_id":"19772f06-0a91-48a8-843e-0268f1ca227e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:12.832813775Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1660,"client":"1","":"downstairs","session_id":"19772f06-0a91-48a8-843e-0268f1ca227e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:12.832827155Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1660,"client":"2","":"downstairs","session_id":"19772f06-0a91-48a8-843e-0268f1ca227e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:12.832841354Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1660,"session_id":"19772f06-0a91-48a8-843e-0268f1ca227e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:12.832853723Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1660,"session_id":"19772f06-0a91-48a8-843e-0268f1ca227e"} {"msg":"5365937f-6388-4db6-9f07-a98fdc977438 is now active with session: 19772f06-0a91-48a8-843e-0268f1ca227e","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:12.832866523Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1660,"session_id":"19772f06-0a91-48a8-843e-0268f1ca227e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:12.832878332Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1660,"session_id":"19772f06-0a91-48a8-843e-0268f1ca227e"} {"msg":"The guest has finished waiting for activation with:120","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:12.832890612Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1660} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 232 len 36864 data: 8 6 5 7 8 8 11 10 11 02/30 Read block 537 len 40960 03/30 Write block 385 len 20480 data: 5 5 5 6 6 04/30 Read block 254 len 36864 05/30 Read block 342 len 24576 06/30 Read block 102 len 32768 07/30 Flush 08/30 Write block 233 len 4096 data: 7 09/30 Write block 594 len 8192 data: 6 8 10/30 Read block 40 len 24576 11/30 Read block 24 len 4096 12/30 Read block 39 len 8192 13/30 Read block 380 len 20480 14/30 Read block 522 len 28672 15/30 Read block 253 len 36864 16/30 Read block 67 len 40960 17/30 Read block 251 len 32768 18/30 Write block 134 len 16384 data: 12 14 12 12 19/30 Read block 153 len 36864 20/30 Read block 191 len 12288 21/30 Read block 297 len 8192 22/30 Read block 494 len 8192 23/30 Read block 458 len 8192 24/30 Read block 503 len 28672 25/30 Write block 183 len 4096 data: 9 26/30 Write block 229 len 4096 data: 9 27/30 Read block 464 len 32768 28/30 Write block 317 len 16384 data: 8 10 12 11 29/30 Read block 286 len 36864 30/30 Write block 105 len 12288 data: 7 7 10 ---------------------------------------------------------------- Crucible gen:120 GIO:true work queues: Upstairs:0 downstairs:23 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1007 Acked Write 1 Done Done Done false 1008 Acked Write 2 Done Done Done false 1009 Acked Read 6 Done Done Done false 1010 Acked Read 1 Done Done Done false 1011 Acked Read 2 Done Done Done false 1012 Acked Read 5 Done Done Done false 1013 Acked Read 7 Done Done Done false 1014 Acked Read 9 Done Done Done false 1015 Acked Read 10 Done Done Done false 1016 Acked Read 8 Done Done Done false 1017 Acked Write 4 Done Done Done false 1018 Acked Read 9 Done Done Done false 1019 Acked Read 3 Done Done Done false 1020 Acked Read 2 Done Done Done false 1021 Acked Read 2 Done Done Done false 1022 Acked Read 2 Done Done Done false 1023 Acked Read 7 Done Done Done false 1024 Acked Write 1 Done Done Done false 1025 Acked Write 1 Done Done Done false 1026 Acked Read 8 Done Done Done false 1027 Acked Write 4 Done Done Done false 1028 Acked Read 9 Done Done Done false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 22 22 22 66 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1006 1006 1006 Downstairs last five completed: 1006 1005 1004 1003 1002 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:27:12.854030648Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1660} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 112 112 112 199 199 199 F F F 1 020-039 106 106 106 192 192 192 F F F 2 040-059 116 116 116 208 208 208 F F F 3 060-079 118 118 118 212 212 212 F F F 4 080-099 119 119 119 217 217 217 F F F 5 100-119 117 117 117 211 211 211 T T T 6 120-139 118 118 118 216 216 216 T T T 7 140-159 119 119 119 217 217 217 F F F 8 160-179 114 114 114 204 204 204 F F F 9 180-199 118 118 118 213 213 213 T T T 10 200-219 118 118 118 214 214 214 F F F 11 220-239 120 120 120 218 218 218 T T T 12 240-259 120 120 120 218 218 218 F F F 13 260-279 118 118 118 212 212 212 F F F 14 280-299 118 118 118 212 212 212 F F F 15 300-319 118 118 118 214 214 214 T T T 16 320-339 118 118 118 214 214 214 T T T 17 340-359 119 119 119 217 217 217 F F F 18 360-379 119 119 119 217 217 217 F F F 19 380-399 120 120 120 218 218 218 F F F 20 400-419 116 116 116 206 206 206 F F F 21 420-439 114 114 114 202 202 202 F F F 22 440-459 119 119 119 217 217 217 F F F 23 460-479 118 118 118 212 212 212 F F F 24 480-499 118 118 118 212 212 212 F F F 25 500-519 116 116 116 210 210 210 F F F 26 520-539 115 115 115 205 205 205 F F F 27 540-559 116 116 116 209 209 209 F F F 28 560-579 116 116 116 209 209 209 F F F 29 580-599 118 118 118 212 212 212 T T T Max gen: 120, Max flush: 218 A Difference in extent metadata is expected here On loop 60 Verifying data now No change after verify Loop: 60 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 112 112 112 199 199 199 F F F 1 020-039 106 106 106 192 192 192 F F F 2 040-059 116 116 116 208 208 208 F F F 3 060-079 118 118 118 212 212 212 F F F 4 080-099 119 119 119 217 217 217 F F F 5 100-119 121 121 121 219 219 219 F F F 6 120-139 121 121 121 219 219 219 F F F 7 140-159 119 119 119 217 217 217 F F F 8 160-179 114 114 114 204 204 204 F F F 9 180-199 121 121 121 219 219 219 F F F 10 200-219 118 118 118 214 214 214 F F F 11 220-239 121 121 121 219 219 219 F F F 12 240-259 120 120 120 218 218 218 F F F 13 260-279 118 118 118 212 212 212 F F F 14 280-299 118 118 118 212 212 212 F F F 15 300-319 121 121 121 219 219 219 F F F 16 320-339 121 121 121 219 219 219 F F F 17 340-359 119 119 119 217 217 217 F F F 18 360-379 119 119 119 217 217 217 F F F 19 380-399 120 120 120 218 218 218 F F F 20 400-419 116 116 116 206 206 206 F F F 21 420-439 114 114 114 202 202 202 F F F 22 440-459 119 119 119 217 217 217 F F F 23 460-479 118 118 118 212 212 212 F F F 24 480-499 118 118 118 212 212 212 F F F 25 500-519 116 116 116 210 210 210 F F F 26 520-539 115 115 115 205 205 205 F F F 27 540-559 116 116 116 209 209 209 F F F 28 560-579 116 116 116 209 209 209 F F F 29 580-599 121 121 121 219 219 219 F F F Max gen: 121, Max flush: 219 Begin loop 61 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:17.213635247Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1668,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:27:17.258195689Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1668,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:17.2585918Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1668,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:17.290789926Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1668,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:17.291442564Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1668} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"0fd98d0d5d909611f8b408ccda2559d35fe117bb\",\n git_commit_timestamp: \"2024-11-08T20:05:44.000000000Z\",\n git_branch: \"mkeeter/drop-skipped-backpressure-guard\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:17.291479362Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1668} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:17.291493372Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1668} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:17.291507641Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1668} {"msg":"Crucible 784848a0-a44a-4aba-9de8-88c2245745fc has session id: 2772b901-e5b8-4f6f-9a66-dc5e25c56073","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:17.291988738Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1668,"session_id":"2772b901-e5b8-4f6f-9a66-dc5e25c56073"} {"msg":"Upstairs opts: Upstairs UUID: 784848a0-a44a-4aba-9de8-88c2245745fc, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:17.292035595Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1668,"session_id":"2772b901-e5b8-4f6f-9a66-dc5e25c56073"} {"msg":"Crucible stats registered with UUID: 784848a0-a44a-4aba-9de8-88c2245745fc","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:17.292060584Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1668,"session_id":"2772b901-e5b8-4f6f-9a66-dc5e25c56073"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:17.323112006Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1668,"task":"crutest"} {"msg":"The guest has requested activation with gen:122","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:17.323147465Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1668} {"msg":"784848a0-a44a-4aba-9de8-88c2245745fc active request set","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:17.323182773Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1668,"session_id":"2772b901-e5b8-4f6f-9a66-dc5e25c56073"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:17.323200052Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1668,"client":"0","":"downstairs","session_id":"2772b901-e5b8-4f6f-9a66-dc5e25c56073"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:17.323228221Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1668,"client":"0","":"downstairs","session_id":"2772b901-e5b8-4f6f-9a66-dc5e25c56073"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:17.32324631Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1668,"client":"1","":"downstairs","session_id":"2772b901-e5b8-4f6f-9a66-dc5e25c56073"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:17.323263669Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1668,"client":"1","":"downstairs","session_id":"2772b901-e5b8-4f6f-9a66-dc5e25c56073"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:17.323280378Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1668,"client":"2","":"downstairs","session_id":"2772b901-e5b8-4f6f-9a66-dc5e25c56073"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:17.323297497Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1668,"client":"2","":"downstairs","session_id":"2772b901-e5b8-4f6f-9a66-dc5e25c56073"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:17.323319326Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1668,"":"io task","client":"2","":"downstairs","session_id":"2772b901-e5b8-4f6f-9a66-dc5e25c56073"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:17.323350525Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1668,"":"io task","client":"0","":"downstairs","session_id":"2772b901-e5b8-4f6f-9a66-dc5e25c56073"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:17.323383213Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1668,"":"io task","client":"1","":"downstairs","session_id":"2772b901-e5b8-4f6f-9a66-dc5e25c56073"} {"msg":"ds_connection connected from Ok(127.0.0.1:46546)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:17.323471519Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1668,"":"io task","client":"1","":"downstairs","session_id":"2772b901-e5b8-4f6f-9a66-dc5e25c56073"} {"msg":"ds_connection connected from Ok(127.0.0.1:63755)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:17.323520486Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1668,"":"io task","client":"2","":"downstairs","session_id":"2772b901-e5b8-4f6f-9a66-dc5e25c56073"} {"msg":"ds_connection connected from Ok(127.0.0.1:55710)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:17.323548205Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1668,"":"io task","client":"0","":"downstairs","session_id":"2772b901-e5b8-4f6f-9a66-dc5e25c56073"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:17.32426976Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1668,"client":"1","":"downstairs","session_id":"2772b901-e5b8-4f6f-9a66-dc5e25c56073"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:17.324314747Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1668,"client":"1","":"downstairs","session_id":"2772b901-e5b8-4f6f-9a66-dc5e25c56073"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:17.324382324Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1668,"client":"2","":"downstairs","session_id":"2772b901-e5b8-4f6f-9a66-dc5e25c56073"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:17.324419762Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1668,"client":"2","":"downstairs","session_id":"2772b901-e5b8-4f6f-9a66-dc5e25c56073"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:17.324451631Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1668,"client":"0","":"downstairs","session_id":"2772b901-e5b8-4f6f-9a66-dc5e25c56073"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:17.324480059Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1668,"client":"0","":"downstairs","session_id":"2772b901-e5b8-4f6f-9a66-dc5e25c56073"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:17.324972585Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1668,"client":"1","":"downstairs","session_id":"2772b901-e5b8-4f6f-9a66-dc5e25c56073"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:17.325006974Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1668,"client":"2","":"downstairs","session_id":"2772b901-e5b8-4f6f-9a66-dc5e25c56073"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:17.325036432Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1668,"client":"0","":"downstairs","session_id":"2772b901-e5b8-4f6f-9a66-dc5e25c56073"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:17.325560587Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1668,"client":"1","":"downstairs","session_id":"2772b901-e5b8-4f6f-9a66-dc5e25c56073"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:17.325600075Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1668,"client":"1","":"downstairs","session_id":"2772b901-e5b8-4f6f-9a66-dc5e25c56073"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:17.325621654Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1668,"session_id":"2772b901-e5b8-4f6f-9a66-dc5e25c56073"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:17.325635013Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1668,"client":"2","":"downstairs","session_id":"2772b901-e5b8-4f6f-9a66-dc5e25c56073"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:17.325648572Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1668,"client":"2","":"downstairs","session_id":"2772b901-e5b8-4f6f-9a66-dc5e25c56073"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:17.325661742Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1668,"session_id":"2772b901-e5b8-4f6f-9a66-dc5e25c56073"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:17.325673821Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1668,"client":"0","":"downstairs","session_id":"2772b901-e5b8-4f6f-9a66-dc5e25c56073"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:17.325687251Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1668,"client":"0","":"downstairs","session_id":"2772b901-e5b8-4f6f-9a66-dc5e25c56073"} {"msg":"[0]R flush_numbers[0..12]: [199, 192, 208, 212, 217, 219, 219, 217, 204, 219, 214, 219]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:17.32570413Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1668,"":"downstairs","":"downstairs","session_id":"2772b901-e5b8-4f6f-9a66-dc5e25c56073"} {"msg":"[0]R generation[0..12]: [112, 106, 116, 118, 119, 121, 121, 119, 114, 121, 118, 121]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:17.325719329Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1668,"":"downstairs","":"downstairs","session_id":"2772b901-e5b8-4f6f-9a66-dc5e25c56073"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:17.325733478Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1668,"":"downstairs","":"downstairs","session_id":"2772b901-e5b8-4f6f-9a66-dc5e25c56073"} {"msg":"[1]R flush_numbers[0..12]: [199, 192, 208, 212, 217, 219, 219, 217, 204, 219, 214, 219]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:17.325747268Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1668,"":"downstairs","":"downstairs","session_id":"2772b901-e5b8-4f6f-9a66-dc5e25c56073"} {"msg":"[1]R generation[0..12]: [112, 106, 116, 118, 119, 121, 121, 119, 114, 121, 118, 121]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:17.325760707Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1668,"":"downstairs","":"downstairs","session_id":"2772b901-e5b8-4f6f-9a66-dc5e25c56073"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:17.325774666Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1668,"":"downstairs","":"downstairs","session_id":"2772b901-e5b8-4f6f-9a66-dc5e25c56073"} {"msg":"[2]R flush_numbers[0..12]: [199, 192, 208, 212, 217, 219, 219, 217, 204, 219, 214, 219]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:17.325797265Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1668,"":"downstairs","":"downstairs","session_id":"2772b901-e5b8-4f6f-9a66-dc5e25c56073"} Wait for a query_work_queue command to finish before sending IO {"msg":"[2]R generation[0..12]: [112, 106, 116, 118, 119, 121, 121, 119, 114, 121, 118, 121]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:17.325832873Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1668,"":"downstairs","":"downstairs","session_id":"2772b901-e5b8-4f6f-9a66-dc5e25c56073"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:17.325856012Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1668,"":"downstairs","":"downstairs","session_id":"2772b901-e5b8-4f6f-9a66-dc5e25c56073"} {"msg":"Max found gen is 122","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:17.325870622Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1668,"":"downstairs","":"downstairs","session_id":"2772b901-e5b8-4f6f-9a66-dc5e25c56073"} {"msg":"Generation requested: 122 >= found:122","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:17.325884121Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1668,"":"downstairs","":"downstairs","session_id":"2772b901-e5b8-4f6f-9a66-dc5e25c56073"} {"msg":"Next flush: 220","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:17.32589704Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1668,"":"downstairs","":"downstairs","session_id":"2772b901-e5b8-4f6f-9a66-dc5e25c56073"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:17.32591004Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1668,"":"downstairs","":"downstairs","session_id":"2772b901-e5b8-4f6f-9a66-dc5e25c56073"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:17.325923149Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1668,"session_id":"2772b901-e5b8-4f6f-9a66-dc5e25c56073"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:17.325937598Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1668,"client":"0","":"downstairs","session_id":"2772b901-e5b8-4f6f-9a66-dc5e25c56073"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:17.325951307Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1668,"client":"1","":"downstairs","session_id":"2772b901-e5b8-4f6f-9a66-dc5e25c56073"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:17.325964507Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1668,"client":"2","":"downstairs","session_id":"2772b901-e5b8-4f6f-9a66-dc5e25c56073"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:17.325977776Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1668,"session_id":"2772b901-e5b8-4f6f-9a66-dc5e25c56073"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:17.325989906Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1668,"session_id":"2772b901-e5b8-4f6f-9a66-dc5e25c56073"} {"msg":"784848a0-a44a-4aba-9de8-88c2245745fc is now active with session: 2772b901-e5b8-4f6f-9a66-dc5e25c56073","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:17.326002535Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1668,"session_id":"2772b901-e5b8-4f6f-9a66-dc5e25c56073"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:17.326014555Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1668,"session_id":"2772b901-e5b8-4f6f-9a66-dc5e25c56073"} {"msg":"The guest has finished waiting for activation with:122","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:17.326026714Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1668} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 514 len 40960 data: 9 7 8 6 9 11 8 8 7 8 02/30 Write block 55 len 16384 data: 10 9 9 7 03/30 Read block 124 len 28672 04/30 Read block 28 len 20480 05/30 Read block 280 len 28672 06/30 Flush 07/30 Write block 375 len 12288 data: 8 8 10 08/30 Flush 09/30 Write block 278 len 32768 data: 9 9 10 7 9 9 9 10 10/30 Read block 431 len 40960 11/30 Write block 186 len 36864 data: 12 11 10 10 13 10 10 10 10 12/30 Write block 40 len 12288 data: 13 10 12 13/30 Flush 14/30 Write block 453 len 16384 data: 9 9 6 8 15/30 Write block 439 len 4096 data: 14 16/30 Write block 215 len 16384 data: 8 9 9 10 17/30 Read block 68 len 8192 18/30 Read block 120 len 32768 19/30 Flush 20/30 Write block 90 len 4096 data: 8 21/30 Write block 116 len 12288 data: 10 10 9 22/30 Write block 426 len 24576 data: 8 6 6 11 12 11 23/30 Write block 63 len 20480 data: 8 8 8 7 7 24/30 Write block 255 len 16384 data: 12 7 8 8 25/30 Flush 26/30 Write block 179 len 20480 data: 7 8 7 8 10 27/30 Read block 152 len 24576 28/30 Write block 106 len 32768 data: 8 11 9 10 10 7 7 8 29/30 Write block 172 len 28672 data: 6 5 5 6 5 7 6 30/30 Write block 97 len 20480 data: 4 3 4 6 7 ---------------------------------------------------------------- Crucible gen:122 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 5 Done Done Done false 1026 Acked Read 6 Done Done Done false 1027 Acked Write 8 Done Done Done false 1028 Acked Write 7 Sent Sent Sent false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:27:17.355054585Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1668} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 112 112 112 199 199 199 F F F 1 020-039 106 106 106 192 192 192 F F F 2 040-059 122 122 122 222 222 222 F F F 3 060-079 122 122 122 224 224 224 F F F 4 080-099 122 122 122 224 224 224 T T T 5 100-119 122 122 122 224 224 224 T T T 6 120-139 121 121 121 219 219 219 F F F 7 140-159 119 119 119 217 217 217 F F F 8 160-179 114 114 114 204 204 204 T T T 9 180-199 122 122 122 222 222 222 T T T 10 200-219 122 122 122 223 223 223 F F F 11 220-239 121 121 121 219 219 219 F F F 12 240-259 122 122 122 224 224 224 F F F 13 260-279 122 122 122 222 222 222 F F F 14 280-299 122 122 122 222 222 222 F F F 15 300-319 121 121 121 219 219 219 F F F 16 320-339 121 121 121 219 219 219 F F F 17 340-359 119 119 119 217 217 217 F F F 18 360-379 122 122 122 221 221 221 F F F 19 380-399 120 120 120 218 218 218 F F F 20 400-419 116 116 116 206 206 206 F F F 21 420-439 122 122 122 224 224 224 F F F 22 440-459 122 122 122 223 223 223 F F F 23 460-479 118 118 118 212 212 212 F F F 24 480-499 118 118 118 212 212 212 F F F 25 500-519 122 122 122 220 220 220 F F F 26 520-539 122 122 122 220 220 220 F F F 27 540-559 116 116 116 209 209 209 F F F 28 560-579 116 116 116 209 209 209 F F F 29 580-599 121 121 121 219 219 219 F F F Max gen: 122, Max flush: 224 A Difference in extent metadata is expected here On loop 61 Verifying data now No change after verify Loop: 61 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 112 112 112 199 199 199 F F F 1 020-039 106 106 106 192 192 192 F F F 2 040-059 122 122 122 222 222 222 F F F 3 060-079 122 122 122 224 224 224 F F F 4 080-099 123 123 123 225 225 225 F F F 5 100-119 123 123 123 225 225 225 F F F 6 120-139 121 121 121 219 219 219 F F F 7 140-159 119 119 119 217 217 217 F F F 8 160-179 123 123 123 225 225 225 F F F 9 180-199 123 123 123 225 225 225 F F F 10 200-219 122 122 122 223 223 223 F F F 11 220-239 121 121 121 219 219 219 F F F 12 240-259 122 122 122 224 224 224 F F F 13 260-279 122 122 122 222 222 222 F F F 14 280-299 122 122 122 222 222 222 F F F 15 300-319 121 121 121 219 219 219 F F F 16 320-339 121 121 121 219 219 219 F F F 17 340-359 119 119 119 217 217 217 F F F 18 360-379 122 122 122 221 221 221 F F F 19 380-399 120 120 120 218 218 218 F F F 20 400-419 116 116 116 206 206 206 F F F 21 420-439 122 122 122 224 224 224 F F F 22 440-459 122 122 122 223 223 223 F F F 23 460-479 118 118 118 212 212 212 F F F 24 480-499 118 118 118 212 212 212 F F F 25 500-519 122 122 122 220 220 220 F F F 26 520-539 122 122 122 220 220 220 F F F 27 540-559 116 116 116 209 209 209 F F F 28 560-579 116 116 116 209 209 209 F F F 29 580-599 121 121 121 219 219 219 F F F Max gen: 123, Max flush: 225 Begin loop 62 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:21.691162082Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1676,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:27:21.734001559Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1676,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:21.734406339Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1676,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:21.766245293Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1676,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:21.766814745Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1676} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"0fd98d0d5d909611f8b408ccda2559d35fe117bb\",\n git_commit_timestamp: \"2024-11-08T20:05:44.000000000Z\",\n git_branch: \"mkeeter/drop-skipped-backpressure-guard\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:21.766851283Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1676} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:21.766865953Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1676} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:21.766880412Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1676} {"msg":"Crucible 5165806e-fc5b-4cc4-bf66-53f6b2e0a6fe has session id: 20cfa6d9-5981-4990-bde2-3ce3688654c1","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:21.767353958Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1676,"session_id":"20cfa6d9-5981-4990-bde2-3ce3688654c1"} {"msg":"Upstairs opts: Upstairs UUID: 5165806e-fc5b-4cc4-bf66-53f6b2e0a6fe, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:21.767389397Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1676,"session_id":"20cfa6d9-5981-4990-bde2-3ce3688654c1"} {"msg":"Crucible stats registered with UUID: 5165806e-fc5b-4cc4-bf66-53f6b2e0a6fe","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:21.767406566Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1676,"session_id":"20cfa6d9-5981-4990-bde2-3ce3688654c1"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:21.798234539Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1676,"task":"crutest"} {"msg":"The guest has requested activation with gen:124","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:21.798270687Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1676} {"msg":"5165806e-fc5b-4cc4-bf66-53f6b2e0a6fe active request set","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:21.798336474Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1676,"session_id":"20cfa6d9-5981-4990-bde2-3ce3688654c1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:21.798376222Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1676,"client":"0","":"downstairs","session_id":"20cfa6d9-5981-4990-bde2-3ce3688654c1"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:21.79840728Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1676,"client":"0","":"downstairs","session_id":"20cfa6d9-5981-4990-bde2-3ce3688654c1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:21.79842573Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1676,"client":"1","":"downstairs","session_id":"20cfa6d9-5981-4990-bde2-3ce3688654c1"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:21.798442878Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1676,"client":"1","":"downstairs","session_id":"20cfa6d9-5981-4990-bde2-3ce3688654c1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:21.798459678Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1676,"client":"2","":"downstairs","session_id":"20cfa6d9-5981-4990-bde2-3ce3688654c1"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:21.798476377Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1676,"client":"2","":"downstairs","session_id":"20cfa6d9-5981-4990-bde2-3ce3688654c1"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:21.798494716Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1676,"":"io task","client":"0","":"downstairs","session_id":"20cfa6d9-5981-4990-bde2-3ce3688654c1"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:21.798521425Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1676,"":"io task","client":"2","":"downstairs","session_id":"20cfa6d9-5981-4990-bde2-3ce3688654c1"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:21.798581032Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1676,"":"io task","client":"1","":"downstairs","session_id":"20cfa6d9-5981-4990-bde2-3ce3688654c1"} {"msg":"ds_connection connected from Ok(127.0.0.1:37058)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:21.798767113Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1676,"":"io task","client":"1","":"downstairs","session_id":"20cfa6d9-5981-4990-bde2-3ce3688654c1"} {"msg":"ds_connection connected from Ok(127.0.0.1:45703)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:21.79881697Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1676,"":"io task","client":"2","":"downstairs","session_id":"20cfa6d9-5981-4990-bde2-3ce3688654c1"} {"msg":"ds_connection connected from Ok(127.0.0.1:63150)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:21.798852029Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1676,"":"io task","client":"0","":"downstairs","session_id":"20cfa6d9-5981-4990-bde2-3ce3688654c1"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:21.799428181Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1676,"client":"1","":"downstairs","session_id":"20cfa6d9-5981-4990-bde2-3ce3688654c1"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:21.799466488Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1676,"client":"1","":"downstairs","session_id":"20cfa6d9-5981-4990-bde2-3ce3688654c1"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:21.799558994Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1676,"client":"2","":"downstairs","session_id":"20cfa6d9-5981-4990-bde2-3ce3688654c1"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:21.799588053Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1676,"client":"2","":"downstairs","session_id":"20cfa6d9-5981-4990-bde2-3ce3688654c1"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:21.799602852Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1676,"client":"0","":"downstairs","session_id":"20cfa6d9-5981-4990-bde2-3ce3688654c1"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:21.799672458Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1676,"client":"0","":"downstairs","session_id":"20cfa6d9-5981-4990-bde2-3ce3688654c1"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:21.800107777Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1676,"client":"1","":"downstairs","session_id":"20cfa6d9-5981-4990-bde2-3ce3688654c1"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:21.800137886Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1676,"client":"2","":"downstairs","session_id":"20cfa6d9-5981-4990-bde2-3ce3688654c1"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:21.800154265Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1676,"client":"0","":"downstairs","session_id":"20cfa6d9-5981-4990-bde2-3ce3688654c1"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:21.800687169Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1676,"client":"1","":"downstairs","session_id":"20cfa6d9-5981-4990-bde2-3ce3688654c1"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:21.800723957Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1676,"client":"1","":"downstairs","session_id":"20cfa6d9-5981-4990-bde2-3ce3688654c1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:21.800750886Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1676,"session_id":"20cfa6d9-5981-4990-bde2-3ce3688654c1"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:21.800767625Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1676,"client":"2","":"downstairs","session_id":"20cfa6d9-5981-4990-bde2-3ce3688654c1"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:21.800781404Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1676,"client":"2","":"downstairs","session_id":"20cfa6d9-5981-4990-bde2-3ce3688654c1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:21.800794624Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1676,"session_id":"20cfa6d9-5981-4990-bde2-3ce3688654c1"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:21.800806863Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1676,"client":"0","":"downstairs","session_id":"20cfa6d9-5981-4990-bde2-3ce3688654c1"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:21.800820312Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1676,"client":"0","":"downstairs","session_id":"20cfa6d9-5981-4990-bde2-3ce3688654c1"} {"msg":"[0]R flush_numbers[0..12]: [199, 192, 222, 224, 225, 225, 219, 217, 225, 225, 223, 219]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:21.800834192Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1676,"":"downstairs","":"downstairs","session_id":"20cfa6d9-5981-4990-bde2-3ce3688654c1"} {"msg":"[0]R generation[0..12]: [112, 106, 122, 122, 123, 123, 121, 119, 123, 123, 122, 121]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:21.800851531Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1676,"":"downstairs","":"downstairs","session_id":"20cfa6d9-5981-4990-bde2-3ce3688654c1"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:21.80086613Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1676,"":"downstairs","":"downstairs","session_id":"20cfa6d9-5981-4990-bde2-3ce3688654c1"} {"msg":"[1]R flush_numbers[0..12]: [199, 192, 222, 224, 225, 225, 219, 217, 225, 225, 223, 219]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:21.80088011Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1676,"":"downstairs","":"downstairs","session_id":"20cfa6d9-5981-4990-bde2-3ce3688654c1"} {"msg":"[1]R generation[0..12]: [112, 106, 122, 122, 123, 123, 121, 119, 123, 123, 122, 121]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:21.800894099Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1676,"":"downstairs","":"downstairs","session_id":"20cfa6d9-5981-4990-bde2-3ce3688654c1"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:21.800908168Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1676,"":"downstairs","":"downstairs","session_id":"20cfa6d9-5981-4990-bde2-3ce3688654c1"} {"msg":"[2]R flush_numbers[0..12]: [199, 192, 222, 224, 225, 225, 219, 217, 225, 225, 223, 219]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:21.800926177Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1676,"":"downstairs","":"downstairs","session_id":"20cfa6d9-5981-4990-bde2-3ce3688654c1"} {"msg":"[2]R generation[0..12]: [112, 106, 122, 122, 123, 123, 121, 119, 123, 123, 122, 121]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-08T20:27:21.800951946Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1676,"":"downstairs","":"downstairs","session_id":"20cfa6d9-5981-4990-bde2-3ce3688654c1"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:21.801027212Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1676,"":"downstairs","":"downstairs","session_id":"20cfa6d9-5981-4990-bde2-3ce3688654c1"} {"msg":"Max found gen is 124","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:21.801045761Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1676,"":"downstairs","":"downstairs","session_id":"20cfa6d9-5981-4990-bde2-3ce3688654c1"} {"msg":"Generation requested: 124 >= found:124","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:21.801059181Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1676,"":"downstairs","":"downstairs","session_id":"20cfa6d9-5981-4990-bde2-3ce3688654c1"} {"msg":"Next flush: 226","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:21.80107212Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1676,"":"downstairs","":"downstairs","session_id":"20cfa6d9-5981-4990-bde2-3ce3688654c1"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:21.80108484Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1676,"":"downstairs","":"downstairs","session_id":"20cfa6d9-5981-4990-bde2-3ce3688654c1"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:21.801097919Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1676,"session_id":"20cfa6d9-5981-4990-bde2-3ce3688654c1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:21.801109948Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1676,"client":"0","":"downstairs","session_id":"20cfa6d9-5981-4990-bde2-3ce3688654c1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:21.801123207Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1676,"client":"1","":"downstairs","session_id":"20cfa6d9-5981-4990-bde2-3ce3688654c1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:21.801139247Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1676,"client":"2","":"downstairs","session_id":"20cfa6d9-5981-4990-bde2-3ce3688654c1"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:21.801152856Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1676,"session_id":"20cfa6d9-5981-4990-bde2-3ce3688654c1"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:21.801165176Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1676,"session_id":"20cfa6d9-5981-4990-bde2-3ce3688654c1"} {"msg":"5165806e-fc5b-4cc4-bf66-53f6b2e0a6fe is now active with session: 20cfa6d9-5981-4990-bde2-3ce3688654c1","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:21.801178045Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1676,"session_id":"20cfa6d9-5981-4990-bde2-3ce3688654c1"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:21.801190094Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1676,"session_id":"20cfa6d9-5981-4990-bde2-3ce3688654c1"} {"msg":"The guest has finished waiting for activation with:124","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:21.801202194Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1676} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 366 len 8192 data: 9 11 02/30 Write block 295 len 40960 data: 11 8 10 10 13 12 13 12 12 10 03/30 Write block 597 len 8192 data: 5 3 04/30 Read block 330 len 12288 05/30 Write block 107 len 16384 data: 12 10 11 11 06/30 Read block 231 len 20480 07/30 Read block 486 len 32768 08/30 Read block 7 len 8192 09/30 Write block 124 len 16384 data: 12 9 10 8 10/30 Write block 174 len 24576 data: 6 7 6 8 7 8 11/30 Flush 12/30 Write block 80 len 24576 data: 8 8 8 10 9 11 13/30 Flush 14/30 Write block 307 len 24576 data: 3 5 7 8 10 11 15/30 Write block 29 len 4096 data: 6 16/30 Read block 141 len 8192 17/30 Write block 308 len 36864 data: 6 8 9 11 12 13 11 13 11 18/30 Read block 124 len 40960 19/30 Write block 149 len 40960 data: 10 10 12 9 8 8 6 5 5 5 20/30 Read block 423 len 24576 21/30 Flush 22/30 Write block 212 len 20480 data: 9 10 9 9 10 23/30 Write block 424 len 8192 data: 8 9 24/30 Read block 24 len 20480 25/30 Read block 84 len 16384 26/30 Read block 482 len 40960 27/30 Read block 96 len 28672 28/30 Read block 467 len 20480 29/30 Write block 360 len 20480 data: 17 18 15 13 12 30/30 Write block 154 len 12288 data: 9 7 6 ---------------------------------------------------------------- Crucible gen:124 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 5 Done Done Done false 1022 Acked Write 2 Done Done Done false 1023 Acked Read 5 Done Done Done false 1024 Acked Read 4 Done Done Done false 1025 Acked Read 10 Done Done Done false 1026 Acked Read 7 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Write 5 Sent Sent Sent false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 7 7 7 21 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1020 1020 1020 Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:27:21.829679232Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1676} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 112 112 112 199 199 199 F F F 1 020-039 124 124 124 228 228 228 F F F 2 040-059 122 122 122 222 222 222 F F F 3 060-079 122 122 122 224 224 224 F F F 4 080-099 124 124 124 227 227 227 F F F 5 100-119 124 124 124 226 226 226 F F F 6 120-139 124 124 124 226 226 226 F F F 7 140-159 124 124 124 228 228 228 T T T 8 160-179 124 124 124 226 226 226 F F F 9 180-199 123 123 123 225 225 225 F F F 10 200-219 122 122 122 223 223 223 T T T 11 220-239 121 121 121 219 219 219 F F F 12 240-259 122 122 122 224 224 224 F F F 13 260-279 122 122 122 222 222 222 F F F 14 280-299 124 124 124 226 226 226 F F F 15 300-319 124 124 124 228 228 228 F F F 16 320-339 121 121 121 219 219 219 F F F 17 340-359 119 119 119 217 217 217 F F F 18 360-379 124 124 124 226 226 226 T T T 19 380-399 120 120 120 218 218 218 F F F 20 400-419 116 116 116 206 206 206 F F F 21 420-439 122 122 122 224 224 224 T T T 22 440-459 122 122 122 223 223 223 F F F 23 460-479 118 118 118 212 212 212 F F F 24 480-499 118 118 118 212 212 212 F F F 25 500-519 122 122 122 220 220 220 F F F 26 520-539 122 122 122 220 220 220 F F F 27 540-559 116 116 116 209 209 209 F F F 28 560-579 116 116 116 209 209 209 F F F 29 580-599 124 124 124 226 226 226 F F F Max gen: 124, Max flush: 228 A Difference in extent metadata is expected here On loop 62 Verifying data now No change after verify Loop: 62 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 112 112 112 199 199 199 F F F 1 020-039 124 124 124 228 228 228 F F F 2 040-059 122 122 122 222 222 222 F F F 3 060-079 122 122 122 224 224 224 F F F 4 080-099 124 124 124 227 227 227 F F F 5 100-119 124 124 124 226 226 226 F F F 6 120-139 124 124 124 226 226 226 F F F 7 140-159 125 125 125 229 229 229 F F F 8 160-179 124 124 124 226 226 226 F F F 9 180-199 123 123 123 225 225 225 F F F 10 200-219 125 125 125 229 229 229 F F F 11 220-239 121 121 121 219 219 219 F F F 12 240-259 122 122 122 224 224 224 F F F 13 260-279 122 122 122 222 222 222 F F F 14 280-299 124 124 124 226 226 226 F F F 15 300-319 124 124 124 228 228 228 F F F 16 320-339 121 121 121 219 219 219 F F F 17 340-359 119 119 119 217 217 217 F F F 18 360-379 125 125 125 229 229 229 F F F 19 380-399 120 120 120 218 218 218 F F F 20 400-419 116 116 116 206 206 206 F F F 21 420-439 125 125 125 229 229 229 F F F 22 440-459 122 122 122 223 223 223 F F F 23 460-479 118 118 118 212 212 212 F F F 24 480-499 118 118 118 212 212 212 F F F 25 500-519 122 122 122 220 220 220 F F F 26 520-539 122 122 122 220 220 220 F F F 27 540-559 116 116 116 209 209 209 F F F 28 560-579 116 116 116 209 209 209 F F F 29 580-599 124 124 124 226 226 226 F F F Max gen: 125, Max flush: 229 Begin loop 63 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:26.174313672Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1684,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:26.219026177Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1684,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:26.219737562Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1684} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"0fd98d0d5d909611f8b408ccda2559d35fe117bb\",\n git_commit_timestamp: \"2024-11-08T20:05:44.000000000Z\",\n git_branch: \"mkeeter/drop-skipped-backpressure-guard\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:26.21977493Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1684} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:26.219789499Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1684} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:26.219803748Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1684} {"msg":"Crucible 32b54e7c-a69a-457a-92b5-473b1444b98d has session id: 1fd8edfb-312a-4e5a-bd09-4a2f75732774","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:26.220273205Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1684,"session_id":"1fd8edfb-312a-4e5a-bd09-4a2f75732774"} {"msg":"Upstairs opts: Upstairs UUID: 32b54e7c-a69a-457a-92b5-473b1444b98d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:26.220306654Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1684,"session_id":"1fd8edfb-312a-4e5a-bd09-4a2f75732774"} {"msg":"Crucible stats registered with UUID: 32b54e7c-a69a-457a-92b5-473b1444b98d","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:26.220322553Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1684,"session_id":"1fd8edfb-312a-4e5a-bd09-4a2f75732774"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:26.251756367Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1684,"task":"crutest"} {"msg":"The guest has requested activation with gen:126","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:26.251801564Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1684} {"msg":"32b54e7c-a69a-457a-92b5-473b1444b98d active request set","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:26.251836933Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1684,"session_id":"1fd8edfb-312a-4e5a-bd09-4a2f75732774"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:26.251867521Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1684,"client":"0","":"downstairs","session_id":"1fd8edfb-312a-4e5a-bd09-4a2f75732774"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:26.251915589Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1684,"client":"0","":"downstairs","session_id":"1fd8edfb-312a-4e5a-bd09-4a2f75732774"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:26.251952787Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1684,"client":"1","":"downstairs","session_id":"1fd8edfb-312a-4e5a-bd09-4a2f75732774"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:26.251991525Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1684,"client":"1","":"downstairs","session_id":"1fd8edfb-312a-4e5a-bd09-4a2f75732774"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:26.252025154Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1684,"client":"2","":"downstairs","session_id":"1fd8edfb-312a-4e5a-bd09-4a2f75732774"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:26.252050003Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1684,"client":"2","":"downstairs","session_id":"1fd8edfb-312a-4e5a-bd09-4a2f75732774"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:26.252069391Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1684,"":"io task","client":"2","":"downstairs","session_id":"1fd8edfb-312a-4e5a-bd09-4a2f75732774"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:26.252090541Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1684,"":"io task","client":"0","":"downstairs","session_id":"1fd8edfb-312a-4e5a-bd09-4a2f75732774"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:26.25210667Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1684,"":"io task","client":"1","":"downstairs","session_id":"1fd8edfb-312a-4e5a-bd09-4a2f75732774"} {"msg":"ds_connection connected from Ok(127.0.0.1:62976)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:26.252121009Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1684,"":"io task","client":"0","":"downstairs","session_id":"1fd8edfb-312a-4e5a-bd09-4a2f75732774"} {"msg":"ds_connection connected from Ok(127.0.0.1:54636)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:26.252349868Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1684,"":"io task","client":"1","":"downstairs","session_id":"1fd8edfb-312a-4e5a-bd09-4a2f75732774"} {"msg":"ds_connection connected from Ok(127.0.0.1:51512)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:26.252390886Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1684,"":"io task","client":"2","":"downstairs","session_id":"1fd8edfb-312a-4e5a-bd09-4a2f75732774"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:26.252941819Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1684,"client":"2","":"downstairs","session_id":"1fd8edfb-312a-4e5a-bd09-4a2f75732774"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:26.252982967Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1684,"client":"2","":"downstairs","session_id":"1fd8edfb-312a-4e5a-bd09-4a2f75732774"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:26.253014315Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1684,"client":"0","":"downstairs","session_id":"1fd8edfb-312a-4e5a-bd09-4a2f75732774"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:26.253044134Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1684,"client":"0","":"downstairs","session_id":"1fd8edfb-312a-4e5a-bd09-4a2f75732774"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:26.253075532Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1684,"client":"1","":"downstairs","session_id":"1fd8edfb-312a-4e5a-bd09-4a2f75732774"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:26.253095621Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1684,"client":"1","":"downstairs","session_id":"1fd8edfb-312a-4e5a-bd09-4a2f75732774"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:26.253623826Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1684,"client":"2","":"downstairs","session_id":"1fd8edfb-312a-4e5a-bd09-4a2f75732774"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:26.253653704Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1684,"client":"0","":"downstairs","session_id":"1fd8edfb-312a-4e5a-bd09-4a2f75732774"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:26.253680773Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1684,"client":"1","":"downstairs","session_id":"1fd8edfb-312a-4e5a-bd09-4a2f75732774"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:26.254208487Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1684,"client":"1","":"downstairs","session_id":"1fd8edfb-312a-4e5a-bd09-4a2f75732774"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:26.254248055Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1684,"client":"1","":"downstairs","session_id":"1fd8edfb-312a-4e5a-bd09-4a2f75732774"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:26.254271024Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1684,"session_id":"1fd8edfb-312a-4e5a-bd09-4a2f75732774"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:26.254283963Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1684,"client":"2","":"downstairs","session_id":"1fd8edfb-312a-4e5a-bd09-4a2f75732774"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:26.254297442Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1684,"client":"2","":"downstairs","session_id":"1fd8edfb-312a-4e5a-bd09-4a2f75732774"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:26.254310632Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1684,"session_id":"1fd8edfb-312a-4e5a-bd09-4a2f75732774"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:26.254322701Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1684,"client":"0","":"downstairs","session_id":"1fd8edfb-312a-4e5a-bd09-4a2f75732774"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:26.254335991Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1684,"client":"0","":"downstairs","session_id":"1fd8edfb-312a-4e5a-bd09-4a2f75732774"} {"msg":"[0]R flush_numbers[0..12]: [199, 228, 222, 224, 227, 226, 226, 229, 226, 225, 229, 219]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:26.25434966Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1684,"":"downstairs","":"downstairs","session_id":"1fd8edfb-312a-4e5a-bd09-4a2f75732774"} {"msg":"[0]R generation[0..12]: [112, 124, 122, 122, 124, 124, 124, 125, 124, 123, 125, 121]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:26.254363629Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1684,"":"downstairs","":"downstairs","session_id":"1fd8edfb-312a-4e5a-bd09-4a2f75732774"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:26.254377709Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1684,"":"downstairs","":"downstairs","session_id":"1fd8edfb-312a-4e5a-bd09-4a2f75732774"} {"msg":"[1]R flush_numbers[0..12]: [199, 228, 222, 224, 227, 226, 226, 229, 226, 225, 229, 219]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:26.254391438Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1684,"":"downstairs","":"downstairs","session_id":"1fd8edfb-312a-4e5a-bd09-4a2f75732774"} {"msg":"[1]R generation[0..12]: [112, 124, 122, 122, 124, 124, 124, 125, 124, 123, 125, 121]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:26.254409497Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1684,"":"downstairs","":"downstairs","session_id":"1fd8edfb-312a-4e5a-bd09-4a2f75732774"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:26.254423946Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1684,"":"downstairs","":"downstairs","session_id":"1fd8edfb-312a-4e5a-bd09-4a2f75732774"} {"msg":"[2]R flush_numbers[0..12]: [199, 228, 222, 224, 227, 226, 226, 229, 226, 225, 229, 219]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:26.254437536Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1684,"":"downstairs","":"downstairs","session_id":"1fd8edfb-312a-4e5a-bd09-4a2f75732774"} {"msg":"[2]R generation[0..12]: [112, 124, 122, 122, 124, 124, 124, 125, 124, 123, 125, 121]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:26.254451135Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1684,"":"downstairs","":"downstairs","session_id":"1fd8edfb-312a-4e5a-bd09-4a2f75732774"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:26.254465244Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1684,"":"downstairs","":"downstairs","session_id":"1fd8edfb-312a-4e5a-bd09-4a2f75732774"} Wait for a query_work_queue command to finish before sending IO {"msg":"Max found gen is 126","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:26.254485443Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1684,"":"downstairs","":"downstairs","session_id":"1fd8edfb-312a-4e5a-bd09-4a2f75732774"} {"msg":"Generation requested: 126 >= found:126","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:26.254498753Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1684,"":"downstairs","":"downstairs","session_id":"1fd8edfb-312a-4e5a-bd09-4a2f75732774"} {"msg":"Next flush: 230","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:26.254511732Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1684,"":"downstairs","":"downstairs","session_id":"1fd8edfb-312a-4e5a-bd09-4a2f75732774"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:26.254524702Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1684,"":"downstairs","":"downstairs","session_id":"1fd8edfb-312a-4e5a-bd09-4a2f75732774"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:26.254537641Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1684,"session_id":"1fd8edfb-312a-4e5a-bd09-4a2f75732774"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:26.25454965Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1684,"client":"0","":"downstairs","session_id":"1fd8edfb-312a-4e5a-bd09-4a2f75732774"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:26.25456281Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1684,"client":"1","":"downstairs","session_id":"1fd8edfb-312a-4e5a-bd09-4a2f75732774"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:26.254575809Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1684,"client":"2","":"downstairs","session_id":"1fd8edfb-312a-4e5a-bd09-4a2f75732774"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:26.254589178Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1684,"session_id":"1fd8edfb-312a-4e5a-bd09-4a2f75732774"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:26.254601278Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1684,"session_id":"1fd8edfb-312a-4e5a-bd09-4a2f75732774"} {"msg":"32b54e7c-a69a-457a-92b5-473b1444b98d is now active with session: 1fd8edfb-312a-4e5a-bd09-4a2f75732774","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:26.254615847Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1684,"session_id":"1fd8edfb-312a-4e5a-bd09-4a2f75732774"} {"msg":"Set Active after no reconciliation","v":0,"Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 name":"crucible","level":30,"time":"2024-11-08T20:27:26.254631276Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1684,"session_id":"1fd8edfb-312a-4e5a-bd09-4a2f75732774"} {"msg":"The guest has finished waiting for activation with:126","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:26.254643696Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1684} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 167 len 40960 data: 6 11 11 12 11 7 6 7 8 7 02/30 Read block 297 len 36864 03/30 Read block 125 len 4096 04/30 Read block 159 len 12288 05/30 Read block 467 len 36864 06/30 Write block 422 len 40960 data: 9 9 9 10 9 7 7 12 13 12 07/30 Flush 08/30 Write block 184 len 8192 data: 8 8 09/30 Read block 497 len 12288 10/30 Write block 390 len 8192 data: 8 7 11/30 Read block 422 len 36864 12/30 Read block 148 len 8192 13/30 Read block 95 len 8192 14/30 Write block 197 len 8192 data: 10 10 15/30 Write block 410 len 32768 data: 14 13 12 11 10 9 9 9 16/30 Read block 217 len 24576 17/30 Flush 18/30 Write block 480 len 32768 data: 13 14 13 12 11 9 9 8 19/30 Write block 362 len 40960 data: 16 14 13 10 10 12 12 10 9 10 20/30 Write block 147 len 36864 data: 10 9 11 11 13 10 9 10 8 21/30 Write block 223 len 20480 data: 10 10 10 11 13 22/30 Write block 55 len 4096 data: 11 23/30 Read block 165 len 32768 24/30 Read block 141 len 32768 25/30 Write block 552 len 40960 data: 7 6 7 8 8 8 8 9 8 8 26/30 Write block 571 len 12288 data: 11 13 14 27/30 Read block 162 len 28672 28/30 Write block 506 len 28672 data: 8 7 10 9 10 11 9 29/30 Write block 241 len 20480 data: 10 12 11 10 9 30/30 Read block 410 len 4096 ---------------------------------------------------------------- Crucible gen:126 GIO:true work queues: Upstairs:0 downstairs:13 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1017 Acked Write 8 Done Done Done false 1018 Acked Write 10 Done Done Done false 1019 Acked Write 9 Done Done Done false 1020 Acked Write 5 Done Done Done false 1021 Acked Write 1 Done Done Done false 1022 Acked Read 8 Done Done Done false 1023 Acked Read 8 Done Done Done false 1024 Acked Write 10 Done Done Done false 1025 Acked Write 3 Done Done Done false 1026 Acked Read 7 Done Done Done false 1027 Acked Write 7 Done Done Done false 1028 Acked Write 5 Sent Done Sent false 1029 Acked Read 1 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 0 2 4 Done 11 13 11 35 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1016 1016 1016 Downstairs last five completed: 1016 1015 1014 1013 1012 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:27:26.279161547Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1684} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 112 112 112 199 199 199 F F F 1 020-039 124 124 124 228 228 228 F F F 2 040-059 122 122 122 222 222 222 T T T 3 060-079 122 122 122 224 224 224 F F F 4 080-099 124 124 124 227 227 227 F F F 5 100-119 124 124 124 226 226 226 F F F 6 120-139 124 124 124 226 226 226 F F F 7 140-159 125 125 125 229 229 229 T T T 8 160-179 126 126 126 230 230 230 F F F 9 180-199 126 126 126 231 231 231 F F F 10 200-219 125 125 125 229 229 229 F F F 11 220-239 121 121 121 219 219 219 T T T 12 240-259 122 122 122 224 224 224 T T T 13 260-279 122 122 122 222 222 222 F F F 14 280-299 124 124 124 226 226 226 F F F 15 300-319 124 124 124 228 228 228 F F F 16 320-339 121 121 121 219 219 219 F F F 17 340-359 119 119 119 217 217 217 F F F 18 360-379 125 125 125 229 229 229 T T T 19 380-399 126 126 126 231 231 231 F F F 20 400-419 126 126 126 231 231 231 F F F 21 420-439 126 126 126 230 230 230 F F F 22 440-459 122 122 122 223 223 223 F F F 23 460-479 118 118 118 212 212 212 F F F 24 480-499 118 118 118 212 212 212 T T T 25 500-519 122 122 122 220 220 220 T T T 26 520-539 122 122 122 220 220 220 F F F 27 540-559 116 116 116 209 209 209 T T T 28 560-579 116 116 116 209 209 209 T T T 29 580-599 124 124 124 226 226 226 F F F Max gen: 126, Max flush: 231 A Difference in extent metadata is expected here On loop 63 Verifying data now No change after verify Loop: 63 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 112 112 112 199 199 199 F F F 1 020-039 124 124 124 228 228 228 F F F 2 040-059 127 127 127 232 232 232 F F F 3 060-079 122 122 122 224 224 224 F F F 4 080-099 124 124 124 227 227 227 F F F 5 100-119 124 124 124 226 226 226 F F F 6 120-139 124 124 124 226 226 226 F F F 7 140-159 127 127 127 232 232 232 F F F 8 160-179 126 126 126 230 230 230 F F F 9 180-199 126 126 126 231 231 231 F F F 10 200-219 125 125 125 229 229 229 F F F 11 220-239 127 127 127 232 232 232 F F F 12 240-259 127 127 127 232 232 232 F F F 13 260-279 122 122 122 222 222 222 F F F 14 280-299 124 124 124 226 226 226 F F F 15 300-319 124 124 124 228 228 228 F F F 16 320-339 121 121 121 219 219 219 F F F 17 340-359 119 119 119 217 217 217 F F F 18 360-379 127 127 127 232 232 232 F F F 19 380-399 126 126 126 231 231 231 F F F 20 400-419 126 126 126 231 231 231 F F F 21 420-439 126 126 126 230 230 230 F F F 22 440-459 122 122 122 223 223 223 F F F 23 460-479 118 118 118 212 212 212 F F F 24 480-499 127 127 127 232 232 232 F F F 25 500-519 127 127 127 232 232 232 F F F 26 520-539 122 122 122 220 220 220 F F F 27 540-559 127 127 127 232 232 232 F F F 28 560-579 127 127 127 232 232 232 F F F 29 580-599 124 124 124 226 226 226 F F F Max gen: 127, Max flush: 232 Begin loop 64 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:30.668652605Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1692,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:30.715963123Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1692,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:30.716627461Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1692} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"0fd98d0d5d909611f8b408ccda2559d35fe117bb\",\n git_commit_timestamp: \"2024-11-08T20:05:44.000000000Z\",\n git_branch: \"mkeeter/drop-skipped-backpressure-guard\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:30.716682308Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1692} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:30.716711226Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1692} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:30.716739215Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1692} {"msg":"Crucible 83ab84d6-89ef-4f70-8ada-ea2ba5c734df has session id: 8b307ab1-da4b-4b47-ab33-170d01a37ce8","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:30.717142605Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1692,"session_id":"8b307ab1-da4b-4b47-ab33-170d01a37ce8"} {"msg":"Upstairs opts: Upstairs UUID: 83ab84d6-89ef-4f70-8ada-ea2ba5c734df, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:30.717185033Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1692,"session_id":"8b307ab1-da4b-4b47-ab33-170d01a37ce8"} {"msg":"Crucible stats registered with UUID: 83ab84d6-89ef-4f70-8ada-ea2ba5c734df","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:30.717202812Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1692,"session_id":"8b307ab1-da4b-4b47-ab33-170d01a37ce8"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:30.748221356Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1692,"task":"crutest"} {"msg":"The guest has requested activation with gen:128","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:30.748253035Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1692} {"msg":"83ab84d6-89ef-4f70-8ada-ea2ba5c734df active request set","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:30.748304112Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1692,"session_id":"8b307ab1-da4b-4b47-ab33-170d01a37ce8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:30.748323791Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1692,"client":"0","":"downstairs","session_id":"8b307ab1-da4b-4b47-ab33-170d01a37ce8"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:30.74835069Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1692,"client":"0","":"downstairs","session_id":"8b307ab1-da4b-4b47-ab33-170d01a37ce8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:30.748368439Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1692,"client":"1","":"downstairs","session_id":"8b307ab1-da4b-4b47-ab33-170d01a37ce8"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:30.748388378Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1692,"client":"1","":"downstairs","session_id":"8b307ab1-da4b-4b47-ab33-170d01a37ce8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:30.748424287Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1692,"client":"2","":"downstairs","session_id":"8b307ab1-da4b-4b47-ab33-170d01a37ce8"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:30.748457375Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1692,"client":"2","":"downstairs","session_id":"8b307ab1-da4b-4b47-ab33-170d01a37ce8"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:30.748478834Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1692,"":"io task","client":"2","":"downstairs","session_id":"8b307ab1-da4b-4b47-ab33-170d01a37ce8"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:30.748503943Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1692,"":"io task","client":"0","":"downstairs","session_id":"8b307ab1-da4b-4b47-ab33-170d01a37ce8"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:30.748535851Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1692,"":"io task","client":"1","":"downstairs","session_id":"8b307ab1-da4b-4b47-ab33-170d01a37ce8"} {"msg":"ds_connection connected from Ok(127.0.0.1:38017)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:30.748591838Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1692,"":"io task","client":"2","":"downstairs","session_id":"8b307ab1-da4b-4b47-ab33-170d01a37ce8"} {"msg":"ds_connection connected from Ok(127.0.0.1:33432)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:30.748630436Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1692,"":"io task","client":"0","":"downstairs","session_id":"8b307ab1-da4b-4b47-ab33-170d01a37ce8"} {"msg":"ds_connection connected from Ok(127.0.0.1:49485)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:30.748660495Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1692,"":"io task","client":"1","":"downstairs","session_id":"8b307ab1-da4b-4b47-ab33-170d01a37ce8"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:30.749353671Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1692,"client":"2","":"downstairs","session_id":"8b307ab1-da4b-4b47-ab33-170d01a37ce8"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:30.749383329Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1692,"client":"2","":"downstairs","session_id":"8b307ab1-da4b-4b47-ab33-170d01a37ce8"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:30.749398409Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1692,"client":"0","":"downstairs","session_id":"8b307ab1-da4b-4b47-ab33-170d01a37ce8"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:30.749412598Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1692,"client":"0","":"downstairs","session_id":"8b307ab1-da4b-4b47-ab33-170d01a37ce8"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:30.749535272Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1692,"client":"1","":"downstairs","session_id":"8b307ab1-da4b-4b47-ab33-170d01a37ce8"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:30.7495791Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1692,"client":"1","":"downstairs","session_id":"8b307ab1-da4b-4b47-ab33-170d01a37ce8"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:30.74999082Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1692,"client":"0","":"downstairs","session_id":"8b307ab1-da4b-4b47-ab33-170d01a37ce8"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:30.750266616Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1692,"client":"1","":"downstairs","session_id":"8b307ab1-da4b-4b47-ab33-170d01a37ce8"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:30.750313254Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1692,"client":"2","":"downstairs","session_id":"8b307ab1-da4b-4b47-ab33-170d01a37ce8"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:30.750707305Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1692,"client":"0","":"downstairs","session_id":"8b307ab1-da4b-4b47-ab33-170d01a37ce8"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:30.750752823Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1692,"client":"0","":"downstairs","session_id":"8b307ab1-da4b-4b47-ab33-170d01a37ce8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:30.750779431Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1692,"session_id":"8b307ab1-da4b-4b47-ab33-170d01a37ce8"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:30.750839438Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1692,"client":"1","":"downstairs","session_id":"8b307ab1-da4b-4b47-ab33-170d01a37ce8"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:30.750868187Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1692,"client":"1","":"downstairs","session_id":"8b307ab1-da4b-4b47-ab33-170d01a37ce8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:30.750882786Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1692,"session_id":"8b307ab1-da4b-4b47-ab33-170d01a37ce8"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:30.750895676Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1692,"client":"2","":"downstairs","session_id":"8b307ab1-da4b-4b47-ab33-170d01a37ce8"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:30.750909515Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1692,"client":"2","":"downstairs","session_id":"8b307ab1-da4b-4b47-ab33-170d01a37ce8"} {"msg":"[0]R flush_numbers[0..12]: [199, 228, 232, 224, 227, 226, 226, 232, 230, 231, 229, 232]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:30.750937164Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1692,"":"downstairs","":"downstairs","session_id":"8b307ab1-da4b-4b47-ab33-170d01a37ce8"} {"msg":"[0]R generation[0..12]: [112, 124, 127, 122, 124, 124, 124, 127, 126, 126, 125, 127]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:30.750951893Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1692,"":"downstairs","":"downstairs","session_id":"8b307ab1-da4b-4b47-ab33-170d01a37ce8"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:30.750966052Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1692,"":"downstairs","":"downstairs","session_id":"8b307ab1-da4b-4b47-ab33-170d01a37ce8"} {"msg":"[1]R flush_numbers[0..12]: [199, 228, 232, 224, 227, 226, 226, 232, 230, 231, 229, 232]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:30.750979722Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1692,"":"downstairs","":"downstairs","session_id":"8b307ab1-da4b-4b47-ab33-170d01a37ce8"} {"msg":"[1]R generation[0..12]: [112, 124, 127, 122, 124, 124, 124, 127, 126, 126, 125, 127]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:30.750993431Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1692,"":"downstairs","":"downstairs","session_id":"8b307ab1-da4b-4b47-ab33-170d01a37ce8"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:30.75101234Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1692,"":"downstairs","":"downstairs","session_id":"8b307ab1-da4b-4b47-ab33-170d01a37ce8"} {"msg":"[2]R flush_numbers[0..12]: [199, 228, 232, 224, 227, 226, 226, 232, 230, 231, 229, 232]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:30.751026969Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1692,"":"downstairs","":"downstairs","session_id":"8b307ab1-da4b-4b47-ab33-170d01a37ce8"} {"msg":"[2]R generation[0..12]: [112, 124, 127, 122, 124, 124, 124, 127, 126, 126, 125, 127]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:30.751040639Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1692,"":"downstairs","":"downstairs","session_id":"8b307ab1-da4b-4b47-ab33-170d01a37ce8"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:30.751054568Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1692,"":"downstairs","":"downstairs","session_id":"8b307ab1-da4b-4b47-ab33-170d01a37ce8"} {"msg":"Max found gen is 128","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:30.751067687Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1692,"":"downstairs","":"downstairs","session_id":"8b307ab1-da4b-4b47-ab33-170d01a37ce8"} {"msg":"Generation requested: 128 >= found:128","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:30.751080866Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1692,"":"downstairs","":"downstairs","session_id":"8b307ab1-da4b-4b47-ab33-170d01a37ce8"} {"msg":"Next flush: 233","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:30.751093996Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1692,"":"downstairs","":"downstairs","session_id":"8b307ab1-da4b-4b47-ab33-170d01a37ce8"} {"msg":"All extents match","Wait for a query_work_queue command to finish before sending IO v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:30.751114165Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1692,"":"downstairs","":"downstairs","session_id":"8b307ab1-da4b-4b47-ab33-170d01a37ce8"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:30.751127504Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1692,"session_id":"8b307ab1-da4b-4b47-ab33-170d01a37ce8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:30.751139674Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1692,"client":"0","":"downstairs","session_id":"8b307ab1-da4b-4b47-ab33-170d01a37ce8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:30.751152733Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1692,"client":"1","":"downstairs","session_id":"8b307ab1-da4b-4b47-ab33-170d01a37ce8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:30.751165913Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1692,"client":"2","":"downstairs","session_id":"8b307ab1-da4b-4b47-ab33-170d01a37ce8"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:30.751179092Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1692,"session_id":"8b307ab1-da4b-4b47-ab33-170d01a37ce8"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:30.751191441Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1692,"session_id":"8b307ab1-da4b-4b47-ab33-170d01a37ce8"} {"msg":"83ab84d6-89ef-4f70-8ada-ea2ba5c734df is now active with session: 8b307ab1-da4b-4b47-ab33-170d01a37ce8","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:30.751204131Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1692,"session_id":"8b307ab1-da4b-4b47-ab33-170d01a37ce8"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:30.75121817Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1692,"session_id":"8b307ab1-da4b-4b47-ab33-170d01a37ce8"} {"msg":"The guest has finished waiting for activation with:128","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:30.751230499Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1692} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 143 len 32768 data: 8 9 9 9 11 10 12 12 03/30 Write block 304 len 40960 data: 11 8 8 4 7 9 10 12 13 14 04/30 Read block 343 len 8192 05/30 Read block 300 len 20480 06/30 Read block 576 len 8192 07/30 Flush 08/30 Write block 325 len 8192 data: 7 10 09/30 Flush 10/30 Write block 50 len 4096 data: 7 11/30 Write block 531 len 36864 data: 11 9 10 10 11 11 10 9 9 12/30 Flush 13/30 Write block 351 len 16384 data: 13 13 17 15 14/30 Read block 436 len 32768 15/30 Write block 272 len 32768 data: 11 12 14 12 13 12 10 10 16/30 Read block 566 len 32768 17/30 Write block 477 len 4096 data: 16 18/30 Read block 129 len 20480 19/30 Write block 562 len 40960 data: 8 7 8 10 8 6 10 10 8 12 20/30 Read block 547 len 16384 21/30 Read block 532 len 8192 22/30 Write block 580 len 4096 data: 10 23/30 Read block 577 len 8192 24/30 Read block 325 len 16384 25/30 Write block 117 len 8192 data: 11 10 26/30 Flush 27/30 Write block 218 len 28672 data: 11 10 9 12 12 11 11 28/30 Read block 59 len 32768 29/30 Read block 105 len 12288 30/30 Read block 234 len 4096 ---------------------------------------------------------------- Crucible gen:128 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 7 Done Done Done false 1027 Acked Read 8 Done Done Done false 1028 Acked Read 3 Done Done Done false 1029 Acked Read 1 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 0 1 Done 4 3 4 11 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:27:30.777389441Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1692} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 112 112 112 199 199 199 F F F 1 020-039 124 124 124 228 228 228 F F F 2 040-059 128 128 128 236 236 236 F F F 3 060-079 122 122 122 224 224 224 F F F 4 080-099 124 124 124 227 227 227 F F F 5 100-119 128 128 128 237 237 237 F F F 6 120-139 124 124 124 226 226 226 F F F 7 140-159 128 128 128 234 234 234 F F F 8 160-179 126 126 126 230 230 230 F F F 9 180-199 126 126 126 231 231 231 F F F 10 200-219 125 125 125 229 229 229 T T T 11 220-239 127 127 127 232 232 232 T T T 12 240-259 127 127 127 232 232 232 F F F 13 260-279 128 128 128 237 237 237 F F F 14 280-299 124 124 124 226 226 226 F F F 15 300-319 128 128 128 234 234 234 F F F 16 320-339 128 128 128 235 235 235 F F F 17 340-359 128 128 128 237 237 237 F F F 18 360-379 127 127 127 232 232 232 F F F 19 380-399 126 126 126 231 231 231 F F F 20 400-419 126 126 126 231 231 231 F F F 21 420-439 126 126 126 230 230 230 F F F 22 440-459 122 122 122 223 223 223 F F F 23 460-479 128 128 128 237 237 237 F F F 24 480-499 127 127 127 232 232 232 F F F 25 500-519 127 127 127 232 232 232 F F F 26 520-539 128 128 128 236 236 236 F F F 27 540-559 127 127 127 232 232 232 F F F 28 560-579 128 128 128 237 237 237 F F F 29 580-599 128 128 128 237 237 237 F F F Max gen: 128, Max flush: 237 A Difference in extent metadata is expected here On loop 64 Verifying data now No change after verify Loop: 64 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 112 112 112 199 199 199 F F F 1 020-039 124 124 124 228 228 228 F F F 2 040-059 128 128 128 236 236 236 F F F 3 060-079 122 122 122 224 224 224 F F F 4 080-099 124 124 124 227 227 227 F F F 5 100-119 128 128 128 237 237 237 F F F 6 120-139 124 124 124 226 226 226 F F F 7 140-159 128 128 128 234 234 234 F F F 8 160-179 126 126 126 230 230 230 F F F 9 180-199 126 126 126 231 231 231 F F F 10 200-219 129 129 129 238 238 238 F F F 11 220-239 129 129 129 238 238 238 F F F 12 240-259 127 127 127 232 232 232 F F F 13 260-279 128 128 128 237 237 237 F F F 14 280-299 124 124 124 226 226 226 F F F 15 300-319 128 128 128 234 234 234 F F F 16 320-339 128 128 128 235 235 235 F F F 17 340-359 128 128 128 237 237 237 F F F 18 360-379 127 127 127 232 232 232 F F F 19 380-399 126 126 126 231 231 231 F F F 20 400-419 126 126 126 231 231 231 F F F 21 420-439 126 126 126 230 230 230 F F F 22 440-459 122 122 122 223 223 223 F F F 23 460-479 128 128 128 237 237 237 F F F 24 480-499 127 127 127 232 232 232 F F F 25 500-519 127 127 127 232 232 232 F F F 26 520-539 128 128 128 236 236 236 F F F 27 540-559 127 127 127 232 232 232 F F F 28 560-579 128 128 128 237 237 237 F F F 29 580-599 128 128 128 237 237 237 F F F Max gen: 129, Max flush: 238 Begin loop 65 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:35.053208554Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1700,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:27:35.096240811Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1700,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:35.096623112Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1700,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:35.128861546Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1700,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:35.12939317Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1700} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"0fd98d0d5d909611f8b408ccda2559d35fe117bb\",\n git_commit_timestamp: \"2024-11-08T20:05:44.000000000Z\",\n git_branch: \"mkeeter/drop-skipped-backpressure-guard\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:35.129440928Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1700} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:35.129461987Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1700} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:35.129477716Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1700} {"msg":"Crucible 95a07c53-c869-497d-827c-9787c8d0aa53 has session id: 60658aa1-7513-474e-bfb8-0ba7fcca10b8","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:35.129968952Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1700,"session_id":"60658aa1-7513-474e-bfb8-0ba7fcca10b8"} {"msg":"Upstairs opts: Upstairs UUID: 95a07c53-c869-497d-827c-9787c8d0aa53, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:35.130027679Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1700,"session_id":"60658aa1-7513-474e-bfb8-0ba7fcca10b8"} {"msg":"Crucible stats registered with UUID: 95a07c53-c869-497d-827c-9787c8d0aa53","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:35.130057518Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1700,"session_id":"60658aa1-7513-474e-bfb8-0ba7fcca10b8"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:35.161171247Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1700,"task":"crutest"} {"msg":"The guest has requested activation with gen:130","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:35.161201376Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1700} {"msg":"95a07c53-c869-497d-827c-9787c8d0aa53 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:35.161271212Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1700,"session_id":"60658aa1-7513-474e-bfb8-0ba7fcca10b8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:35.161293801Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1700,"client":"0","":"downstairs","session_id":"60658aa1-7513-474e-bfb8-0ba7fcca10b8"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:35.16132178Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1700,"client":"0","":"downstairs","session_id":"60658aa1-7513-474e-bfb8-0ba7fcca10b8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:35.161339819Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1700,"client":"1","":"downstairs","session_id":"60658aa1-7513-474e-bfb8-0ba7fcca10b8"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:35.161357148Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1700,"client":"1","":"downstairs","session_id":"60658aa1-7513-474e-bfb8-0ba7fcca10b8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:35.161373827Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1700,"client":"2","":"downstairs","session_id":"60658aa1-7513-474e-bfb8-0ba7fcca10b8"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:35.161402926Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1700,"client":"2","":"downstairs","session_id":"60658aa1-7513-474e-bfb8-0ba7fcca10b8"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:35.161452593Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1700,"":"io task","client":"2","":"downstairs","session_id":"60658aa1-7513-474e-bfb8-0ba7fcca10b8"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:35.161491602Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1700,"":"io task","client":"0","":"downstairs","session_id":"60658aa1-7513-474e-bfb8-0ba7fcca10b8"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:35.161566078Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1700,"":"io task","client":"1","":"downstairs","session_id":"60658aa1-7513-474e-bfb8-0ba7fcca10b8"} {"msg":"ds_connection connected from Ok(127.0.0.1:48363)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:35.161600436Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1700,"":"io task","client":"2","":"downstairs","session_id":"60658aa1-7513-474e-bfb8-0ba7fcca10b8"} {"msg":"ds_connection connected from Ok(127.0.0.1:46633)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:35.16173143Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1700,"":"io task","client":"1","":"downstairs","session_id":"60658aa1-7513-474e-bfb8-0ba7fcca10b8"} {"msg":"ds_connection connected from Ok(127.0.0.1:61464)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:35.161762668Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1700,"":"io task","client":"0","":"downstairs","session_id":"60658aa1-7513-474e-bfb8-0ba7fcca10b8"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:35.162264994Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1700,"client":"1","":"downstairs","session_id":"60658aa1-7513-474e-bfb8-0ba7fcca10b8"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:35.162301662Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1700,"client":"1","":"downstairs","session_id":"60658aa1-7513-474e-bfb8-0ba7fcca10b8"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:35.162319311Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1700,"client":"2","":"downstairs","session_id":"60658aa1-7513-474e-bfb8-0ba7fcca10b8"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:35.16233371Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1700,"client":"2","":"downstairs","session_id":"60658aa1-7513-474e-bfb8-0ba7fcca10b8"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:35.162403197Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1700,"client":"0","":"downstairs","session_id":"60658aa1-7513-474e-bfb8-0ba7fcca10b8"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:35.162440675Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1700,"client":"0","":"downstairs","session_id":"60658aa1-7513-474e-bfb8-0ba7fcca10b8"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:35.162973029Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1700,"client":"1","":"downstairs","session_id":"60658aa1-7513-474e-bfb8-0ba7fcca10b8"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:35.163028667Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1700,"client":"2","":"downstairs","session_id":"60658aa1-7513-474e-bfb8-0ba7fcca10b8"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:35.163059675Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1700,"client":"0","":"downstairs","session_id":"60658aa1-7513-474e-bfb8-0ba7fcca10b8"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:35.163602888Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1700,"client":"1","":"downstairs","session_id":"60658aa1-7513-474e-bfb8-0ba7fcca10b8"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:35.163633407Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1700,"client":"1","":"downstairs","session_id":"60658aa1-7513-474e-bfb8-0ba7fcca10b8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:35.163648806Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1700,"session_id":"60658aa1-7513-474e-bfb8-0ba7fcca10b8"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:35.163661505Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1700,"client":"0","":"downstairs","session_id":"60658aa1-7513-474e-bfb8-0ba7fcca10b8"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:35.163674895Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1700,"client":"0","":"downstairs","session_id":"60658aa1-7513-474e-bfb8-0ba7fcca10b8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:35.163688184Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1700,"session_id":"60658aa1-7513-474e-bfb8-0ba7fcca10b8"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:35.163700303Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1700,"client":"2","":"downstairs","session_id":"60658aa1-7513-474e-bfb8-0ba7fcca10b8"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:35.163713853Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1700,"client":"2","":"downstairs","session_id":"60658aa1-7513-474e-bfb8-0ba7fcca10b8"} {"msg":"[0]R flush_numbers[0..12]: [199, 228, 236, 224, 227, 237, 226, 234, 230, 231, 238, 238]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:35.163727722Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1700,"":"downstairs","":"downstairs","session_id":"60658aa1-7513-474e-bfb8-0ba7fcca10b8"} {"msg":"[0]R generation[0..12]: [112, 124, 128, 122, 124, 128, 124, 128, 126, 126, 129, 129]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:35.163741512Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1700,"":"downstairs","":"downstairs","session_id":"60658aa1-7513-474e-bfb8-0ba7fcca10b8"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:35.163755431Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1700,"":"downstairs","":"downstairs","session_id":"60658aa1-7513-474e-bfb8-0ba7fcca10b8"} {"msg":"[1]R flush_numbers[0..12]: [199, 228, 236, 224, 227, 237, 226, 234, 230, 231, 238, 238]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:35.16376918Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1700,"":"downstairs","":"downstairs","session_id":"60658aa1-7513-474e-bfb8-0ba7fcca10b8"} {"msg":"[1]R generation[0..12]: [112, 124, 128, 122, 124, 128, 124, 128, 126, 126, 129, 129]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:35.163782919Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1700,"":"downstairs","":"downstairs","session_id":"60658aa1-7513-474e-bfb8-0ba7fcca10b8"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:35.163796799Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1700,"":"downstairs","":"downstairs","session_id":"60658aa1-7513-474e-bfb8-0ba7fcca10b8"} {"msg":"[2]R flush_numbers[0..12]: [199, 228, 236, 224, 227, 237, 226, 234, 230, 231, 238, 238]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:35.163810428Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1700,"":"downstairs","":"downstairs","session_id":"60658aa1-7513-474e-bfb8-0ba7fcca10b8"} {"msg":"[2]R generation[0..12]: [112, 124, 128, 122, 124, 128, 124, 128, 126, 126, 129, 129]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:35.163825908Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1700,"":"downstairs","":"downstairs","session_id":"60658aa1-7513-474e-bfb8-0ba7fcca10b8"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:35.163839947Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1700,"":"downstairs","":"downstairs","session_id":"60658aa1-7513-474e-bfb8-0ba7fcca10b8"} {"msg":"Max found gen is 130","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:35.163852986Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1700,"":"downstairs","":"downstairs","session_id":"60658aa1-7513-474e-bfb8-0ba7fcca10b8"} {"msg":"Generation requested: 130 >= found:130","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:35.163866295Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1700,"":"downstairs","":"downstairs"Wait for a query_work_queue command to finish before sending IO ,"session_id":"60658aa1-7513-474e-bfb8-0ba7fcca10b8"} {"msg":"Next flush: 239","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:35.163887845Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1700,"":"downstairs","":"downstairs","session_id":"60658aa1-7513-474e-bfb8-0ba7fcca10b8"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:35.163900924Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1700,"":"downstairs","":"downstairs","session_id":"60658aa1-7513-474e-bfb8-0ba7fcca10b8"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:35.163914053Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1700,"session_id":"60658aa1-7513-474e-bfb8-0ba7fcca10b8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:35.163926063Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1700,"client":"0","":"downstairs","session_id":"60658aa1-7513-474e-bfb8-0ba7fcca10b8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:35.163939212Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1700,"client":"1","":"downstairs","session_id":"60658aa1-7513-474e-bfb8-0ba7fcca10b8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:35.163952261Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1700,"client":"2","":"downstairs","session_id":"60658aa1-7513-474e-bfb8-0ba7fcca10b8"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:35.163965551Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1700,"session_id":"60658aa1-7513-474e-bfb8-0ba7fcca10b8"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:35.16397798Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1700,"session_id":"60658aa1-7513-474e-bfb8-0ba7fcca10b8"} {"msg":"95a07c53-c869-497d-827c-9787c8d0aa53 is now active with session: 60658aa1-7513-474e-bfb8-0ba7fcca10b8","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:35.163990879Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1700,"session_id":"60658aa1-7513-474e-bfb8-0ba7fcca10b8"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:35.164020298Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1700Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"session_id":"60658aa1-7513-474e-bfb8-0ba7fcca10b8"} {"msg":"The guest has finished waiting for activation with:130","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:35.164037247Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1700} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 244 len 16384 data: 11 10 10 14 02/30 Flush 03/30 Write block 518 len 32768 data: 10 12 9 9 8 9 4 6 04/30 Read block 420 len 16384 05/30 Read block 369 len 32768 06/30 Read block 571 len 12288 07/30 Read block 137 len 8192 08/30 Read block 139 len 40960 09/30 Write block 72 len 12288 data: 7 8 9 10/30 Read block 363 len 32768 11/30 Read block 288 len 36864 12/30 Read block 285 len 16384 13/30 Read block 444 len 40960 14/30 Read block 421 len 36864 15/30 Write block 481 len 40960 data: 15 14 13 12 10 10 9 10 12 11 16/30 Write block 68 len 24576 data: 7 6 8 7 8 9 17/30 Read block 398 len 16384 18/30 Read block 38 len 20480 19/30 Write block 161 len 36864 data: 5 4 4 6 7 7 7 12 12 20/30 Read block 534 len 32768 21/30 Read block 4 len 36864 22/30 Read block 515 len 28672 23/30 Read block 344 len 32768 24/30 Write block 162 len 28672 data: 5 5 7 8 8 8 13 25/30 Read block 485 len 12288 26/30 Write block 49 len 20480 data: 6 8 7 6 8 27/30 Write block 226 len 16384 data: 12 14 10 10 28/30 Read block 396 len 16384 29/30 Read block 69 len 16384 30/30 Read block 404 len 12288 ---------------------------------------------------------------- Crucible gen:130 GIO:true work queues: Upstairs:0 downstairs:28 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1002 Acked Write 8 Done Done Done false 1003 Acked Read 4 Done Done Done false 1004 Acked Read 8 Done Done Done false 1005 Acked Read 3 Done Done Done false 1006 Acked Read 2 Done Done Done false 1007 Acked Read 10 Done Done Done false 1008 Acked Write 3 Done Done Done false 1009 Acked Read 8 Done Done Done false 1010 Acked Read 9 Done Done Done false 1011 Acked Read 4 Done Done Done false 1012 Acked Read 10 Done Done Done false 1013 Acked Read 9 Done Done Done false 1014 Acked Write 10 Done Done Done false 1015 Acked Write 6 Done Done Done false 1016 Acked Read 4 Done Done Done false 1017 Acked Read 5 Done Done Done false 1018 Acked Write 9 Done Done Done false 1019 Acked Read 8 Done Done Done false 1020 Acked Read 9 Done Done Done false 1021 Acked Read 7 Done Done Done false 1022 Acked Read 8 Done Done Done false 1023 Acked Write 7 Done Done Done false 1024 Acked Read 3 Done Done Done false 1025 Acked Write 5 Done Done Done false 1026 Acked Write 4 Done Done Done false 1027 Acked Read 4 Done Done Done false 1028 Acked Read 4 Done Done Done false 1029 Acked Read 3 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 28 27 27 82 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1001 1001 1001 Downstairs last five completed: 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:27:35.186851582Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1700} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 112 112 112 199 199 199 F F F 1 020-039 124 124 124 228 228 228 F F F 2 040-059 128 128 128 236 236 236 T T T 3 060-079 122 122 122 224 224 224 T T T 4 080-099 124 124 124 227 227 227 F F F 5 100-119 128 128 128 237 237 237 F F F 6 120-139 124 124 124 226 226 226 F F F 7 140-159 128 128 128 234 234 234 F F F 8 160-179 126 126 126 230 230 230 T T T 9 180-199 126 126 126 231 231 231 F F F 10 200-219 129 129 129 238 238 238 F F F 11 220-239 129 129 129 238 238 238 T T T 12 240-259 130 130 130 239 239 239 F F F 13 260-279 128 128 128 237 237 237 F F F 14 280-299 124 124 124 226 226 226 F F F 15 300-319 128 128 128 234 234 234 F F F 16 320-339 128 128 128 235 235 235 F F F 17 340-359 128 128 128 237 237 237 F F F 18 360-379 127 127 127 232 232 232 F F F 19 380-399 126 126 126 231 231 231 F F F 20 400-419 126 126 126 231 231 231 F F F 21 420-439 126 126 126 230 230 230 F F F 22 440-459 122 122 122 223 223 223 F F F 23 460-479 128 128 128 237 237 237 F F F 24 480-499 127 127 127 232 232 232 T T T 25 500-519 127 127 127 232 232 232 T T T 26 520-539 128 128 128 236 236 236 T T T 27 540-559 127 127 127 232 232 232 F F F 28 560-579 128 128 128 237 237 237 F F F 29 580-599 128 128 128 237 237 237 F F F Max gen: 130, Max flush: 239 A Difference in extent metadata is expected here On loop 65 Verifying data now No change after verify Loop: 65 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 112 112 112 199 199 199 F F F 1 020-039 124 124 124 228 228 228 F F F 2 040-059 131 131 131 240 240 240 F F F 3 060-079 131 131 131 240 240 240 F F F 4 080-099 124 124 124 227 227 227 F F F 5 100-119 128 128 128 237 237 237 F F F 6 120-139 124 124 124 226 226 226 F F F 7 140-159 128 128 128 234 234 234 F F F 8 160-179 131 131 131 240 240 240 F F F 9 180-199 126 126 126 231 231 231 F F F 10 200-219 129 129 129 238 238 238 F F F 11 220-239 131 131 131 240 240 240 F F F 12 240-259 130 130 130 239 239 239 F F F 13 260-279 128 128 128 237 237 237 F F F 14 280-299 124 124 124 226 226 226 F F F 15 300-319 128 128 128 234 234 234 F F F 16 320-339 128 128 128 235 235 235 F F F 17 340-359 128 128 128 237 237 237 F F F 18 360-379 127 127 127 232 232 232 F F F 19 380-399 126 126 126 231 231 231 F F F 20 400-419 126 126 126 231 231 231 F F F 21 420-439 126 126 126 230 230 230 F F F 22 440-459 122 122 122 223 223 223 F F F 23 460-479 128 128 128 237 237 237 F F F 24 480-499 131 131 131 240 240 240 F F F 25 500-519 131 131 131 240 240 240 F F F 26 520-539 131 131 131 240 240 240 F F F 27 540-559 127 127 127 232 232 232 F F F 28 560-579 128 128 128 237 237 237 F F F 29 580-599 128 128 128 237 237 237 F F F Max gen: 131, Max flush: 240 Begin loop 66 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:39.568188818Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1708,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:27:39.611274892Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1708,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:39.611675963Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1708,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:39.643775904Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1708,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:39.64445967Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1708} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"0fd98d0d5d909611f8b408ccda2559d35fe117bb\",\n git_commit_timestamp: \"2024-11-08T20:05:44.000000000Z\",\n git_branch: \"mkeeter/drop-skipped-backpressure-guard\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:39.644494069Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1708} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:39.644510098Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1708} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:39.644528007Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1708} {"msg":"Crucible 9526f14a-b801-4502-9da0-c7c916f2b111 has session id: 5a8d7ad8-5131-4aae-88bd-42584654df3f","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:39.644955466Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1708,"session_id":"5a8d7ad8-5131-4aae-88bd-42584654df3f"} {"msg":"Upstairs opts: Upstairs UUID: 9526f14a-b801-4502-9da0-c7c916f2b111, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:39.645010014Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1708,"session_id":"5a8d7ad8-5131-4aae-88bd-42584654df3f"} {"msg":"Crucible stats registered with UUID: 9526f14a-b801-4502-9da0-c7c916f2b111","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:39.645041372Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1708,"session_id":"5a8d7ad8-5131-4aae-88bd-42584654df3f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:39.676487435Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1708,"task":"crutest"} {"msg":"The guest has requested activation with gen:132","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:39.676534073Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1708} {"msg":"9526f14a-b801-4502-9da0-c7c916f2b111 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:39.676571841Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1708,"session_id":"5a8d7ad8-5131-4aae-88bd-42584654df3f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:39.67660315Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1708,"client":"0","":"downstairs","session_id":"5a8d7ad8-5131-4aae-88bd-42584654df3f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:39.676650877Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1708,"client":"0","":"downstairs","session_id":"5a8d7ad8-5131-4aae-88bd-42584654df3f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:39.676688166Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1708,"client":"1","":"downstairs","session_id":"5a8d7ad8-5131-4aae-88bd-42584654df3f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:39.676721684Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1708,"client":"1","":"downstairs","session_id":"5a8d7ad8-5131-4aae-88bd-42584654df3f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:39.676761252Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1708,"client":"2","":"downstairs","session_id":"5a8d7ad8-5131-4aae-88bd-42584654df3f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:39.67679656Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1708,"client":"2","":"downstairs","session_id":"5a8d7ad8-5131-4aae-88bd-42584654df3f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:39.676823759Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1708,"":"io task","client":"0","":"downstairs","session_id":"5a8d7ad8-5131-4aae-88bd-42584654df3f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:39.676857457Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1708,"":"io task","client":"2","":"downstairs","session_id":"5a8d7ad8-5131-4aae-88bd-42584654df3f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:39.676891065Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1708,"":"io task","client":"1","":"downstairs","session_id":"5a8d7ad8-5131-4aae-88bd-42584654df3f"} {"msg":"ds_connection connected from Ok(127.0.0.1:57039)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:39.676926654Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1708,"":"io task","client":"2","":"downstairs","session_id":"5a8d7ad8-5131-4aae-88bd-42584654df3f"} {"msg":"ds_connection connected from Ok(127.0.0.1:34358)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:39.676962702Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1708,"":"io task","client":"1","":"downstairs","session_id":"5a8d7ad8-5131-4aae-88bd-42584654df3f"} {"msg":"ds_connection connected from Ok(127.0.0.1:62670)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:39.677033898Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1708,"":"io task","client":"0","":"downstairs","session_id":"5a8d7ad8-5131-4aae-88bd-42584654df3f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:39.677697786Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1708,"client":"1","":"downstairs","session_id":"5a8d7ad8-5131-4aae-88bd-42584654df3f"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:39.677728055Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1708,"client":"1","":"downstairs","session_id":"5a8d7ad8-5131-4aae-88bd-42584654df3f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:39.677742914Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1708,"client":"2","":"downstairs","session_id":"5a8d7ad8-5131-4aae-88bd-42584654df3f"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:39.677756573Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1708,"client":"2","":"downstairs","session_id":"5a8d7ad8-5131-4aae-88bd-42584654df3f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:39.67781681Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1708,"client":"0","":"downstairs","session_id":"5a8d7ad8-5131-4aae-88bd-42584654df3f"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:39.677835039Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1708,"client":"0","":"downstairs","session_id":"5a8d7ad8-5131-4aae-88bd-42584654df3f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:39.678372203Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1708,"client":"1","":"downstairs","session_id":"5a8d7ad8-5131-4aae-88bd-42584654df3f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:39.678404062Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1708,"client":"2","":"downstairs","session_id":"5a8d7ad8-5131-4aae-88bd-42584654df3f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:39.67842861Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1708,"client":"0","":"downstairs","session_id":"5a8d7ad8-5131-4aae-88bd-42584654df3f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:39.678961274Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1708,"client":"2","":"downstairs","session_id":"5a8d7ad8-5131-4aae-88bd-42584654df3f"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:39.679002882Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1708,"client":"2","":"downstairs","session_id":"5a8d7ad8-5131-4aae-88bd-42584654df3f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:39.679032761Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1708,"session_id":"5a8d7ad8-5131-4aae-88bd-42584654df3f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:39.67905501Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1708,"client":"1","":"downstairs","session_id":"5a8d7ad8-5131-4aae-88bd-42584654df3f"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:39.679078209Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1708,"client":"1","":"downstairs","session_id":"5a8d7ad8-5131-4aae-88bd-42584654df3f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:39.679101277Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1708,"session_id":"5a8d7ad8-5131-4aae-88bd-42584654df3f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:39.679122967Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1708,"client":"0","":"downstairs","session_id":"5a8d7ad8-5131-4aae-88bd-42584654df3f"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:39.679147475Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1708,"client":"0","":"downstairs","session_id":"5a8d7ad8-5131-4aae-88bd-42584654df3f"} {"msg":"[0]R flush_numbers[0..12]: [199, 228, 240, 240, 227, 237, 226, 234, 240, 231, 238, 240]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:39.679171484Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1708,"":"downstairs","":"downstairs","session_id":"5a8d7ad8-5131-4aae-88bd-42584654df3f"} {"msg":"[0]R generation[0..12]: [112, 124, 131, 131, 124, 128, 124, 128, 131, 126, 129, 131]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:39.679194983Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1708,"":"downstairs","":"downstairs","session_id":"5a8d7ad8-5131-4aae-88bd-42584654df3f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:39.679219682Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1708,"":"downstairs","":"downstairs","session_id":"5a8d7ad8-5131-4aae-88bd-42584654df3f"} {"msg":"[1]R flush_numbers[0..12]: [199, 228, 240, 240, 227, 237, 226, 234, 240, 231, 238, 240]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:39.679246161Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1708,"":"downstairs","":"downstairs","session_id":"5a8d7ad8-5131-4aae-88bd-42584654df3f"} {"msg":"[1]R generation[0..12]: [112, 124, 131, 131, 124, 128, 124, 128, 131, 126, 129, 131]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:39.679272259Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1708,"":"downstairs","":"downstairs","session_id":"5a8d7ad8-5131-4aae-88bd-42584654df3f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:39.679302748Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1708,"":"downstairs","":"downstairs","session_id":"5a8d7ad8-5131-4aae-88bd-42584654df3f"} {"msg":"[2]R flush_numbers[0..12]: [199, 228, 240, 240, 227, 237, 226, 234, 240, 231, 238, 240]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:39.679332396Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1708,"":"downstairs","":"downstairs","session_id":"5a8d7ad8-5131-4aae-88bd-42584654df3f"} {Wait for a query_work_queue command to finish before sending IO "msg":"[2]R generation[0..12]: [112, 124, 131, 131, 124, 128, 124, 128, 131, 126, 129, 131]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:39.679370324Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1708,"":"downstairs","":"downstairs","session_id":"5a8d7ad8-5131-4aae-88bd-42584654df3f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:39.679401503Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1708,"":"downstairs","":"downstairs","session_id":"5a8d7ad8-5131-4aae-88bd-42584654df3f"} {"msg":"Max found gen is 132","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:39.679429661Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1708,"":"downstairs","":"downstairs","session_id":"5a8d7ad8-5131-4aae-88bd-42584654df3f"} {"msg":"Generation requested: 132 >= found:132","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:39.67945767Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1708,"":"downstairs","":"downstairs","session_id":"5a8d7ad8-5131-4aae-88bd-42584654df3f"} {"msg":"Next flush: 241","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:39.679485389Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1708,"":"downstairs","":"downstairs","session_id":"5a8d7ad8-5131-4aae-88bd-42584654df3f"} {"msg":"All extents match","v":0,"name":"crucible","level":30Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2024-11-08T20:27:39.679512807Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1708,"":"downstairs","":"downstairs","session_id":"5a8d7ad8-5131-4aae-88bd-42584654df3f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:39.679543986Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1708,"session_id":"5a8d7ad8-5131-4aae-88bd-42584654df3f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:39.679568895Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1708,"client":"0","":"downstairs","session_id":"5a8d7ad8-5131-4aae-88bd-42584654df3f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:39.679593833Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1708,"client":"1","":"downstairs","session_id":"5a8d7ad8-5131-4aae-88bd-42584654df3f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:39.679608633Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1708,"client":"2","":"downstairs","session_id":"5a8d7ad8-5131-4aae-88bd-42584654df3f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:39.679622312Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1708,"session_id":"5a8d7ad8-5131-4aae-88bd-42584654df3f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:39.679635011Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1708,"session_id":"5a8d7ad8-5131-4aae-88bd-42584654df3f"} {"msg":"9526f14a-b801-4502-9da0-c7c916f2b111 is now active with session: 5a8d7ad8-5131-4aae-88bd-42584654df3f","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:39.679648081Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1708,"session_id":"5a8d7ad8-5131-4aae-88bd-42584654df3f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:39.67966319Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1708,"session_id":"5a8d7ad8-5131-4aae-88bd-42584654df3f"} {"msg":"The guest has finished waiting for activation with:132","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:39.6796761Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1708} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 292 len 8192 data: 12 12 03/30 Read block 32 len 12288 04/30 Read block 62 len 16384 05/30 Read block 479 len 20480 06/30 Read block 217 len 12288 07/30 Read block 22 len 8192 08/30 Read block 584 len 12288 09/30 Read block 391 len 24576 10/30 Read block 9 len 8192 11/30 Write block 332 len 36864 data: 8 10 11 11 11 12 14 13 12 12/30 Write block 538 len 28672 data: 10 10 10 11 12 12 14 13/30 Flush 14/30 Write block 372 len 12288 data: 12 9 9 15/30 Read block 280 len 40960 16/30 Read block 112 len 28672 17/30 Write block 538 len 32768 data: 11 11 11 12 13 13 15 11 18/30 Flush 19/30 Write block 477 len 16384 data: 17 17 16 14 20/30 Flush 21/30 Write block 538 len 4096 data: 12 22/30 Flush 23/30 Write block 289 len 16384 data: 14 14 12 13 24/30 Read block 276 len 16384 25/30 Read block 470 len 36864 26/30 Write block 540 len 8192 data: 12 13 27/30 Read block 195 len 12288 28/30 Read block 331 len 40960 29/30 Read block 42 len 40960 30/30 Write block 268 len 36864 data: 11 12 12 11 12 13 15 13 14 ---------------------------------------------------------------- Crucible gen:132 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 4 Done Done Done false 1023 Acked Read 4 Done Done Done false 1024 Acked Read 9 Done Done Done false 1025 Acked Write 2 Done Done Done false 1026 Acked Read 3 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Read 10 Done Done Done false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 7 7 7 21 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1021 1021 1021 Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:27:39.708844684Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1708} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 112 112 112 199 199 199 F F F 1 020-039 124 124 124 228 228 228 F F F 2 040-059 131 131 131 240 240 240 F F F 3 060-079 131 131 131 240 240 240 F F F 4 080-099 124 124 124 227 227 227 F F F 5 100-119 128 128 128 237 237 237 F F F 6 120-139 124 124 124 226 226 226 F F F 7 140-159 128 128 128 234 234 234 F F F 8 160-179 131 131 131 240 240 240 F F F 9 180-199 126 126 126 231 231 231 F F F 10 200-219 129 129 129 238 238 238 F F F 11 220-239 131 131 131 240 240 240 F F F 12 240-259 130 130 130 239 239 239 F F F 13 260-279 128 128 128 237 237 237 T T T 14 280-299 132 132 132 242 242 242 T T T 15 300-319 128 128 128 234 234 234 F F F 16 320-339 132 132 132 242 242 242 F F F 17 340-359 132 132 132 242 242 242 F F F 18 360-379 132 132 132 243 243 243 F F F 19 380-399 126 126 126 231 231 231 F F F 20 400-419 126 126 126 231 231 231 F F F 21 420-439 126 126 126 230 230 230 F F F 22 440-459 122 122 122 223 223 223 F F F 23 460-479 132 132 132 244 244 244 F F F 24 480-499 132 132 132 244 244 244 F F F 25 500-519 131 131 131 240 240 240 F F F 26 520-539 132 132 132 245 245 245 F F F 27 540-559 132 132 132 243 243 243 T T T 28 560-579 128 128 128 237 237 237 F F F 29 580-599 128 128 128 237 237 237 F F F Max gen: 132, Max flush: 245 A Difference in extent metadata is expected here On loop 66 Verifying data now No change after verify Loop: 66 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 112 112 112 199 199 199 F F F 1 020-039 124 124 124 228 228 228 F F F 2 040-059 131 131 131 240 240 240 F F F 3 060-079 131 131 131 240 240 240 F F F 4 080-099 124 124 124 227 227 227 F F F 5 100-119 128 128 128 237 237 237 F F F 6 120-139 124 124 124 226 226 226 F F F 7 140-159 128 128 128 234 234 234 F F F 8 160-179 131 131 131 240 240 240 F F F 9 180-199 126 126 126 231 231 231 F F F 10 200-219 129 129 129 238 238 238 F F F 11 220-239 131 131 131 240 240 240 F F F 12 240-259 130 130 130 239 239 239 F F F 13 260-279 133 133 133 246 246 246 F F F 14 280-299 133 133 133 246 246 246 F F F 15 300-319 128 128 128 234 234 234 F F F 16 320-339 132 132 132 242 242 242 F F F 17 340-359 132 132 132 242 242 242 F F F 18 360-379 132 132 132 243 243 243 F F F 19 380-399 126 126 126 231 231 231 F F F 20 400-419 126 126 126 231 231 231 F F F 21 420-439 126 126 126 230 230 230 F F F 22 440-459 122 122 122 223 223 223 F F F 23 460-479 132 132 132 244 244 244 F F F 24 480-499 132 132 132 244 244 244 F F F 25 500-519 131 131 131 240 240 240 F F F 26 520-539 132 132 132 245 245 245 F F F 27 540-559 133 133 133 246 246 246 F F F 28 560-579 128 128 128 237 237 237 F F F 29 580-599 128 128 128 237 237 237 F F F Max gen: 133, Max flush: 246 Begin loop 67 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:44.029875256Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1716,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:44.076388833Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1716,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:44.077029042Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1716} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"0fd98d0d5d909611f8b408ccda2559d35fe117bb\",\n git_commit_timestamp: \"2024-11-08T20:05:44.000000000Z\",\n git_branch: \"mkeeter/drop-skipped-backpressure-guard\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:44.07706503Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1716} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:44.077079489Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1716} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:44.077099008Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1716} {"msg":"Crucible 9ffff6a0-2dbf-4fae-b9b4-74e20697c504 has session id: 2d0d2e28-3a01-4b5c-9108-07127e5116db","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:44.077568906Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1716,"session_id":"2d0d2e28-3a01-4b5c-9108-07127e5116db"} {"msg":"Upstairs opts: Upstairs UUID: 9ffff6a0-2dbf-4fae-b9b4-74e20697c504, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:44.077627113Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1716,"session_id":"2d0d2e28-3a01-4b5c-9108-07127e5116db"} {"msg":"Crucible stats registered with UUID: 9ffff6a0-2dbf-4fae-b9b4-74e20697c504","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:44.077657971Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1716,"session_id":"2d0d2e28-3a01-4b5c-9108-07127e5116db"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:44.108561171Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1716,"task":"crutest"} {"msg":"The guest has requested activation with gen:134","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:44.108598329Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1716} {"msg":"9ffff6a0-2dbf-4fae-b9b4-74e20697c504 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:44.108636847Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1716,"session_id":"2d0d2e28-3a01-4b5c-9108-07127e5116db"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:44.108656836Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1716,"client":"0","":"downstairs","session_id":"2d0d2e28-3a01-4b5c-9108-07127e5116db"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:44.108700224Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1716,"client":"0","":"downstairs","session_id":"2d0d2e28-3a01-4b5c-9108-07127e5116db"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:44.108736382Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1716,"client":"1","":"downstairs","session_id":"2d0d2e28-3a01-4b5c-9108-07127e5116db"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:44.108771201Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1716,"client":"1","":"downstairs","session_id":"2d0d2e28-3a01-4b5c-9108-07127e5116db"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:44.108803979Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1716,"client":"2","":"downstairs","session_id":"2d0d2e28-3a01-4b5c-9108-07127e5116db"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:44.108838077Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1716,"client":"2","":"downstairs","session_id":"2d0d2e28-3a01-4b5c-9108-07127e5116db"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:44.108870786Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1716,"":"io task","client":"2","":"downstairs","session_id":"2d0d2e28-3a01-4b5c-9108-07127e5116db"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:44.108905804Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1716,"":"io task","client":"0","":"downstairs","session_id":"2d0d2e28-3a01-4b5c-9108-07127e5116db"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:44.109131503Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1716,"":"io task","client":"1","":"downstairs","session_id":"2d0d2e28-3a01-4b5c-9108-07127e5116db"} {"msg":"ds_connection connected from Ok(127.0.0.1:43265)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:44.109178961Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1716,"":"io task","client":"2","":"downstairs","session_id":"2d0d2e28-3a01-4b5c-9108-07127e5116db"} {"msg":"ds_connection connected from Ok(127.0.0.1:53416)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:44.109211149Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1716,"":"io task","client":"0","":"downstairs","session_id":"2d0d2e28-3a01-4b5c-9108-07127e5116db"} {"msg":"ds_connection connected from Ok(127.0.0.1:51117)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:44.109244977Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1716,"":"io task","client":"1","":"downstairs","session_id":"2d0d2e28-3a01-4b5c-9108-07127e5116db"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:44.109643918Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1716,"client":"1","":"downstairs","session_id":"2d0d2e28-3a01-4b5c-9108-07127e5116db"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:44.109672207Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1716,"client":"1","":"downstairs","session_id":"2d0d2e28-3a01-4b5c-9108-07127e5116db"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:44.109821619Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1716,"client":"0","":"downstairs","session_id":"2d0d2e28-3a01-4b5c-9108-07127e5116db"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:44.109865807Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1716,"client":"0","":"downstairs","session_id":"2d0d2e28-3a01-4b5c-9108-07127e5116db"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:44.109899365Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1716,"client":"2","":"downstairs","session_id":"2d0d2e28-3a01-4b5c-9108-07127e5116db"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:44.109928114Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1716,"client":"2","":"downstairs","session_id":"2d0d2e28-3a01-4b5c-9108-07127e5116db"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:44.110285497Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1716,"client":"1","":"downstairs","session_id":"2d0d2e28-3a01-4b5c-9108-07127e5116db"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:44.110330775Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1716,"client":"0","":"downstairs","session_id":"2d0d2e28-3a01-4b5c-9108-07127e5116db"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:44.110581602Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1716,"client":"2","":"downstairs","session_id":"2d0d2e28-3a01-4b5c-9108-07127e5116db"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:44.110916096Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1716,"client":"1","":"downstairs","session_id":"2d0d2e28-3a01-4b5c-9108-07127e5116db"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:44.110942425Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1716,"client":"1","":"downstairs","session_id":"2d0d2e28-3a01-4b5c-9108-07127e5116db"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:44.110960224Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1716,"session_id":"2d0d2e28-3a01-4b5c-9108-07127e5116db"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:44.110973133Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1716,"client":"0","":"downstairs","session_id":"2d0d2e28-3a01-4b5c-9108-07127e5116db"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:44.110986872Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1716,"client":"0","":"downstairs","session_id":"2d0d2e28-3a01-4b5c-9108-07127e5116db"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:44.111000562Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1716,"session_id":"2d0d2e28-3a01-4b5c-9108-07127e5116db"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:44.111165774Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1716,"client":"2","":"downstairs","session_id":"2d0d2e28-3a01-4b5c-9108-07127e5116db"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:44.111201902Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1716,"client":"2","":"downstairs","session_id":"2d0d2e28-3a01-4b5c-9108-07127e5116db"} {"msg":"[0]R flush_numbers[0..12]: [199, 228, 240, 240, 227, 237, 226, 234, 240, 231, 238, 240]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:44.11122782Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1716,"":"downstairs","":"downstairs","session_id":"2d0d2e28-3a01-4b5c-9108-07127e5116db"} {"msg":"[0]R generation[0..12]: [112, 124, 131, 131, 124, 128, 124, 128, 131, 126, 129, 131]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:44.111252619Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1716,"":"downstairs","":"downstairs","session_id":"2d0d2e28-3a01-4b5c-9108-07127e5116db"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:44.111279038Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1716,"":"downstairs","":"downstairs","session_id":"2d0d2e28-3a01-4b5c-9108-07127e5116db"} {"msg":"[1]R flush_numbers[0..12]: [199, 228, 240, 240, 227, 237, 226, 234, 240, 231, 238, 240]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:44.111303817Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1716,"":"downstairs","":"downstairs","session_id":"2d0d2e28-3a01-4b5c-9108-07127e5116db"} {"msg":"[1]R generation[0..12]: [112, 124, 131, 131, 124, 128, 124, 128, 131, 126, 129, 131]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:44.111327586Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1716,"":"downstairs","":"downstairs","session_id":"2d0d2e28-3a01-4b5c-9108-07127e5116db"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:44.111353214Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1716,"":"downstairs","":"downstairs","session_id":"2d0d2e28-3a01-4b5c-9108-07127e5116db"} {"msg":"[2]R flush_numbers[0..12]: [199, 228, 240, 240, 227, 237, 226, 234, 240, 231, 238, 240]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:44.111380783Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1716,"":"downstairs","":"downstairs","session_id":"2d0d2e28-3a01-4b5c-9108-07127e5116db"} {"msg":"[2]R generation[0..12]: [112, 124, 131, 131, 124, 128, 124, 128, 131, 126, 129, 131]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:44.111406722Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1716,"":"downstairs","":"downstairs","session_id":"2d0d2e28-3a01-4b5c-9108-07127e5116db"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:44.1114374Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1716,"":"downstairs","":"downstairs","session_id":"2d0d2e28-3a01-4b5c-9108-07127e5116db"} {"msg":"Max found gen is 134","v":0,"name":"Wait for a query_work_queue command to finish before sending IO crucible","level":30,"time":"2024-11-08T20:27:44.111476619Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1716,"":"downstairs","":"downstairs","session_id":"2d0d2e28-3a01-4b5c-9108-07127e5116db"} {"msg":"Generation requested: 134 >= found:134","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:44.111508427Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1716,"":"downstairs","":"downstairs","session_id":"2d0d2e28-3a01-4b5c-9108-07127e5116db"} {"msg":"Next flush: 247","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:44.111536465Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1716,"":"downstairs","":"downstairs","session_id":"2d0d2e28-3a01-4b5c-9108-07127e5116db"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:44.111563284Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1716,"":"downstairs","":"downstairs","session_id":"2d0d2e28-3a01-4b5c-9108-07127e5116db"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:44.111591473Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1716,"session_id":"2d0d2e28-3a01-4b5c-9108-07127e5116db"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:44.111616752Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1716,"client":"0","":"downstairs","session_id":"2d0d2e28-3a01-4b5c-9108-07127e5116db"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:44.11164485Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1716Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"client":"1","":"downstairs","session_id":"2d0d2e28-3a01-4b5c-9108-07127e5116db"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:44.111677139Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1716,"client":"2","":"downstairs","session_id":"2d0d2e28-3a01-4b5c-9108-07127e5116db"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:44.111704837Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1716,"session_id":"2d0d2e28-3a01-4b5c-9108-07127e5116db"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:44.111721056Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1716,"session_id":"2d0d2e28-3a01-4b5c-9108-07127e5116db"} {"msg":"9ffff6a0-2dbf-4fae-b9b4-74e20697c504 is now active with session: 2d0d2e28-3a01-4b5c-9108-07127e5116db","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:44.111734426Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1716,"session_id":"2d0d2e28-3a01-4b5c-9108-07127e5116db"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:44.111746645Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1716,"session_id":"2d0d2e28-3a01-4b5c-9108-07127e5116db"} {"msg":"The guest has finished waiting for activation with:134","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:44.111759185Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1716} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 215 len 32768 data: 10 11 10 12 11 10 13 13 02/30 Write block 463 len 32768 data: 6 5 7 10 12 14 13 13 03/30 Flush 04/30 Write block 388 len 28672 data: 7 7 9 8 9 10 10 05/30 Read block 86 len 32768 06/30 Flush 07/30 Write block 251 len 36864 data: 13 16 15 14 13 8 9 9 8 08/30 Write block 173 len 40960 data: 7 8 9 8 9 8 9 9 8 9 09/30 Write block 527 len 8192 data: 7 9 10/30 Read block 386 len 16384 11/30 Write block 145 len 8192 data: 10 10 12/30 Write block 421 len 16384 data: 10 10 10 10 13/30 Read block 291 len 12288 14/30 Read block 430 len 40960 15/30 Flush 16/30 Write block 319 len 32768 data: 13 12 8 9 9 8 8 11 17/30 Flush 18/30 Write block 283 len 16384 data: 10 10 11 14 19/30 Read block 273 len 12288 20/30 Flush 21/30 Write block 283 len 24576 data: 11 11 12 15 15 16 22/30 Read block 191 len 20480 23/30 Read block 572 len 20480 24/30 Flush 25/30 Write block 505 len 40960 data: 9 9 8 11 10 11 12 10 10 10 26/30 Read block 501 len 28672 27/30 Write block 310 len 12288 data: 11 13 14 28/30 Read block 202 len 24576 29/30 Read block 444 len 32768 30/30 Read block 85 len 4096 ---------------------------------------------------------------- Crucible gen:134 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 10 Done Done Done false 1025 Acked Read 7 Done Done Done false 1026 Acked Write 3 Done Done Done false 1027 Acked Read 6 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Read 1 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 6 5 5 16 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:27:44.14314972Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1716} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 112 112 112 199 199 199 F F F 1 020-039 124 124 124 228 228 228 F F F 2 040-059 131 131 131 240 240 240 F F F 3 060-079 131 131 131 240 240 240 F F F 4 080-099 124 124 124 227 227 227 F F F 5 100-119 128 128 128 237 237 237 F F F 6 120-139 124 124 124 226 226 226 F F F 7 140-159 134 134 134 249 249 249 F F F 8 160-179 134 134 134 249 249 249 F F F 9 180-199 134 134 134 249 249 249 F F F 10 200-219 134 134 134 247 247 247 F F F 11 220-239 134 134 134 247 247 247 F F F 12 240-259 134 134 134 249 249 249 F F F 13 260-279 133 133 133 246 246 246 F F F 14 280-299 134 134 134 252 252 252 F F F 15 300-319 134 134 134 250 250 250 T T T 16 320-339 134 134 134 250 250 250 F F F 17 340-359 132 132 132 242 242 242 F F F 18 360-379 132 132 132 243 243 243 F F F 19 380-399 134 134 134 248 248 248 F F F 20 400-419 126 126 126 231 231 231 F F F 21 420-439 134 134 134 249 249 249 F F F 22 440-459 122 122 122 223 223 223 F F F 23 460-479 134 134 134 247 247 247 F F F 24 480-499 132 132 132 244 244 244 F F F 25 500-519 131 131 131 240 240 240 T T T 26 520-539 134 134 134 249 249 249 F F F 27 540-559 133 133 133 246 246 246 F F F 28 560-579 128 128 128 237 237 237 F F F 29 580-599 128 128 128 237 237 237 F F F Max gen: 134, Max flush: 252 A Difference in extent metadata is expected here On loop 67 Verifying data now No change after verify Loop: 67 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 112 112 112 199 199 199 F F F 1 020-039 124 124 124 228 228 228 F F F 2 040-059 131 131 131 240 240 240 F F F 3 060-079 131 131 131 240 240 240 F F F 4 080-099 124 124 124 227 227 227 F F F 5 100-119 128 128 128 237 237 237 F F F 6 120-139 124 124 124 226 226 226 F F F 7 140-159 134 134 134 249 249 249 F F F 8 160-179 134 134 134 249 249 249 F F F 9 180-199 134 134 134 249 249 249 F F F 10 200-219 134 134 134 247 247 247 F F F 11 220-239 134 134 134 247 247 247 F F F 12 240-259 134 134 134 249 249 249 F F F 13 260-279 133 133 133 246 246 246 F F F 14 280-299 134 134 134 252 252 252 F F F 15 300-319 135 135 135 253 253 253 F F F 16 320-339 134 134 134 250 250 250 F F F 17 340-359 132 132 132 242 242 242 F F F 18 360-379 132 132 132 243 243 243 F F F 19 380-399 134 134 134 248 248 248 F F F 20 400-419 126 126 126 231 231 231 F F F 21 420-439 134 134 134 249 249 249 F F F 22 440-459 122 122 122 223 223 223 F F F 23 460-479 134 134 134 247 247 247 F F F 24 480-499 132 132 132 244 244 244 F F F 25 500-519 135 135 135 253 253 253 F F F 26 520-539 134 134 134 249 249 249 F F F 27 540-559 133 133 133 246 246 246 F F F 28 560-579 128 128 128 237 237 237 F F F 29 580-599 128 128 128 237 237 237 F F F Max gen: 135, Max flush: 253 Begin loop 68 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:48.418105566Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1724,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:27:48.461479356Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1724,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:48.461859398Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1724,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:48.494162479Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1724,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:48.494721291Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1724} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"0fd98d0d5d909611f8b408ccda2559d35fe117bb\",\n git_commit_timestamp: \"2024-11-08T20:05:44.000000000Z\",\n git_branch: \"mkeeter/drop-skipped-backpressure-guard\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:48.49475302Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1724} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:48.494766759Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1724} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:48.494781478Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1724} {"msg":"Crucible 35565d4b-4954-4c92-ab03-ac425241f2b6 has session id: 5e98aedd-8bc9-418a-9a0c-17b214fd3b1b","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:48.495204728Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1724,"session_id":"5e98aedd-8bc9-418a-9a0c-17b214fd3b1b"} {"msg":"Upstairs opts: Upstairs UUID: 35565d4b-4954-4c92-ab03-ac425241f2b6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:48.495250046Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1724,"session_id":"5e98aedd-8bc9-418a-9a0c-17b214fd3b1b"} {"msg":"Crucible stats registered with UUID: 35565d4b-4954-4c92-ab03-ac425241f2b6","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:48.495267635Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1724,"session_id":"5e98aedd-8bc9-418a-9a0c-17b214fd3b1b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:48.526390373Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1724,"task":"crutest"} {"msg":"The guest has requested activation with gen:136","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:48.526423422Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1724} {"msg":"35565d4b-4954-4c92-ab03-ac425241f2b6 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:48.526448211Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1724,"session_id":"5e98aedd-8bc9-418a-9a0c-17b214fd3b1b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:48.52646486Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1724,"client":"0","":"downstairs","session_id":"5e98aedd-8bc9-418a-9a0c-17b214fd3b1b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:48.526491049Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1724,"client":"0","":"downstairs","session_id":"5e98aedd-8bc9-418a-9a0c-17b214fd3b1b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:48.526508478Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1724,"client":"1","":"downstairs","session_id":"5e98aedd-8bc9-418a-9a0c-17b214fd3b1b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:48.526525537Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1724,"client":"1","":"downstairs","session_id":"5e98aedd-8bc9-418a-9a0c-17b214fd3b1b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:48.526542116Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1724,"client":"2","":"downstairs","session_id":"5e98aedd-8bc9-418a-9a0c-17b214fd3b1b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:48.526559025Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1724,"client":"2","":"downstairs","session_id":"5e98aedd-8bc9-418a-9a0c-17b214fd3b1b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:48.526575484Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1724,"":"io task","client":"2","":"downstairs","session_id":"5e98aedd-8bc9-418a-9a0c-17b214fd3b1b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:48.526592544Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1724,"":"io task","client":"0","":"downstairs","session_id":"5e98aedd-8bc9-418a-9a0c-17b214fd3b1b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:48.526614833Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1724,"":"io task","client":"1","":"downstairs","session_id":"5e98aedd-8bc9-418a-9a0c-17b214fd3b1b"} {"msg":"ds_connection connected from Ok(127.0.0.1:63805)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:48.526689819Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1724,"":"io task","client":"2","":"downstairs","session_id":"5e98aedd-8bc9-418a-9a0c-17b214fd3b1b"} {"msg":"ds_connection connected from Ok(127.0.0.1:59657)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:48.526745516Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1724,"":"io task","client":"0","":"downstairs","session_id":"5e98aedd-8bc9-418a-9a0c-17b214fd3b1b"} {"msg":"ds_connection connected from Ok(127.0.0.1:63989)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:48.526784204Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1724,"":"io task","client":"1","":"downstairs","session_id":"5e98aedd-8bc9-418a-9a0c-17b214fd3b1b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:48.527502629Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1724,"client":"2","":"downstairs","session_id":"5e98aedd-8bc9-418a-9a0c-17b214fd3b1b"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:48.527530648Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1724,"client":"2","":"downstairs","session_id":"5e98aedd-8bc9-418a-9a0c-17b214fd3b1b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:48.52768025Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1724,"client":"1","":"downstairs","session_id":"5e98aedd-8bc9-418a-9a0c-17b214fd3b1b"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:48.527721239Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1724,"client":"1","":"downstairs","session_id":"5e98aedd-8bc9-418a-9a0c-17b214fd3b1b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:48.527751237Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1724,"client":"0","":"downstairs","session_id":"5e98aedd-8bc9-418a-9a0c-17b214fd3b1b"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:48.527777126Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1724,"client":"0","":"downstairs","session_id":"5e98aedd-8bc9-418a-9a0c-17b214fd3b1b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:48.528272141Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1724,"client":"2","":"downstairs","session_id":"5e98aedd-8bc9-418a-9a0c-17b214fd3b1b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:48.528314379Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1724,"client":"1","":"downstairs","session_id":"5e98aedd-8bc9-418a-9a0c-17b214fd3b1b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:48.528455563Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1724,"client":"0","":"downstairs","session_id":"5e98aedd-8bc9-418a-9a0c-17b214fd3b1b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:48.528871852Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1724,"client":"1","":"downstairs","session_id":"5e98aedd-8bc9-418a-9a0c-17b214fd3b1b"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:48.52890963Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1724,"client":"1","":"downstairs","session_id":"5e98aedd-8bc9-418a-9a0c-17b214fd3b1b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:48.528932399Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1724,"session_id":"5e98aedd-8bc9-418a-9a0c-17b214fd3b1b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:48.528987537Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1724,"client":"0","":"downstairs","session_id":"5e98aedd-8bc9-418a-9a0c-17b214fd3b1b"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:48.529020225Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1724,"client":"0","":"downstairs","session_id":"5e98aedd-8bc9-418a-9a0c-17b214fd3b1b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:48.529035344Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1724,"session_id":"5e98aedd-8bc9-418a-9a0c-17b214fd3b1b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:48.529047774Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1724,"client":"2","":"downstairs","session_id":"5e98aedd-8bc9-418a-9a0c-17b214fd3b1b"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:48.529061353Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1724,"client":"2","":"downstairs","session_id":"5e98aedd-8bc9-418a-9a0c-17b214fd3b1b"} {"msg":"[0]R flush_numbers[0..12]: [199, 228, 240, 240, 227, 237, 226, 249, 249, 249, 247, 247]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:48.529075202Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1724,"":"downstairs","":"downstairs","session_id":"5e98aedd-8bc9-418a-9a0c-17b214fd3b1b"} {"msg":"[0]R generation[0..12]: [112, 124, 131, 131, 124, 128, 124, 134, 134, 134, 134, 134]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:48.529089002Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1724,"":"downstairs","":"downstairs","session_id":"5e98aedd-8bc9-418a-9a0c-17b214fd3b1b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:48.529102891Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1724,"":"downstairs","":"downstairs","session_id":"5e98aedd-8bc9-418a-9a0c-17b214fd3b1b"} {"msg":"[1]R flush_numbers[0..12]: [199, 228, 240, 240, 227, 237, 226, 249, 249, 249, 247, 247]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:48.52911665Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1724,"":"downstairs","":"downstairs","session_id":"5e98aedd-8bc9-418a-9a0c-17b214fd3b1b"} {"msg":"[1]R generation[0..12]: [112, 124, 131, 131, 124, 128, 124, 134, 134, 134, 134, 134]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:48.529131579Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1724,"":"downstairs","":"downstairs","session_id":"5e98aedd-8bc9-418a-9a0c-17b214fd3b1b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:48.529145789Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1724,"":"downstairs","":"downstairs","session_id":"5e98aedd-8bc9-418a-9a0c-17b214fd3b1b"} {"msg":"[2]R flush_numbers[0..12]: [199, 228, 240, 240, 227, 237, 226, 249, 249, 249, 247, 247]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:48.529159408Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1724,"":"downstairs","":"downstairs","session_id":"5e98aedd-8bc9-418a-9a0c-17b214fd3b1b"} {"msg":"[2]R generation[0..12]: [112, 124, 131, 131, 124, 128, 124, 134, 134, 134, 134, 134]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:48.529172928Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1724,"":"downstairs","":"downstairs","session_id":"5e98aedd-8bc9-418a-9a0c-17b214fd3b1b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:48.529195247Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1724,"":"downstairs","":"downstairs","session_id":"5e98aedd-8bc9-418a-9a0c-17b214fd3b1b"} {"msg":"Max found gen is 136","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:48.529221075Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1724,"":"downstairs","":"downstairs","session_id":"5e98aedd-8bc9-418a-9a0c-17b214fd3b1b"} {"msg":"Generation requested: 136 >= found:136","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:48.529248684Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1724,"":"downstairs","":"downstairs","session_id":"5e98aedd-8bc9-418a-9a0c-17b214fd3b1b"} {"msg":"Next flush: 254","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:48.529266743Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1724,"":"downstairs","":"downstairs","session_id":"5e98aedd-8bc9-418a-9a0c-17b214fd3b1b"} {Wait for a query_work_queue command to finish before sending IO "msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:48.529299681Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1724,"":"downstairs","":"downstairs","session_id":"5e98aedd-8bc9-418a-9a0c-17b214fd3b1b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:48.52932765Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1724,"session_id":"5e98aedd-8bc9-418a-9a0c-17b214fd3b1b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:48.529347139Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1724,"client":"0","":"downstairs","session_id":"5e98aedd-8bc9-418a-9a0c-17b214fd3b1b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:48.529361158Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1724,"client":"1","":"downstairs","session_id":"5e98aedd-8bc9-418a-9a0c-17b214fd3b1b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:48.529376308Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1724,"client":"2","":"downstairs","session_id":"5e98aedd-8bc9-418a-9a0c-17b214fd3b1b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:48.529400737Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1724,"session_id":"5e98aedd-8bc9-418a-9a0c-17b214fd3b1b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:48.529426325Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1724,"session_id":"5e98aedd-8bc9-418a-9a0c-17b214fd3b1b"} {"msg":"35565d4b-4954-4c92-ab03-ac425241f2b6 is now active with session: 5e98aedd-8bc9-418a-9a0c-17b214fd3b1b","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:48.529448654Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1724,"session_id":"5e98aedd-8bc9-418a-9a0c-17b214fd3b1b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:48.529461594Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1724,"session_id":"5e98aedd-8bc9-418a-9a0c-17b214fd3b1b"} {"msg":"The guest has finished waiting for activation with:136","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:48.529479673Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1724} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 490 len 16384 data: 12 9 7 8 02/30 Write block 582 len 32768 data: 10 13 14 17 13 13 10 9 03/30 Write block 86 len 40960 data: 11 10 11 10 9 7 8 8 6 7 04/30 Read block 123 len 20480 05/30 Write block 342 len 36864 data: 14 12 10 11 13 11 9 12 12 06/30 Read block 332 len 16384 07/30 Flush 08/30 Write block 299 len 36864 data: 14 13 14 13 13 12 9 9 5 09/30 Read block 66 len 8192 10/30 Read block 138 len 8192 11/30 Read block 74 len 12288 12/30 Read block 300 len 16384 13/30 Read block 134 len 36864 14/30 Read block 6 len 12288 15/30 Write block 462 len 8192 data: 6 7 16/30 Read block 504 len 16384 17/30 Read block 8 len 40960 18/30 Read block 537 len 28672 19/30 Write block 488 len 36864 data: 11 13 13 10 8 9 8 8 8 20/30 Read block 572 len 4096 21/30 Read block 456 len 8192 22/30 Flush 23/30 Write block 459 len 12288 data: 8 7 7 24/30 Read block 509 len 40960 25/30 Write block 6 len 12288 data: 8 8 9 26/30 Read block 355 len 24576 27/30 Read block 230 len 4096 28/30 Write block 527 len 24576 data: 8 10 8 8 12 10 29/30 Write block 25 len 4096 data: 11 30/30 Write block 429 len 24576 data: 13 14 13 11 10 10 ---------------------------------------------------------------- Crucible gen:136 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 3 Done Done Done false 1023 Acked Read 10 Done Done Done false 1024 Acked Write 3 Done Done Done false 1025 Acked Read 6 Done Done Done false 1026 Acked Read 1 Done Done Done false 1027 Acked Write 6 Sent Sent Sent false 1028 Acked Write 1 Sent Sent Sent false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 3 3 9 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1021 1021 1021 Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:27:48.553933478Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1724} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 112 112 112 199 199 199 T T T 1 020-039 124 124 124 228 228 228 T T T 2 040-059 131 131 131 240 240 240 F F F 3 060-079 131 131 131 240 240 240 F F F 4 080-099 136 136 136 254 254 254 F F F 5 100-119 128 128 128 237 237 237 F F F 6 120-139 124 124 124 226 226 226 F F F 7 140-159 134 134 134 249 249 249 F F F 8 160-179 134 134 134 249 249 249 F F F 9 180-199 134 134 134 249 249 249 F F F 10 200-219 134 134 134 247 247 247 F F F 11 220-239 134 134 134 247 247 247 F F F 12 240-259 134 134 134 249 249 249 F F F 13 260-279 133 133 133 246 246 246 F F F 14 280-299 136 136 136 255 255 255 F F F 15 300-319 136 136 136 255 255 255 F F F 16 320-339 134 134 134 250 250 250 F F F 17 340-359 136 136 136 254 254 254 F F F 18 360-379 132 132 132 243 243 243 F F F 19 380-399 134 134 134 248 248 248 F F F 20 400-419 126 126 126 231 231 231 F F F 21 420-439 134 134 134 249 249 249 T T T 22 440-459 122 122 122 223 223 223 T T T 23 460-479 136 136 136 255 255 255 T T T 24 480-499 136 136 136 255 255 255 F F F 25 500-519 135 135 135 253 253 253 F F F 26 520-539 134 134 134 249 249 249 T T T 27 540-559 133 133 133 246 246 246 F F F 28 560-579 128 128 128 237 237 237 F F F 29 580-599 136 136 136 254 254 254 F F F Max gen: 136, Max flush: 255 A Difference in extent metadata is expected here On loop 68 Verifying data now No change after verify Loop: 68 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 137 137 137 256 256 256 F F F 1 020-039 137 137 137 256 256 256 F F F 2 040-059 131 131 131 240 240 240 F F F 3 060-079 131 131 131 240 240 240 F F F 4 080-099 136 136 136 254 254 254 F F F 5 100-119 128 128 128 237 237 237 F F F 6 120-139 124 124 124 226 226 226 F F F 7 140-159 134 134 134 249 249 249 F F F 8 160-179 134 134 134 249 249 249 F F F 9 180-199 134 134 134 249 249 249 F F F 10 200-219 134 134 134 247 247 247 F F F 11 220-239 134 134 134 247 247 247 F F F 12 240-259 134 134 134 249 249 249 F F F 13 260-279 133 133 133 246 246 246 F F F 14 280-299 136 136 136 255 255 255 F F F 15 300-319 136 136 136 255 255 255 F F F 16 320-339 134 134 134 250 250 250 F F F 17 340-359 136 136 136 254 254 254 F F F 18 360-379 132 132 132 243 243 243 F F F 19 380-399 134 134 134 248 248 248 F F F 20 400-419 126 126 126 231 231 231 F F F 21 420-439 137 137 137 256 256 256 F F F 22 440-459 137 137 137 256 256 256 F F F 23 460-479 137 137 137 256 256 256 F F F 24 480-499 136 136 136 255 255 255 F F F 25 500-519 135 135 135 253 253 253 F F F 26 520-539 137 137 137 256 256 256 F F F 27 540-559 133 133 133 246 246 246 F F F 28 560-579 128 128 128 237 237 237 F F F 29 580-599 136 136 136 254 254 254 F F F Max gen: 137, Max flush: 256 Begin loop 69 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:52.918042744Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1732,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:52.962343099Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1732,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:52.963058694Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1732} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"0fd98d0d5d909611f8b408ccda2559d35fe117bb\",\n git_commit_timestamp: \"2024-11-08T20:05:44.000000000Z\",\n git_branch: \"mkeeter/drop-skipped-backpressure-guard\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:52.963103231Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1732} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:52.963120921Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1732} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:52.9631399Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1732} {"msg":"Crucible 69253ebd-6a2e-4afb-a3d8-b04ad42c2721 has session id: 9b821d27-4e90-4baa-9ab7-25af0d02d858","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:52.963573959Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1732,"session_id":"9b821d27-4e90-4baa-9ab7-25af0d02d858"} {"msg":"Upstairs opts: Upstairs UUID: 69253ebd-6a2e-4afb-a3d8-b04ad42c2721, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:52.963630696Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1732,"session_id":"9b821d27-4e90-4baa-9ab7-25af0d02d858"} {"msg":"Crucible stats registered with UUID: 69253ebd-6a2e-4afb-a3d8-b04ad42c2721","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:52.963653695Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1732,"session_id":"9b821d27-4e90-4baa-9ab7-25af0d02d858"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:52.994935236Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1732,"task":"crutest"} {"msg":"The guest has requested activation with gen:138","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:52.994980994Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1732} {"msg":"69253ebd-6a2e-4afb-a3d8-b04ad42c2721 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:52.99504669Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1732,"session_id":"9b821d27-4e90-4baa-9ab7-25af0d02d858"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:52.995074929Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1732,"client":"0","":"downstairs","session_id":"9b821d27-4e90-4baa-9ab7-25af0d02d858"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:52.995098818Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1732,"client":"0","":"downstairs","session_id":"9b821d27-4e90-4baa-9ab7-25af0d02d858"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:52.995113827Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1732,"client":"1","":"downstairs","session_id":"9b821d27-4e90-4baa-9ab7-25af0d02d858"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:52.995127546Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1732,"client":"1","":"downstairs","session_id":"9b821d27-4e90-4baa-9ab7-25af0d02d858"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:52.995141156Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1732,"client":"2","":"downstairs","session_id":"9b821d27-4e90-4baa-9ab7-25af0d02d858"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:52.995155945Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1732,"client":"2","":"downstairs","session_id":"9b821d27-4e90-4baa-9ab7-25af0d02d858"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:52.995179754Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1732,"":"io task","client":"2","":"downstairs","session_id":"9b821d27-4e90-4baa-9ab7-25af0d02d858"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:52.995204413Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1732,"":"io task","client":"0","":"downstairs","session_id":"9b821d27-4e90-4baa-9ab7-25af0d02d858"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:52.995228132Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1732,"":"io task","client":"1","":"downstairs","session_id":"9b821d27-4e90-4baa-9ab7-25af0d02d858"} {"msg":"ds_connection connected from Ok(127.0.0.1:43314)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:52.99525348Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1732,"":"io task","client":"2","":"downstairs","session_id":"9b821d27-4e90-4baa-9ab7-25af0d02d858"} {"msg":"ds_connection connected from Ok(127.0.0.1:52925)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:52.995349476Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1732,"":"io task","client":"0","":"downstairs","session_id":"9b821d27-4e90-4baa-9ab7-25af0d02d858"} {"msg":"ds_connection connected from Ok(127.0.0.1:33786)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:52.995400133Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1732,"":"io task","client":"1","":"downstairs","session_id":"9b821d27-4e90-4baa-9ab7-25af0d02d858"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:52.996108449Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1732,"client":"2","":"downstairs","session_id":"9b821d27-4e90-4baa-9ab7-25af0d02d858"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:52.996153206Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1732,"client":"2","":"downstairs","session_id":"9b821d27-4e90-4baa-9ab7-25af0d02d858"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:52.996178825Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1732,"client":"0","":"downstairs","session_id":"9b821d27-4e90-4baa-9ab7-25af0d02d858"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:52.996203984Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1732,"client":"0","":"downstairs","session_id":"9b821d27-4e90-4baa-9ab7-25af0d02d858"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:52.996310349Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1732,"client":"1","":"downstairs","session_id":"9b821d27-4e90-4baa-9ab7-25af0d02d858"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:52.996352526Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1732,"client":"1","":"downstairs","session_id":"9b821d27-4e90-4baa-9ab7-25af0d02d858"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:52.996816664Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1732,"client":"2","":"downstairs","session_id":"9b821d27-4e90-4baa-9ab7-25af0d02d858"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:52.99689481Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1732,"client":"0","":"downstairs","session_id":"9b821d27-4e90-4baa-9ab7-25af0d02d858"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:52.996938218Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1732,"client":"1","":"downstairs","session_id":"9b821d27-4e90-4baa-9ab7-25af0d02d858"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:52.997436713Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1732,"client":"0","":"downstairs","session_id":"9b821d27-4e90-4baa-9ab7-25af0d02d858"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:52.997465752Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1732,"client":"0","":"downstairs","session_id":"9b821d27-4e90-4baa-9ab7-25af0d02d858"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:52.997480802Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1732,"session_id":"9b821d27-4e90-4baa-9ab7-25af0d02d858"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:52.997493631Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1732,"client":"2","":"downstairs","session_id":"9b821d27-4e90-4baa-9ab7-25af0d02d858"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:52.99750726Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1732,"client":"2","":"downstairs","session_id":"9b821d27-4e90-4baa-9ab7-25af0d02d858"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:52.99752082Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1732,"session_id":"9b821d27-4e90-4baa-9ab7-25af0d02d858"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:52.997533109Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1732,"client":"1","":"downstairs","session_id":"9b821d27-4e90-4baa-9ab7-25af0d02d858"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:52.997549818Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1732,"client":"1","":"downstairs","session_id":"9b821d27-4e90-4baa-9ab7-25af0d02d858"} {"msg":"[0]R flush_numbers[0..12]: [256, 256, 240, 240, 254, 237, 226, 249, 249, 249, 247, 247]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:52.997563977Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1732,"":"downstairs","":"downstairs","session_id":"9b821d27-4e90-4baa-9ab7-25af0d02d858"} {"msg":"[0]R generation[0..12]: [137, 137, 131, 131, 136, 128, 124, 134, 134, 134, 134, 134]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:52.997577917Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1732,"":"downstairs","":"downstairs","session_id":"9b821d27-4e90-4baa-9ab7-25af0d02d858"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:52.997591836Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1732,"":"downstairs","":"downstairs","session_id":"9b821d27-4e90-4baa-9ab7-25af0d02d858"} {"msg":"[1]R flush_numbers[0..12]: [256, 256, 240, 240, 254, 237, 226, 249, 249, 249, 247, 247]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:52.997605515Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1732,"":"downstairs","":"downstairs","session_id":"9b821d27-4e90-4baa-9ab7-25af0d02d858"} {"msg":"[1]R generation[0..12]: [137, 137, 131, 131, 136, 128, 124, 134, 134, 134, 134, 134]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:52.997618975Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1732,"":"downstairs","":"downstairs","session_id":"9b821d27-4e90-4baa-9ab7-25af0d02d858"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:52.997632964Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1732,"":"downstairs","":"downstairs","session_id":"9b821d27-4e90-4baa-9ab7-25af0d02d858"} {"msg":"[2]R flush_numbers[0..12]: [256, 256, 240, 240, 254, 237, 226, 249, 249, 249, 247, 247]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:52.997646383Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1732,"":"downstairs","":"downstairs","session_id":"9b821d27-4e90-4baa-9ab7-25af0d02d858"} {"msg":"[2]R generation[0..12]: [137, 137, 131, 131, 136, 128, 124, 134, 134, 134, 134, 134]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:52.997660173Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1732,"":"downstairs","":"downstairs","session_id":"9b821d27-4e90-4baa-9ab7-25af0d02d858"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:52.997674032Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1732,"":"downstairs","":"downstairs","session_id":"9b821d27-4e90-4baa-9ab7-25af0d02d858"} {"msg":"Max found gen is 138","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:52.997687161Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1732,"":"downstairs","":"downstairs","session_id":"9b821d27-4e90-4baa-9ab7-25af0d02d858"} {"msg":"Generation requested: 138 >= found:138","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:52.997700171Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1732,"":"downstairs","":"downstairs","session_id":"9b821d27-4e90-4baa-9ab7-25af0d02d858"} {Wait for a query_work_queue command to finish before sending IO "msg":"Next flush: 257","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:52.99772067Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1732,"":"downstairs","":"downstairs","session_id":"9b821d27-4e90-4baa-9ab7-25af0d02d858"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:52.997733969Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1732,"":"downstairs","":"downstairs","session_id":"9b821d27-4e90-4baa-9ab7-25af0d02d858"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:52.997748748Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1732,"session_id":"9b821d27-4e90-4baa-9ab7-25af0d02d858"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:52.997760758Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1732,"client":"0","":"downstairs","session_id":"9b821d27-4e90-4baa-9ab7-25af0d02d858"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:52.997774127Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1732,"client":"1","":"downstairs","session_id":"9b821d27-4e90-4baa-9ab7-25af0d02d858"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:52.997787576Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1732,"client":"2","":"downstairs","session_id":"9b821d27-4e90-4baa-9ab7-25af0d02d858"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:52.997800966Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1732,"session_id":"9b821d27-4e90-4baa-9ab7-25af0d02d858"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:52.997813445Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1732,"session_id":"9b821d27-4e90-4baa-9ab7-25af0d02d858"} {"msg":"69253ebd-6a2e-4afb-a3d8-b04ad42c2721 is now active with session: 9b821d27-4e90-4baa-9ab7-25af0d02d858","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:52.997826265Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1732,"session_id":"9b821d27-4e90-4baa-9ab7-25af0d02d858"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:52.997838294Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1732,"session_id":"9b821d27-4e90-4baa-9ab7-25af0d02d858"} {"msg":"The guest has finished waiting for activation with:138","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:52.997850503Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1732} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 315 len 8192 data: 14 12 02/30 Read block 216 len 4096 03/30 Write block 584 len 20480 data: 15 18 14 14 11 04/30 Write block 375 len 28672 data: 9 9 11 11 11 10 9 05/30 Read block 558 len 28672 06/30 Read block 510 len 20480 07/30 Read block 66 len 36864 08/30 Read block 293 len 40960 09/30 Read block 546 len 16384 10/30 Write block 481 len 24576 data: 16 15 14 13 11 11 11/30 Write block 427 len 28672 data: 8 8 14 15 14 12 11 12/30 Write block 531 len 12288 data: 13 11 11 13/30 Read block 98 len 8192 14/30 Read block 56 len 16384 15/30 Write block 130 len 16384 data: 9 9 10 12 16/30 Read block 170 len 32768 17/30 Read block 354 len 4096 18/30 Read block 23 len 28672 19/30 Read block 228 len 4096 20/30 Read block 163 len 4096 21/30 Write block 202 len 16384 data: 7 6 8 9 22/30 Read block 409 len 12288 23/30 Write block 174 len 40960 data: 9 10 9 10 9 10 10 9 10 11 24/30 Write block 54 len 36864 data: 11 12 10 10 8 5 7 7 7 25/30 Write block 222 len 4096 data: 14 26/30 Write block 70 len 24576 data: 9 8 9 10 10 9 27/30 Read block 542 len 36864 28/30 Read block 430 len 32768 29/30 Write block 77 len 8192 data: 10 9 30/30 Read block 581 len 36864 ---------------------------------------------------------------- Crucible gen:138 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 2 Done Done Done false 1001 Acked Read 1 Done Done Done false 1002 Acked Write 5 Done Done Done false 1003 Acked Write 7 Done Done Done false 1004 Acked Read 7 Done Done Done false 1005 Acked Read 5 Done Done Done false 1006 Acked Read 9 Done Done Done false 1007 Acked Read 10 Done Done Done false 1008 Acked Read 4 Done Done Done false 1009 Acked Write 6 Done Done Done false 1010 Acked Write 7 Done Done Done false 1011 Acked Write 3 Done Done Done false 1012 Acked Read 2 Done Done Done false 1013 Acked Read 4 Done Done Done false 1014 Acked Write 4 Done Done Done false 1015 Acked Read 8 Done Done Done false 1016 Acked Read 1 Done Done Done false 1017 Acked Read 7 Done Done Done false 1018 Acked Read 1 Done Done Done false 1019 Acked Read 1 Done Done Done false 1020 Acked Write 4 Done Done Done false 1021 Acked Read 3 Done Done Done false 1022 Acked Write 10 Done Done Done false 1023 Acked Write 9 Done Done Done false 1024 Acked Write 1 Done Done Done false 1025 Acked Write 6 Done Done Done false 1026 Acked Read 9 Done Done Done false 1027 Acked Read 8 Done Done Done false 1028 Acked Write 2 Done Done Sent false 1029 Acked Read 9 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 2 3 Done 30 29 28 87 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 0 0 0 Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:27:53.0165412Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1732} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 137 137 137 256 256 256 F F F 1 020-039 137 137 137 256 256 256 F F F 2 040-059 131 131 131 240 240 240 T T T 3 060-079 131 131 131 240 240 240 T T T 4 080-099 136 136 136 254 254 254 F F F 5 100-119 128 128 128 237 237 237 F F F 6 120-139 124 124 124 226 226 226 T T T 7 140-159 134 134 134 249 249 249 F F F 8 160-179 134 134 134 249 249 249 T T T 9 180-199 134 134 134 249 249 249 T T T 10 200-219 134 134 134 247 247 247 T T T 11 220-239 134 134 134 247 247 247 T T T 12 240-259 134 134 134 249 249 249 F F F 13 260-279 133 133 133 246 246 246 F F F 14 280-299 136 136 136 255 255 255 F F F 15 300-319 136 136 136 255 255 255 T T T 16 320-339 134 134 134 250 250 250 F F F 17 340-359 136 136 136 254 254 254 F F F 18 360-379 132 132 132 243 243 243 T T T 19 380-399 134 134 134 248 248 248 T T T 20 400-419 126 126 126 231 231 231 F F F 21 420-439 137 137 137 256 256 256 T T T 22 440-459 137 137 137 256 256 256 F F F 23 460-479 137 137 137 256 256 256 F F F 24 480-499 136 136 136 255 255 255 T T T 25 500-519 135 135 135 253 253 253 F F F 26 520-539 137 137 137 256 256 256 T T T 27 540-559 133 133 133 246 246 246 F F F 28 560-579 128 128 128 237 237 237 F F F 29 580-599 136 136 136 254 254 254 T T T Max gen: 137, Max flush: 256 A Difference in extent metadata is expected here On loop 69 Verifying data now No change after verify Loop: 69 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 137 137 137 256 256 256 F F F 1 020-039 137 137 137 256 256 256 F F F 2 040-059 138 138 138 257 257 257 F F F 3 060-079 138 138 138 257 257 257 F F F 4 080-099 136 136 136 254 254 254 F F F 5 100-119 128 128 128 237 237 237 F F F 6 120-139 138 138 138 257 257 257 F F F 7 140-159 134 134 134 249 249 249 F F F 8 160-179 138 138 138 257 257 257 F F F 9 180-199 138 138 138 257 257 257 F F F 10 200-219 138 138 138 257 257 257 F F F 11 220-239 138 138 138 257 257 257 F F F 12 240-259 134 134 134 249 249 249 F F F 13 260-279 133 133 133 246 246 246 F F F 14 280-299 136 136 136 255 255 255 F F F 15 300-319 138 138 138 257 257 257 F F F 16 320-339 134 134 134 250 250 250 F F F 17 340-359 136 136 136 254 254 254 F F F 18 360-379 138 138 138 257 257 257 F F F 19 380-399 138 138 138 257 257 257 F F F 20 400-419 126 126 126 231 231 231 F F F 21 420-439 138 138 138 257 257 257 F F F 22 440-459 137 137 137 256 256 256 F F F 23 460-479 137 137 137 256 256 256 F F F 24 480-499 138 138 138 257 257 257 F F F 25 500-519 135 135 135 253 253 253 F F F 26 520-539 138 138 138 257 257 257 F F F 27 540-559 133 133 133 246 246 246 F F F 28 560-579 128 128 128 237 237 237 F F F 29 580-599 138 138 138 257 257 257 F F F Max gen: 138, Max flush: 257 Begin loop 70 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:57.46940689Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1740,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:57.513738183Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1740,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:57.514406621Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1740} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"0fd98d0d5d909611f8b408ccda2559d35fe117bb\",\n git_commit_timestamp: \"2024-11-08T20:05:44.000000000Z\",\n git_branch: \"mkeeter/drop-skipped-backpressure-guard\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:57.514442679Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1740} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:57.514456979Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1740} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:57.514471058Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1740} {"msg":"Crucible 52338db7-ad77-461c-857f-3668cc68f4d7 has session id: ae39e557-baa1-494b-8bfe-68a70e86f089","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:57.514949104Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1740,"session_id":"ae39e557-baa1-494b-8bfe-68a70e86f089"} {"msg":"Upstairs opts: Upstairs UUID: 52338db7-ad77-461c-857f-3668cc68f4d7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:57.514982513Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1740,"session_id":"ae39e557-baa1-494b-8bfe-68a70e86f089"} {"msg":"Crucible stats registered with UUID: 52338db7-ad77-461c-857f-3668cc68f4d7","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:57.515000782Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1740,"session_id":"ae39e557-baa1-494b-8bfe-68a70e86f089"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:57.546907193Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1740,"task":"crutest"} {"msg":"The guest has requested activation with gen:140","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:57.546940441Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1740} {"msg":"52338db7-ad77-461c-857f-3668cc68f4d7 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:57.546976459Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1740,"session_id":"ae39e557-baa1-494b-8bfe-68a70e86f089"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:57.546993168Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1740,"client":"0","":"downstairs","session_id":"ae39e557-baa1-494b-8bfe-68a70e86f089"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:57.547021117Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1740,"client":"0","":"downstairs","session_id":"ae39e557-baa1-494b-8bfe-68a70e86f089"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:57.547038806Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1740,"client":"1","":"downstairs","session_id":"ae39e557-baa1-494b-8bfe-68a70e86f089"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:57.547055755Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1740,"client":"1","":"downstairs","session_id":"ae39e557-baa1-494b-8bfe-68a70e86f089"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:57.547080104Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1740,"client":"2","":"downstairs","session_id":"ae39e557-baa1-494b-8bfe-68a70e86f089"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:57.547111472Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1740,"client":"2","":"downstairs","session_id":"ae39e557-baa1-494b-8bfe-68a70e86f089"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:57.547144441Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1740,"":"io task","client":"2","":"downstairs","session_id":"ae39e557-baa1-494b-8bfe-68a70e86f089"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:57.547178239Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1740,"":"io task","client":"0","":"downstairs","session_id":"ae39e557-baa1-494b-8bfe-68a70e86f089"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:57.547212078Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1740,"":"io task","client":"1","":"downstairs","session_id":"ae39e557-baa1-494b-8bfe-68a70e86f089"} {"msg":"ds_connection connected from Ok(127.0.0.1:35316)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:57.547246956Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1740,"":"io task","client":"2","":"downstairs","session_id":"ae39e557-baa1-494b-8bfe-68a70e86f089"} {"msg":"ds_connection connected from Ok(127.0.0.1:52708)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:57.547282374Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1740,"":"io task","client":"0","":"downstairs","session_id":"ae39e557-baa1-494b-8bfe-68a70e86f089"} {"msg":"ds_connection connected from Ok(127.0.0.1:40295)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:57.547499644Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1740,"":"io task","client":"1","":"downstairs","session_id":"ae39e557-baa1-494b-8bfe-68a70e86f089"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:57.548078675Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1740,"client":"0","":"downstairs","session_id":"ae39e557-baa1-494b-8bfe-68a70e86f089"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:57.548109464Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1740,"client":"0","":"downstairs","session_id":"ae39e557-baa1-494b-8bfe-68a70e86f089"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:57.548137032Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1740,"client":"2","":"downstairs","session_id":"ae39e557-baa1-494b-8bfe-68a70e86f089"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:57.548165651Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1740,"client":"2","":"downstairs","session_id":"ae39e557-baa1-494b-8bfe-68a70e86f089"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:57.548195699Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1740,"client":"1","":"downstairs","session_id":"ae39e557-baa1-494b-8bfe-68a70e86f089"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:57.548224658Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1740,"client":"1","":"downstairs","session_id":"ae39e557-baa1-494b-8bfe-68a70e86f089"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:57.548764102Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1740,"client":"1","":"downstairs","session_id":"ae39e557-baa1-494b-8bfe-68a70e86f089"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:57.548809229Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1740,"client":"2","":"downstairs","session_id":"ae39e557-baa1-494b-8bfe-68a70e86f089"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:57.548843978Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1740,"client":"0","":"downstairs","session_id":"ae39e557-baa1-494b-8bfe-68a70e86f089"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:57.549358593Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1740,"client":"1","":"downstairs","session_id":"ae39e557-baa1-494b-8bfe-68a70e86f089"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:57.549388061Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1740,"client":"1","":"downstairs","session_id":"ae39e557-baa1-494b-8bfe-68a70e86f089"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:57.549403711Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1740,"session_id":"ae39e557-baa1-494b-8bfe-68a70e86f089"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:57.54941641Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1740,"client":"0","":"downstairs","session_id":"ae39e557-baa1-494b-8bfe-68a70e86f089"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:57.549429789Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1740,"client":"0","":"downstairs","session_id":"ae39e557-baa1-494b-8bfe-68a70e86f089"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:57.549443169Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1740,"session_id":"ae39e557-baa1-494b-8bfe-68a70e86f089"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:57.549458648Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1740,"client":"2","":"downstairs","session_id":"ae39e557-baa1-494b-8bfe-68a70e86f089"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:57.549472277Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1740,"client":"2","":"downstairs","session_id":"ae39e557-baa1-494b-8bfe-68a70e86f089"} {"msg":"[0]R flush_numbers[0..12]: [256, 256, 257, 257, 254, 237, 257, 249, 257, 257, 257, 257]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:57.549486136Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1740,"":"downstairs","":"downstairs","session_id":"ae39e557-baa1-494b-8bfe-68a70e86f089"} {"msg":"[0]R generation[0..12]: [137, 137, 138, 138, 136, 128, 138, 134, 138, 138, 138, 138]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:57.549500186Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1740,"":"downstairs","":"downstairs","session_id":"ae39e557-baa1-494b-8bfe-68a70e86f089"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:57.549514185Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1740,"":"downstairs","":"downstairs","session_id":"ae39e557-baa1-494b-8bfe-68a70e86f089"} {"msg":"[1]R flush_numbers[0..12]: [256, 256, 257, 257, 254, 237, 257, 249, 257, 257, 257, 257]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:57.549528024Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1740,"":"downstairs","":"downstairs","session_id":"ae39e557-baa1-494b-8bfe-68a70e86f089"} {"msg":"[1]R generation[0..12]: [137, 137, 138, 138, 136, 128, 138, 134, 138, 138, 138, 138]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:57.549541684Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1740,"":"downstairs","":"downstairs","session_id":"ae39e557-baa1-494b-8bfe-68a70e86f089"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:57.549555353Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1740,"":"downstairs","":"downstairs","session_id":"ae39e557-baa1-494b-8bfe-68a70e86f089"} {"msg":"[2]R flush_numbers[0..12]: [256, 256, 257, 257, 254, 237, 257, 249, 257, 257, 257, 257]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:57.549568872Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1740,"":"downstairs","":"downstairs","session_id":"ae39e557-baa1-494b-8bfe-68a70e86f089"} {"msg":"[2]R generation[0..12]: [137, 137, 138, 138, 136, 128, 138, 134, 138, 138, 138, 138]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:57.549582512Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1740,"":"downstairs","":"downstairs","session_id":"ae39e557-baa1-494b-8bfe-68a70e86f089"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:57.549596351Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1740,"":"downstairs","":"downstairs","session_id":"ae39e557-baa1-494b-8bfe-68a70e86f089"} {"msg":"Max found gen is 139","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:57.549609281Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1740,"":"downstairs","":"downstairs","session_id":"ae39e557-baa1-494b-8bfe-68a70e86f089"} {"msg":"Generation requested: 140 >= found:139","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:57.54962224Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1740,"":"downstairs","":"downstairs","session_id":"ae39e557-baa1-494b-8bfe-68a70e86f089"} {"msg":"Next flush: 258","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-08T20:27:57.549635189Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1740,"":"downstairs","":"downstairs","session_id":"ae39e557-baa1-494b-8bfe-68a70e86f089"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:57.549661988Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1740,"":"downstairs","":"downstairs","session_id":"ae39e557-baa1-494b-8bfe-68a70e86f089"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:57.549675247Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1740,"session_id":"ae39e557-baa1-494b-8bfe-68a70e86f089"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:57.549687317Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1740,"client":"0","":"downstairs","session_id":"ae39e557-baa1-494b-8bfe-68a70e86f089"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:57.549700516Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1740,"client":"1","":"downstairs","session_id":"ae39e557-baa1-494b-8bfe-68a70e86f089"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:57.549713725Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1740,"client":"2","":"downstairs","session_id":"ae39e557-baa1-494b-8bfe-68a70e86f089"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:57.549727025Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1740,"session_id":"ae39e557-baa1-494b-8bfe-68a70e86f089"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:57.549739154Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1740,"session_id":"ae39e557-baa1-494b-8bfe-68a70e86f089"} {"msg":"52338db7-ad77-461c-857f-3668cc68f4d7 is now active with session: ae39e557-baa1-494b-8bfe-68a70e86f089","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:57.549752043Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1740,"session_id":"ae39e557-baa1-494b-8bfe-68a70e86f089"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:57.549764013Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1740,"session_id":"ae39e557-baa1-494b-8bfe-68a70e86f089"} {"msg":"The guest has finished waiting for activation with:140","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:27:57.549776282Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1740} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 16 len 12288 data: 13 12 9 02/30 Write block 461 len 12288 data: 8 7 8 03/30 Read block 418 len 32768 04/30 Write block 61 len 40960 data: 8 8 9 9 9 8 8 8 7 10 05/30 Read block 367 len 32768 06/30 Write block 581 len 8192 data: 10 11 07/30 Write block 384 len 16384 data: 7 6 6 6 08/30 Write block 312 len 28672 data: 15 15 12 15 13 9 11 09/30 Write block 474 len 8192 data: 13 15 10/30 Read block 575 len 32768 11/30 Write block 278 len 12288 data: 11 11 11 12/30 Flush 13/30 Write block 410 len 28672 data: 15 14 13 12 11 10 10 14/30 Write block 117 len 32768 data: 12 11 10 10 11 11 12 13 15/30 Read block 337 len 36864 16/30 Write block 5 len 28672 data: 8 9 9 10 9 9 11 17/30 Write block 320 len 28672 data: 13 9 10 10 9 9 12 18/30 Read block 297 len 40960 19/30 Read block 3 len 36864 20/30 Write block 460 len 12288 data: 8 9 8 21/30 Read block 449 len 4096 22/30 Write block 285 len 16384 data: 13 16 16 17 23/30 Read block 476 len 28672 24/30 Write block 104 len 24576 data: 8 8 9 13 11 12 25/30 Read block 468 len 40960 26/30 Read block 502 len 16384 27/30 Read block 254 len 4096 28/30 Write block 470 len 28672 data: 14 16 16 15 14 16 15 29/30 Write block 231 len 24576 data: 6 9 8 6 8 9 30/30 Write block 367 len 20480 data: 13 13 11 10 11 ---------------------------------------------------------------- Crucible gen:140 GIO:true work queues: Upstairs:0 downstairs:18 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1012 Acked Write 7 Done Done Done false 1013 Acked Write 8 Done Done Done false 1014 Acked Read 9 Done Done Done false 1015 Acked Write 7 Done Done Done false 1016 Acked Write 7 Done Done Done false 1017 Acked Read 10 Done Done Done false 1018 Acked Read 9 Done Done Done false 1019 Acked Write 3 Done Done Done false 1020 Acked Read 1 Done Done Done false 1021 Acked Write 4 Done Done Done false 1022 Acked Read 7 Done Done Done false 1023 Acked Write 6 Done Done Done false 1024 Acked Read 10 Done Done Done false 1025 Acked Read 4 Done Done Done false 1026 Acked Read 1 Done Done Done false 1027 Acked Write 7 Done Sent Sent false 1028 Acked Write 6 Sent Sent Sent false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 3 3 8 Done 16 15 15 46 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1011 1011 1011 Downstairs last five completed: 1011 1010 1009 1008 1007 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:27:57.570293139Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1740} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 140 140 140 258 258 258 T T T 1 020-039 137 137 137 256 256 256 F F F 2 040-059 138 138 138 257 257 257 F F F 3 060-079 140 140 140 258 258 258 F F F 4 080-099 136 136 136 254 254 254 F F F 5 100-119 128 128 128 237 237 237 T T T 6 120-139 138 138 138 257 257 257 T T T 7 140-159 134 134 134 249 249 249 F F F 8 160-179 138 138 138 257 257 257 F F F 9 180-199 138 138 138 257 257 257 F F F 10 200-219 138 138 138 257 257 257 F F F 11 220-239 138 138 138 257 257 257 T T T 12 240-259 134 134 134 249 249 249 F F F 13 260-279 140 140 140 258 258 258 F F F 14 280-299 140 140 140 258 258 258 T T T 15 300-319 140 140 140 258 258 258 F F F 16 320-339 134 134 134 250 250 250 T T T 17 340-359 136 136 136 254 254 254 F F F 18 360-379 138 138 138 257 257 257 T T T 19 380-399 140 140 140 258 258 258 F F F 20 400-419 126 126 126 231 231 231 T T T 21 420-439 138 138 138 257 257 257 F F F 22 440-459 137 137 137 256 256 256 F F F 23 460-479 140 140 140 258 258 258 T T T 24 480-499 138 138 138 257 257 257 F F F 25 500-519 135 135 135 253 253 253 F F F 26 520-539 138 138 138 257 257 257 F F F 27 540-559 133 133 133 246 246 246 F F F 28 560-579 128 128 128 237 237 237 F F F 29 580-599 140 140 140 258 258 258 F F F Max gen: 140, Max flush: 258 A Difference in extent metadata is expected here On loop 70 Verifying data now No change after verify Loop: 70 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 141 141 141 259 259 259 F F F 1 020-039 137 137 137 256 256 256 F F F 2 040-059 138 138 138 257 257 257 F F F 3 060-079 140 140 140 258 258 258 F F F 4 080-099 136 136 136 254 254 254 F F F 5 100-119 141 141 141 259 259 259 F F F 6 120-139 141 141 141 259 259 259 F F F 7 140-159 134 134 134 249 249 249 F F F 8 160-179 138 138 138 257 257 257 F F F 9 180-199 138 138 138 257 257 257 F F F 10 200-219 138 138 138 257 257 257 F F F 11 220-239 141 141 141 259 259 259 F F F 12 240-259 134 134 134 249 249 249 F F F 13 260-279 140 140 140 258 258 258 F F F 14 280-299 141 141 141 259 259 259 F F F 15 300-319 140 140 140 258 258 258 F F F 16 320-339 141 141 141 259 259 259 F F F 17 340-359 136 136 136 254 254 254 F F F 18 360-379 141 141 141 259 259 259 F F F 19 380-399 140 140 140 258 258 258 F F F 20 400-419 141 141 141 259 259 259 F F F 21 420-439 138 138 138 257 257 257 F F F 22 440-459 137 137 137 256 256 256 F F F 23 460-479 141 141 141 259 259 259 F F F 24 480-499 138 138 138 257 257 257 F F F 25 500-519 135 135 135 253 253 253 F F F 26 520-539 138 138 138 257 257 257 F F F 27 540-559 133 133 133 246 246 246 F F F 28 560-579 128 128 128 237 237 237 F F F 29 580-599 140 140 140 258 258 258 F F F Max gen: 141, Max flush: 259 Begin loop 71 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:01.948553546Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1748,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:01.99266651Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1748,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:01.99327549Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1748} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"0fd98d0d5d909611f8b408ccda2559d35fe117bb\",\n git_commit_timestamp: \"2024-11-08T20:05:44.000000000Z\",\n git_branch: \"mkeeter/drop-skipped-backpressure-guard\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:01.993313528Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1748} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:01.993328457Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1748} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:01.993342417Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1748} {"msg":"Crucible 8910ae79-7931-4f36-a738-4bf06c5e4cf8 has session id: 65996b6f-4393-4ede-81cb-df38c955c6ec","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:01.993801444Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1748,"session_id":"65996b6f-4393-4ede-81cb-df38c955c6ec"} {"msg":"Upstairs opts: Upstairs UUID: 8910ae79-7931-4f36-a738-4bf06c5e4cf8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:01.993849052Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1748,"session_id":"65996b6f-4393-4ede-81cb-df38c955c6ec"} {"msg":"Crucible stats registered with UUID: 8910ae79-7931-4f36-a738-4bf06c5e4cf8","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:01.9938722Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1748,"session_id":"65996b6f-4393-4ede-81cb-df38c955c6ec"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:02.02558831Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1748,"task":"crutest"} {"msg":"The guest has requested activation with gen:142","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:02.025627248Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1748} {"msg":"8910ae79-7931-4f36-a738-4bf06c5e4cf8 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:02.025658197Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1748,"session_id":"65996b6f-4393-4ede-81cb-df38c955c6ec"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:02.025678946Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1748,"client":"0","":"downstairs","session_id":"65996b6f-4393-4ede-81cb-df38c955c6ec"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:02.025722314Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1748,"client":"0","":"downstairs","session_id":"65996b6f-4393-4ede-81cb-df38c955c6ec"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:02.025758352Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1748,"client":"1","":"downstairs","session_id":"65996b6f-4393-4ede-81cb-df38c955c6ec"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:02.02579407Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1748,"client":"1","":"downstairs","session_id":"65996b6f-4393-4ede-81cb-df38c955c6ec"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:02.025828199Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1748,"client":"2","":"downstairs","session_id":"65996b6f-4393-4ede-81cb-df38c955c6ec"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:02.025863347Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1748,"client":"2","":"downstairs","session_id":"65996b6f-4393-4ede-81cb-df38c955c6ec"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:02.025898295Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1748,"":"io task","client":"2","":"downstairs","session_id":"65996b6f-4393-4ede-81cb-df38c955c6ec"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:02.026015809Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1748,"":"io task","client":"0","":"downstairs","session_id":"65996b6f-4393-4ede-81cb-df38c955c6ec"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:02.026060757Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1748,"":"io task","client":"1","":"downstairs","session_id":"65996b6f-4393-4ede-81cb-df38c955c6ec"} {"msg":"ds_connection connected from Ok(127.0.0.1:60140)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:02.026095596Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1748,"":"io task","client":"2","":"downstairs","session_id":"65996b6f-4393-4ede-81cb-df38c955c6ec"} {"msg":"ds_connection connected from Ok(127.0.0.1:64679)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:02.026128214Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1748,"":"io task","client":"1","":"downstairs","session_id":"65996b6f-4393-4ede-81cb-df38c955c6ec"} {"msg":"ds_connection connected from Ok(127.0.0.1:53932)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:02.026160033Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1748,"":"io task","client":"0","":"downstairs","session_id":"65996b6f-4393-4ede-81cb-df38c955c6ec"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:02.02660942Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1748,"client":"0","":"downstairs","session_id":"65996b6f-4393-4ede-81cb-df38c955c6ec"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:02.026659458Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1748,"client":"0","":"downstairs","session_id":"65996b6f-4393-4ede-81cb-df38c955c6ec"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:02.026693767Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1748,"client":"2","":"downstairs","session_id":"65996b6f-4393-4ede-81cb-df38c955c6ec"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:02.026742324Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1748,"client":"2","":"downstairs","session_id":"65996b6f-4393-4ede-81cb-df38c955c6ec"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:02.026916896Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1748,"client":"1","":"downstairs","session_id":"65996b6f-4393-4ede-81cb-df38c955c6ec"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:02.026937364Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1748,"client":"1","":"downstairs","session_id":"65996b6f-4393-4ede-81cb-df38c955c6ec"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:02.027413321Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1748,"client":"0","":"downstairs","session_id":"65996b6f-4393-4ede-81cb-df38c955c6ec"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:02.02744489Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1748,"client":"2","":"downstairs","session_id":"65996b6f-4393-4ede-81cb-df38c955c6ec"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:02.027534445Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1748,"client":"1","":"downstairs","session_id":"65996b6f-4393-4ede-81cb-df38c955c6ec"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:02.027962125Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1748,"client":"2","":"downstairs","session_id":"65996b6f-4393-4ede-81cb-df38c955c6ec"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:02.027987873Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1748,"client":"2","":"downstairs","session_id":"65996b6f-4393-4ede-81cb-df38c955c6ec"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:02.028003283Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1748,"session_id":"65996b6f-4393-4ede-81cb-df38c955c6ec"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:02.028087648Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1748,"client":"1","":"downstairs","session_id":"65996b6f-4393-4ede-81cb-df38c955c6ec"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:02.028123517Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1748,"client":"1","":"downstairs","session_id":"65996b6f-4393-4ede-81cb-df38c955c6ec"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:02.028147675Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1748,"session_id":"65996b6f-4393-4ede-81cb-df38c955c6ec"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:02.028165384Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1748,"client":"0","":"downstairs","session_id":"65996b6f-4393-4ede-81cb-df38c955c6ec"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:02.028179704Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1748,"client":"0","":"downstairs","session_id":"65996b6f-4393-4ede-81cb-df38c955c6ec"} {"msg":"[0]R flush_numbers[0..12]: [259, 256, 257, 258, 254, 259, 259, 249, 257, 257, 257, 259]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:02.028193793Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1748,"":"downstairs","":"downstairs","session_id":"65996b6f-4393-4ede-81cb-df38c955c6ec"} {"msg":"[0]R generation[0..12]: [141, 137, 138, 140, 136, 141, 141, 134, 138, 138, 138, 141]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:02.028208002Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1748,"":"downstairs","":"downstairs","session_id":"65996b6f-4393-4ede-81cb-df38c955c6ec"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:02.028222252Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1748,"":"downstairs","":"downstairs","session_id":"65996b6f-4393-4ede-81cb-df38c955c6ec"} {"msg":"[1]R flush_numbers[0..12]: [259, 256, 257, 258, 254, 259, 259, 249, 257, 257, 257, 259]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:02.028236321Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1748,"":"downstairs","":"downstairs","session_id":"65996b6f-4393-4ede-81cb-df38c955c6ec"} {"msg":"[1]R generation[0..12]: [141, 137, 138, 140, 136, 141, 141, 134, 138, 138, 138, 141]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:02.02825028Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1748,"":"downstairs","":"downstairs","session_id":"65996b6f-4393-4ede-81cb-df38c955c6ec"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:02.02826448Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1748,"":"downstairs","":"downstairs","session_id":"65996b6f-4393-4ede-81cb-df38c955c6ec"} {"msg":"[2]R flush_numbers[0..12]: [259, 256, 257, 258, 254, 259, 259, 249, 257, 257, 257, 259]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:02.028278049Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1748,"":"downstairs","":"downstairs","session_id":"65996b6f-4393-4ede-81cb-df38c955c6ec"} {"msg":"[2]R generation[0..12]: [141, 137, 138, 140, 136, 141, 141, 134, 138, 138, 138, 141]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:02.028291878Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1748,"":"downstairs","":"downstairs","session_id":"65996b6f-4393-4ede-81cb-df38c955c6ec"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:02.028311557Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1748,"":"downstairs","":"downstairs","session_id":"65996b6f-4393-4ede-81cb-df38c955c6ec"} {"msg":"Max found gen is 142","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-08T20:28:02.028335676Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1748,"":"downstairs","":"downstairs","session_id":"65996b6f-4393-4ede-81cb-df38c955c6ec"} {"msg":"Generation requested: 142 >= found:142","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:02.028361235Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1748,"":"downstairs","":"downstairs","session_id":"65996b6f-4393-4ede-81cb-df38c955c6ec"} {"msg":"Next flush: 260","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:02.028384964Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1748,"":"downstairs","":"downstairs","session_id":"65996b6f-4393-4ede-81cb-df38c955c6ec"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:02.028413082Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1748,"":"downstairs","":"downstairs","session_id":"65996b6f-4393-4ede-81cb-df38c955c6ec"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:02.028439581Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1748,"session_id":"65996b6f-4393-4ede-81cb-df38c955c6ec"} {"msg":"Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:02.02846588Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1748,"client":"0","":"downstairs","session_id":"65996b6f-4393-4ede-81cb-df38c955c6ec"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:02.028489439Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1748,"client":"1","":"downstairs","session_id":"65996b6f-4393-4ede-81cb-df38c955c6ec"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:02.028504168Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1748,"client":"2","":"downstairs","session_id":"65996b6f-4393-4ede-81cb-df38c955c6ec"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:02.028517787Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1748,"session_id":"65996b6f-4393-4ede-81cb-df38c955c6ec"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:02.028530187Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1748,"session_id":"65996b6f-4393-4ede-81cb-df38c955c6ec"} {"msg":"8910ae79-7931-4f36-a738-4bf06c5e4cf8 is now active with session: 65996b6f-4393-4ede-81cb-df38c955c6ec","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:02.028543246Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1748,"session_id":"65996b6f-4393-4ede-81cb-df38c955c6ec"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:02.028555176Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1748,"session_id":"65996b6f-4393-4ede-81cb-df38c955c6ec"} {"msg":"The guest has finished waiting for activation with:142","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:02.028567515Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1748} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 501 len 36864 data: 11 11 9 10 10 10 9 12 11 02/30 Read block 408 len 8192 03/30 Read block 14 len 40960 04/30 Read block 5 len 16384 05/30 Read block 315 len 40960 06/30 Read block 403 len 28672 07/30 Flush 08/30 Write block 587 len 28672 data: 15 12 10 8 7 9 8 09/30 Write block 259 len 40960 data: 9 7 7 8 9 10 13 12 9 12 10/30 Read block 270 len 40960 11/30 Read block 321 len 12288 12/30 Write block 294 len 36864 data: 13 12 9 11 11 15 14 15 14 13/30 Write block 584 len 32768 data: 16 19 15 16 13 11 9 8 14/30 Read block 380 len 16384 15/30 Read block 552 len 32768 16/30 Read block 321 len 32768 17/30 Write block 71 len 36864 data: 9 10 11 11 10 11 11 10 9 18/30 Read block 330 len 16384 19/30 Write block 361 len 28672 data: 19 17 15 14 11 11 14 20/30 Read block 20 len 8192 21/30 Read block 300 len 12288 22/30 Read block 47 len 12288 23/30 Read block 304 len 16384 24/30 Write block 316 len 32768 data: 14 10 12 14 14 10 11 11 25/30 Read block 340 len 8192 26/30 Write block 339 len 12288 data: 14 13 13 27/30 Read block 382 len 16384 28/30 Write block 554 len 32768 data: 8 9 9 9 9 10 9 9 29/30 Read block 331 len 24576 30/30 Write block 505 len 20480 data: 11 11 10 13 12 ---------------------------------------------------------------- Crucible gen:142 GIO:true work queues: Upstairs:0 downstairs:23 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1007 Acked Write 7 Done Done Done false 1008 Acked Write 10 Done Done Done false 1009 Acked Read 10 Done Done Done false 1010 Acked Read 3 Done Done Done false 1011 Acked Write 9 Done Done Done false 1012 Acked Write 8 Done Done Done false 1013 Acked Read 4 Done Done Done false 1014 Acked Read 8 Done Done Done false 1015 Acked Read 8 Done Done Done false 1016 Acked Write 9 Done Done Done false 1017 Acked Read 4 Done Done Done false 1018 Acked Write 7 Done Done Done false 1019 Acked Read 2 Done Done Done false 1020 Acked Read 3 Done Done Done false 1021 Acked Read 3 Done Done Done false 1022 Acked Read 4 Done Done Done false 1023 Acked Write 8 Done Done Done false 1024 Acked Read 2 Done Done Done false 1025 Acked Write 3 Done Done Done false 1026 Acked Read 4 Done Done Done false 1027 Acked Write 8 Done Done Done false 1028 Acked Read 6 Done Done Done false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 22 22 22 66 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1006 1006 1006 Downstairs last five completed: 1006 1005 1004 1003 1002 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:28:02.053950504Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1748} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 141 141 141 259 259 259 F F F 1 020-039 137 137 137 256 256 256 F F F 2 040-059 138 138 138 257 257 257 F F F 3 060-079 140 140 140 258 258 258 T T T 4 080-099 136 136 136 254 254 254 F F F 5 100-119 141 141 141 259 259 259 F F F 6 120-139 141 141 141 259 259 259 F F F 7 140-159 134 134 134 249 249 249 F F F 8 160-179 138 138 138 257 257 257 F F F 9 180-199 138 138 138 257 257 257 F F F 10 200-219 138 138 138 257 257 257 F F F 11 220-239 141 141 141 259 259 259 F F F 12 240-259 134 134 134 249 249 249 T T T 13 260-279 140 140 140 258 258 258 T T T 14 280-299 141 141 141 259 259 259 T T T 15 300-319 140 140 140 258 258 258 T T T 16 320-339 141 141 141 259 259 259 T T T 17 340-359 136 136 136 254 254 254 T T T 18 360-379 141 141 141 259 259 259 T T T 19 380-399 140 140 140 258 258 258 F F F 20 400-419 141 141 141 259 259 259 F F F 21 420-439 138 138 138 257 257 257 F F F 22 440-459 137 137 137 256 256 256 F F F 23 460-479 141 141 141 259 259 259 F F F 24 480-499 138 138 138 257 257 257 F F F 25 500-519 142 142 142 260 260 260 T T T 26 520-539 138 138 138 257 257 257 F F F 27 540-559 133 133 133 246 246 246 T T T 28 560-579 128 128 128 237 237 237 T T T 29 580-599 140 140 140 258 258 258 T T T Max gen: 142, Max flush: 260 A Difference in extent metadata is expected here On loop 71 Verifying data now No change after verify Loop: 71 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 141 141 141 259 259 259 F F F 1 020-039 137 137 137 256 256 256 F F F 2 040-059 138 138 138 257 257 257 F F F 3 060-079 143 143 143 261 261 261 F F F 4 080-099 136 136 136 254 254 254 F F F 5 100-119 141 141 141 259 259 259 F F F 6 120-139 141 141 141 259 259 259 F F F 7 140-159 134 134 134 249 249 249 F F F 8 160-179 138 138 138 257 257 257 F F F 9 180-199 138 138 138 257 257 257 F F F 10 200-219 138 138 138 257 257 257 F F F 11 220-239 141 141 141 259 259 259 F F F 12 240-259 143 143 143 261 261 261 F F F 13 260-279 143 143 143 261 261 261 F F F 14 280-299 143 143 143 261 261 261 F F F 15 300-319 143 143 143 261 261 261 F F F 16 320-339 143 143 143 261 261 261 F F F 17 340-359 143 143 143 261 261 261 F F F 18 360-379 143 143 143 261 261 261 F F F 19 380-399 140 140 140 258 258 258 F F F 20 400-419 141 141 141 259 259 259 F F F 21 420-439 138 138 138 257 257 257 F F F 22 440-459 137 137 137 256 256 256 F F F 23 460-479 141 141 141 259 259 259 F F F 24 480-499 138 138 138 257 257 257 F F F 25 500-519 143 143 143 261 261 261 F F F 26 520-539 138 138 138 257 257 257 F F F 27 540-559 143 143 143 261 261 261 F F F 28 560-579 143 143 143 261 261 261 F F F 29 580-599 143 143 143 261 261 261 F F F Max gen: 143, Max flush: 261 Begin loop 72 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:06.483217739Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1756,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:06.527394Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1756,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:06.52800397Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1756} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"0fd98d0d5d909611f8b408ccda2559d35fe117bb\",\n git_commit_timestamp: \"2024-11-08T20:05:44.000000000Z\",\n git_branch: \"mkeeter/drop-skipped-backpressure-guard\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:06.528040178Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1756} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:06.528054567Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1756} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:06.528068627Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1756} {"msg":"Crucible cb78f572-a59c-41f9-8079-ac347acd0bf1 has session id: 6171ad7b-be12-4641-b610-b98c42bd2d6b","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:06.528466437Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1756,"session_id":"6171ad7b-be12-4641-b610-b98c42bd2d6b"} {"msg":"Upstairs opts: Upstairs UUID: cb78f572-a59c-41f9-8079-ac347acd0bf1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:06.528512085Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1756,"session_id":"6171ad7b-be12-4641-b610-b98c42bd2d6b"} {"msg":"Crucible stats registered with UUID: cb78f572-a59c-41f9-8079-ac347acd0bf1","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:06.528542264Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1756,"session_id":"6171ad7b-be12-4641-b610-b98c42bd2d6b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:06.559527949Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1756,"task":"crutest"} {"msg":"The guest has requested activation with gen:144","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:06.559561008Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1756} {"msg":"cb78f572-a59c-41f9-8079-ac347acd0bf1 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:06.559611675Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1756,"session_id":"6171ad7b-be12-4641-b610-b98c42bd2d6b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:06.559631924Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1756,"client":"0","":"downstairs","session_id":"6171ad7b-be12-4641-b610-b98c42bd2d6b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:06.559659763Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1756,"client":"0","":"downstairs","session_id":"6171ad7b-be12-4641-b610-b98c42bd2d6b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:06.559677592Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1756,"client":"1","":"downstairs","session_id":"6171ad7b-be12-4641-b610-b98c42bd2d6b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:06.559694861Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1756,"client":"1","":"downstairs","session_id":"6171ad7b-be12-4641-b610-b98c42bd2d6b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:06.55971161Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1756,"client":"2","":"downstairs","session_id":"6171ad7b-be12-4641-b610-b98c42bd2d6b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:06.559728789Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1756,"client":"2","":"downstairs","session_id":"6171ad7b-be12-4641-b610-b98c42bd2d6b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:06.559745498Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1756,"":"io task","client":"2","":"downstairs","session_id":"6171ad7b-be12-4641-b610-b98c42bd2d6b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:06.559763167Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1756,"":"io task","client":"0","":"downstairs","session_id":"6171ad7b-be12-4641-b610-b98c42bd2d6b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:06.559780417Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1756,"":"io task","client":"1","":"downstairs","session_id":"6171ad7b-be12-4641-b610-b98c42bd2d6b"} {"msg":"ds_connection connected from Ok(127.0.0.1:64869)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:06.559865002Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1756,"":"io task","client":"0","":"downstairs","session_id":"6171ad7b-be12-4641-b610-b98c42bd2d6b"} {"msg":"ds_connection connected from Ok(127.0.0.1:45465)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:06.55991891Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1756,"":"io task","client":"1","":"downstairs","session_id":"6171ad7b-be12-4641-b610-b98c42bd2d6b"} {"msg":"ds_connection connected from Ok(127.0.0.1:43772)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:06.560096521Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1756,"":"io task","client":"2","":"downstairs","session_id":"6171ad7b-be12-4641-b610-b98c42bd2d6b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:06.56052892Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1756,"client":"2","":"downstairs","session_id":"6171ad7b-be12-4641-b610-b98c42bd2d6b"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:06.560558669Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1756,"client":"2","":"downstairs","session_id":"6171ad7b-be12-4641-b610-b98c42bd2d6b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:06.560677603Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1756,"client":"0","":"downstairs","session_id":"6171ad7b-be12-4641-b610-b98c42bd2d6b"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:06.560757179Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1756,"client":"0","":"downstairs","session_id":"6171ad7b-be12-4641-b610-b98c42bd2d6b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:06.560833645Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1756,"client":"1","":"downstairs","session_id":"6171ad7b-be12-4641-b610-b98c42bd2d6b"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:06.560872483Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1756,"client":"1","":"downstairs","session_id":"6171ad7b-be12-4641-b610-b98c42bd2d6b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:06.561292183Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1756,"client":"2","":"downstairs","session_id":"6171ad7b-be12-4641-b610-b98c42bd2d6b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:06.561323071Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1756,"client":"0","":"downstairs","session_id":"6171ad7b-be12-4641-b610-b98c42bd2d6b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:06.561599328Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1756,"client":"1","":"downstairs","session_id":"6171ad7b-be12-4641-b610-b98c42bd2d6b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:06.561880514Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1756,"client":"0","":"downstairs","session_id":"6171ad7b-be12-4641-b610-b98c42bd2d6b"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:06.561913993Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1756,"client":"0","":"downstairs","session_id":"6171ad7b-be12-4641-b610-b98c42bd2d6b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:06.561941601Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1756,"session_id":"6171ad7b-be12-4641-b610-b98c42bd2d6b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:06.56196547Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1756,"client":"2","":"downstairs","session_id":"6171ad7b-be12-4641-b610-b98c42bd2d6b"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:06.561990539Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1756,"client":"2","":"downstairs","session_id":"6171ad7b-be12-4641-b610-b98c42bd2d6b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:06.562005758Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1756,"session_id":"6171ad7b-be12-4641-b610-b98c42bd2d6b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:06.562031107Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1756,"client":"1","":"downstairs","session_id":"6171ad7b-be12-4641-b610-b98c42bd2d6b"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:06.562045846Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1756,"client":"1","":"downstairs","session_id":"6171ad7b-be12-4641-b610-b98c42bd2d6b"} {"msg":"[0]R flush_numbers[0..12]: [259, 256, 257, 261, 254, 259, 259, 249, 257, 257, 257, 259]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:06.562087744Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1756,"":"downstairs","":"downstairs","session_id":"6171ad7b-be12-4641-b610-b98c42bd2d6b"} {"msg":"[0]R generation[0..12]: [141, 137, 138, 143, 136, 141, 141, 134, 138, 138, 138, 141]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:06.562105583Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1756,"":"downstairs","":"downstairs","session_id":"6171ad7b-be12-4641-b610-b98c42bd2d6b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:06.562120202Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1756,"":"downstairs","":"downstairs","session_id":"6171ad7b-be12-4641-b610-b98c42bd2d6b"} {"msg":"[1]R flush_numbers[0..12]: [259, 256, 257, 261, 254, 259, 259, 249, 257, 257, 257, 259]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:06.562133912Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1756,"":"downstairs","":"downstairs","session_id":"6171ad7b-be12-4641-b610-b98c42bd2d6b"} {"msg":"[1]R generation[0..12]: [141, 137, 138, 143, 136, 141, 141, 134, 138, 138, 138, 141]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:06.562147691Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1756,"":"downstairs","":"downstairs","session_id":"6171ad7b-be12-4641-b610-b98c42bd2d6b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:06.56216153Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1756,"":"downstairs","":"downstairs","session_id":"6171ad7b-be12-4641-b610-b98c42bd2d6b"} {"msg":"[2]R flush_numbers[0..12]: [259, 256, 257, 261, 254, 259, 259, 249, 257, 257, 257, 259]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:06.56217521Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1756,"":"downstairs","":"downstairs","session_id":"6171ad7b-be12-4641-b610-b98c42bd2d6b"} {"msg":"[2]R generation[0..12]: [141, 137, 138, 143, 136, 141, 141, 134, 138, 138, 138, 141]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:06.562188939Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1756,"":"downstairs","":"downstairs","session_id":"6171ad7b-be12-4641-b610-b98c42bd2d6b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:06.562202968Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1756,"":"downstairs","":"downstairs","session_id":"6171ad7b-be12-4641-b610-b98c42bd2d6b"} {"msg":"Max found gen is 144","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:06.562216308Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1756,"":"downstairs","":"downstairs","session_id":"6171ad7b-be12-4641-b610-b98c42bd2d6b"} {"msg":"Generation requested: 144 >= found:144","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:06.562229537Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1756,"":"downstairs","":"downstairs","session_id":"6171ad7b-be12-4641-b610-b98c42bd2d6b"} {"msg":"Next flush: 262","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:06.562242656Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1756,"":"downstairs","":"downstairs","session_id":"6171ad7b-be12-4641-b610-b98c42bd2d6b"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-08T20:28:06.562257996Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1756,"":"downstairs","":"downstairs","session_id":"6171ad7b-be12-4641-b610-b98c42bd2d6b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:06.562277675Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1756,"session_id":"6171ad7b-be12-4641-b610-b98c42bd2d6b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:06.562289894Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1756,"client":"0","":"downstairs","session_id":"6171ad7b-be12-4641-b610-b98c42bd2d6b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:06.562303353Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1756,"client":"1","":"downstairs","session_id":"6171ad7b-be12-4641-b610-b98c42bd2d6b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:06.562316623Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1756,"client":"2","":"downstairs","session_id":"6171ad7b-be12-4641-b610-b98c42bd2d6b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:06.562329902Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1756,"session_id":"6171ad7b-be12-4641-b610-b98c42bd2d6b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:06.562342221Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1756,"session_id":"6171ad7b-be12-4641-b610-b98c42bd2d6b"} {"msg":"cb78f572-a59c-41f9-8079-ac347acd0bf1 is now active with session: 6171ad7b-be12-4641-b610-b98c42bd2d6b","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:06.562354941Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1756,"session_id":"6171ad7b-be12-4641-b610-b98c42bd2d6b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:06.56236692Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1756,"session_id":"6171ad7b-be12-4641-b610-b98c42bd2d6b"} {"msg":"The guest has finished waiting for activation with:144","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:06.56237931Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1756} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 233 len 40960 data: 9 7 9 10 9 12 11 12 11 13 02/30 Write block 142 len 16384 data: 7 9 10 11 03/30 Write block 319 len 20480 data: 15 15 11 12 12 04/30 Write block 349 len 4096 data: 13 05/30 Read block 169 len 20480 06/30 Read block 496 len 4096 07/30 Write block 180 len 32768 data: 11 10 11 12 9 9 13 12 08/30 Read block 528 len 24576 09/30 Read block 226 len 40960 10/30 Write block 200 len 36864 data: 9 8 8 7 9 10 11 11 9 11/30 Read block 238 len 4096 12/30 Read block 317 len 24576 13/30 Write block 341 len 36864 data: 14 15 13 11 12 14 12 10 14 14/30 Write block 97 len 20480 data: 5 4 5 7 8 15/30 Write block 12 len 40960 data: 15 13 11 13 14 13 10 9 8 10 16/30 Read block 95 len 8192 17/30 Write block 327 len 24576 data: 7 8 8 8 8 9 18/30 Flush 19/30 Write block 102 len 28672 data: 8 9 9 9 10 14 12 20/30 Read block 104 len 24576 21/30 Read block 347 len 24576 22/30 Write block 1 len 20480 data: 4 6 6 6 9 23/30 Write block 8 len 12288 data: 11 10 10 24/30 Write block 312 len 16384 data: 16 16 13 16 25/30 Read block 437 len 32768 26/30 Write block 457 len 8192 data: 9 7 27/30 Read block 532 len 16384 28/30 Write block 471 len 36864 data: 17 17 16 15 17 16 18 18 17 29/30 Read block 74 len 36864 30/30 Read block 535 len 4096 ---------------------------------------------------------------- Crucible gen:144 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1018 Acked Write 7 Done Done Done false 1019 Acked Read 6 Done Done Done false 1020 Acked Read 6 Done Done Done false 1021 Acked Write 5 Done Done Done false 1022 Acked Write 3 Done Done Done false 1023 Acked Write 4 Done Done Done false 1024 Acked Read 8 Done Done Done false 1025 Acked Write 2 Done Done Done false 1026 Acked Read 4 Done Done Done false 1027 Acked Write 9 Done Done Done false 1028 Acked Read 9 Done Done Done false 1029 Acked Read 1 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 12 11 11 34 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1017 1017 1017 Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:28:06.586595136Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1756} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 144 144 144 262 262 262 T T T 1 020-039 144 144 144 262 262 262 F F F 2 040-059 138 138 138 257 257 257 F F F 3 060-079 143 143 143 261 261 261 F F F 4 080-099 144 144 144 262 262 262 F F F 5 100-119 144 144 144 262 262 262 T T T 6 120-139 141 141 141 259 259 259 F F F 7 140-159 144 144 144 262 262 262 F F F 8 160-179 138 138 138 257 257 257 F F F 9 180-199 144 144 144 262 262 262 F F F 10 200-219 144 144 144 262 262 262 F F F 11 220-239 144 144 144 262 262 262 F F F 12 240-259 144 144 144 262 262 262 F F F 13 260-279 143 143 143 261 261 261 F F F 14 280-299 143 143 143 261 261 261 F F F 15 300-319 144 144 144 262 262 262 T T T 16 320-339 144 144 144 262 262 262 F F F 17 340-359 144 144 144 262 262 262 F F F 18 360-379 143 143 143 261 261 261 F F F 19 380-399 140 140 140 258 258 258 F F F 20 400-419 141 141 141 259 259 259 F F F 21 420-439 138 138 138 257 257 257 F F F 22 440-459 137 137 137 256 256 256 T T T 23 460-479 141 141 141 259 259 259 T T T 24 480-499 138 138 138 257 257 257 F F F 25 500-519 143 143 143 261 261 261 F F F 26 520-539 138 138 138 257 257 257 F F F 27 540-559 143 143 143 261 261 261 F F F 28 560-579 143 143 143 261 261 261 F F F 29 580-599 143 143 143 261 261 261 F F F Max gen: 144, Max flush: 262 A Difference in extent metadata is expected here On loop 72 Verifying data now No change after verify Loop: 72 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 145 145 145 263 263 263 F F F 1 020-039 144 144 144 262 262 262 F F F 2 040-059 138 138 138 257 257 257 F F F 3 060-079 143 143 143 261 261 261 F F F 4 080-099 144 144 144 262 262 262 F F F 5 100-119 145 145 145 263 263 263 F F F 6 120-139 141 141 141 259 259 259 F F F 7 140-159 144 144 144 262 262 262 F F F 8 160-179 138 138 138 257 257 257 F F F 9 180-199 144 144 144 262 262 262 F F F 10 200-219 144 144 144 262 262 262 F F F 11 220-239 144 144 144 262 262 262 F F F 12 240-259 144 144 144 262 262 262 F F F 13 260-279 143 143 143 261 261 261 F F F 14 280-299 143 143 143 261 261 261 F F F 15 300-319 145 145 145 263 263 263 F F F 16 320-339 144 144 144 262 262 262 F F F 17 340-359 144 144 144 262 262 262 F F F 18 360-379 143 143 143 261 261 261 F F F 19 380-399 140 140 140 258 258 258 F F F 20 400-419 141 141 141 259 259 259 F F F 21 420-439 138 138 138 257 257 257 F F F 22 440-459 145 145 145 263 263 263 F F F 23 460-479 145 145 145 263 263 263 F F F 24 480-499 138 138 138 257 257 257 F F F 25 500-519 143 143 143 261 261 261 F F F 26 520-539 138 138 138 257 257 257 F F F 27 540-559 143 143 143 261 261 261 F F F 28 560-579 143 143 143 261 261 261 F F F 29 580-599 143 143 143 261 261 261 F F F Max gen: 145, Max flush: 263 Begin loop 73 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:10.906523484Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1764,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:28:10.951696286Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1764,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:10.952101156Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1764,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:10.984956441Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1764,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:10.985520003Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1764} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"0fd98d0d5d909611f8b408ccda2559d35fe117bb\",\n git_commit_timestamp: \"2024-11-08T20:05:44.000000000Z\",\n git_branch: \"mkeeter/drop-skipped-backpressure-guard\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:10.985556621Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1764} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:10.98557179Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1764} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:10.98558617Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1764} {"msg":"Crucible 5760a9d2-9f52-4c60-b44f-d3faa409ad55 has session id: 3c6bd070-1492-4433-b8d6-063a3175ab31","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:10.986091945Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1764,"session_id":"3c6bd070-1492-4433-b8d6-063a3175ab31"} {"msg":"Upstairs opts: Upstairs UUID: 5760a9d2-9f52-4c60-b44f-d3faa409ad55, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:10.986137253Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1764,"session_id":"3c6bd070-1492-4433-b8d6-063a3175ab31"} {"msg":"Crucible stats registered with UUID: 5760a9d2-9f52-4c60-b44f-d3faa409ad55","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:10.986166691Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1764,"session_id":"3c6bd070-1492-4433-b8d6-063a3175ab31"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:11.018155808Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1764,"task":"crutest"} {"msg":"The guest has requested activation with gen:146","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:11.018190466Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1764} {"msg":"5760a9d2-9f52-4c60-b44f-d3faa409ad55 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:11.018208945Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1764,"session_id":"3c6bd070-1492-4433-b8d6-063a3175ab31"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:11.018225684Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1764,"client":"0","":"downstairs","session_id":"3c6bd070-1492-4433-b8d6-063a3175ab31"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:11.018251833Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1764,"client":"0","":"downstairs","session_id":"3c6bd070-1492-4433-b8d6-063a3175ab31"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:11.018280252Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1764,"client":"1","":"downstairs","session_id":"3c6bd070-1492-4433-b8d6-063a3175ab31"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:11.01831006Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1764,"client":"1","":"downstairs","session_id":"3c6bd070-1492-4433-b8d6-063a3175ab31"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:11.018338789Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1764,"client":"2","":"downstairs","session_id":"3c6bd070-1492-4433-b8d6-063a3175ab31"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:11.018367677Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1764,"client":"2","":"downstairs","session_id":"3c6bd070-1492-4433-b8d6-063a3175ab31"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:11.018394186Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1764,"":"io task","client":"2","":"downstairs","session_id":"3c6bd070-1492-4433-b8d6-063a3175ab31"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:11.018423575Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1764,"":"io task","client":"0","":"downstairs","session_id":"3c6bd070-1492-4433-b8d6-063a3175ab31"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:11.018453553Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1764,"":"io task","client":"1","":"downstairs","session_id":"3c6bd070-1492-4433-b8d6-063a3175ab31"} {"msg":"ds_connection connected from Ok(127.0.0.1:39431)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:11.018484372Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1764,"":"io task","client":"2","":"downstairs","session_id":"3c6bd070-1492-4433-b8d6-063a3175ab31"} {"msg":"ds_connection connected from Ok(127.0.0.1:46955)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:11.0185144Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1764,"":"io task","client":"0","":"downstairs","session_id":"3c6bd070-1492-4433-b8d6-063a3175ab31"} {"msg":"ds_connection connected from Ok(127.0.0.1:62825)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:11.018641544Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1764,"":"io task","client":"1","":"downstairs","session_id":"3c6bd070-1492-4433-b8d6-063a3175ab31"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:11.019230185Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1764,"client":"1","":"downstairs","session_id":"3c6bd070-1492-4433-b8d6-063a3175ab31"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:11.019259824Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1764,"client":"1","":"downstairs","session_id":"3c6bd070-1492-4433-b8d6-063a3175ab31"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:11.019281263Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1764,"client":"2","":"downstairs","session_id":"3c6bd070-1492-4433-b8d6-063a3175ab31"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:11.019296172Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1764,"client":"2","":"downstairs","session_id":"3c6bd070-1492-4433-b8d6-063a3175ab31"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:11.019440935Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1764,"client":"0","":"downstairs","session_id":"3c6bd070-1492-4433-b8d6-063a3175ab31"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:11.019476553Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1764,"client":"0","":"downstairs","session_id":"3c6bd070-1492-4433-b8d6-063a3175ab31"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:11.01994948Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1764,"client":"2","":"downstairs","session_id":"3c6bd070-1492-4433-b8d6-063a3175ab31"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:11.019980739Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1764,"client":"1","":"downstairs","session_id":"3c6bd070-1492-4433-b8d6-063a3175ab31"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:11.020139791Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1764,"client":"0","":"downstairs","session_id":"3c6bd070-1492-4433-b8d6-063a3175ab31"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:11.020523242Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1764,"client":"1","":"downstairs","session_id":"3c6bd070-1492-4433-b8d6-063a3175ab31"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:11.020555571Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1764,"client":"1","":"downstairs","session_id":"3c6bd070-1492-4433-b8d6-063a3175ab31"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:11.02057359Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1764,"session_id":"3c6bd070-1492-4433-b8d6-063a3175ab31"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:11.020589919Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1764,"client":"2","":"downstairs","session_id":"3c6bd070-1492-4433-b8d6-063a3175ab31"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:11.020606968Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1764,"client":"2","":"downstairs","session_id":"3c6bd070-1492-4433-b8d6-063a3175ab31"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:11.020623467Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1764,"session_id":"3c6bd070-1492-4433-b8d6-063a3175ab31"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:11.020706723Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1764,"client":"0","":"downstairs","session_id":"3c6bd070-1492-4433-b8d6-063a3175ab31"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:11.020731872Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1764,"client":"0","":"downstairs","session_id":"3c6bd070-1492-4433-b8d6-063a3175ab31"} {"msg":"[0]R flush_numbers[0..12]: [263, 262, 257, 261, 262, 263, 259, 262, 257, 262, 262, 262]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:11.020749761Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1764,"":"downstairs","":"downstairs","session_id":"3c6bd070-1492-4433-b8d6-063a3175ab31"} {"msg":"[0]R generation[0..12]: [145, 144, 138, 143, 144, 145, 141, 144, 138, 144, 144, 144]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:11.02076725Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1764,"":"downstairs","":"downstairs","session_id":"3c6bd070-1492-4433-b8d6-063a3175ab31"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:11.020784939Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1764,"":"downstairs","":"downstairs","session_id":"3c6bd070-1492-4433-b8d6-063a3175ab31"} {"msg":"[1]R flush_numbers[0..12]: [263, 262, 257, 261, 262, 263, 259, 262, 257, 262, 262, 262]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:11.020802498Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1764,"":"downstairs","":"downstairs","session_id":"3c6bd070-1492-4433-b8d6-063a3175ab31"} {"msg":"[1]R generation[0..12]: [145, 144, 138, 143, 144, 145, 141, 144, 138, 144, 144, 144]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:11.020819858Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1764,"":"downstairs","":"downstairs","session_id":"3c6bd070-1492-4433-b8d6-063a3175ab31"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:11.020837307Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1764,"":"downstairs","":"downstairs","session_id":"3c6bd070-1492-4433-b8d6-063a3175ab31"} {"msg":"[2]R flush_numbers[0..12]: [263, 262, 257, 261, 262, 263, 259, 262, 257, 262, 262, 262]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:11.020854376Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1764,"":"downstairs","":"downstairs","session_id":"3c6bd070-1492-4433-b8d6-063a3175ab31"} {"msg":"[2]R generation[0..12]: [145, 144, 138, 143, 144, 145, 141, 144, 138, 144, 144, 144]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:11.020871385Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1764,"":"downstairs","":"downstairs","session_id":"3c6bd070-1492-4433-b8d6-063a3175ab31"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:11.020888544Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1764,"":"downstairs","":"downstairs","session_id":"3c6bd070-1492-4433-b8d6-063a3175ab31"} {"msg":"Max found gen is 146","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:11.020904484Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1764,"":"downstairs","":"downstairs","session_id":"3c6bd070-1492-4433-b8d6-063a3175ab31"} {"msg":"Generation requested: 146 >= found:146","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-08T20:28:11.020920793Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1764,"":"downstairs","":"downstairs","session_id":"3c6bd070-1492-4433-b8d6-063a3175ab31"} {"msg":"Next flush: 264","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:11.020942922Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1764,"":"downstairs","":"downstairs","session_id":"3c6bd070-1492-4433-b8d6-063a3175ab31"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:11.020959541Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1764,"":"downstairs","":"downstairs","session_id":"3c6bd070-1492-4433-b8d6-063a3175ab31"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:11.02097593Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1764,"session_id":"3c6bd070-1492-4433-b8d6-063a3175ab31"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:11.020991169Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1764,"client":"0","":"downstairs","session_id":"3c6bd070-1492-4433-b8d6-063a3175ab31"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:11.021007619Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1764,"client":"1","":"downstairs","session_id":"3c6bd070-1492-4433-b8d6-063a3175ab31"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:11.021024018Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1764,"client":"2","":"downstairs","session_id":"3c6bd070-1492-4433-b8d6-063a3175ab31"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:11.021040367Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1764,"session_id":"3c6bd070-1492-4433-b8d6-063a3175ab31"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:11.021055796Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1764,"session_id":"3c6bd070-1492-4433-b8d6-063a3175ab31"} {"msg":"5760a9d2-9f52-4c60-b44f-d3faa409ad55 is now active with session: 3c6bd070-1492-4433-b8d6-063a3175ab31","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:11.021071815Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1764,"session_id":"3c6bd070-1492-4433-b8d6-063a3175ab31"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:11.021086935Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1764,"session_id":"3c6bd070-1492-4433-b8d6-063a3175ab31"} {"msg":"The guest has finished waiting for activation with:146","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:11.021102134Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1764} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 502 len 28672 data: 12 10 11 12 12 11 14 02/30 Write block 142 len 8192 data: 8 10 03/30 Read block 258 len 12288 04/30 Read block 426 len 8192 05/30 Write block 295 len 8192 data: 13 10 06/30 Write block 141 len 36864 data: 9 9 11 11 12 11 12 11 13 07/30 Read block 148 len 8192 08/30 Write block 95 len 24576 data: 8 6 6 5 6 8 09/30 Write block 212 len 40960 data: 10 11 10 11 12 11 13 12 11 14 10/30 Read block 351 len 36864 11/30 Read block 333 len 20480 12/30 Read block 551 len 8192 13/30 Read block 425 len 4096 14/30 Flush 15/30 Write block 495 len 28672 data: 9 9 9 10 9 9 12 16/30 Write block 587 len 32768 data: 17 14 12 10 9 10 9 7 17/30 Read block 444 len 20480 18/30 Read block 151 len 8192 19/30 Read block 510 len 16384 20/30 Write block 278 len 32768 data: 12 12 12 8 10 12 12 14 21/30 Write block 342 len 40960 data: 16 14 12 13 15 13 11 15 13 14 22/30 Read block 207 len 20480 23/30 Write block 218 len 16384 data: 14 13 12 15 24/30 Read block 249 len 12288 25/30 Read block 45 len 32768 26/30 Write block 57 len 40960 data: 11 9 6 8 9 9 10 10 10 9 27/30 Write block 224 len 24576 data: 12 11 13 15 11 11 28/30 Read block 279 len 28672 29/30 Read block 294 len 4096 30/30 Write block 121 len 20480 data: 12 12 13 14 10 ---------------------------------------------------------------- Crucible gen:146 GIO:true work queues: Upstairs:0 downstairs:16 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1014 Acked Write 7 Done Done Done false 1015 Acked Write 8 Done Done Done false 1016 Acked Read 5 Done Done Done false 1017 Acked Read 2 Done Done Done false 1018 Acked Read 4 Done Done Done false 1019 Acked Write 8 Done Done Done false 1020 Acked Write 10 Done Done Done false 1021 Acked Read 5 Done Done Done false 1022 Acked Write 4 Done Done Done false 1023 Acked Read 3 Done Done Done false 1024 Acked Read 8 Done Done Done false 1025 Acked Write 10 Done Done Done false 1026 Acked Write 6 Done Done Done false 1027 Acked Read 7 Done Done Done false 1028 Acked Read 1 Done Done Done false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 15 15 15 45 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1013 1013 1013 Downstairs last five completed: 1013 1012 1011 1010 1009 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:28:11.044404165Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1764} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 145 145 145 263 263 263 F F F 1 020-039 144 144 144 262 262 262 F F F 2 040-059 138 138 138 257 257 257 T T T 3 060-079 143 143 143 261 261 261 T T T 4 080-099 146 146 146 264 264 264 F F F 5 100-119 146 146 146 264 264 264 F F F 6 120-139 141 141 141 259 259 259 T T T 7 140-159 146 146 146 264 264 264 F F F 8 160-179 138 138 138 257 257 257 F F F 9 180-199 144 144 144 262 262 262 F F F 10 200-219 146 146 146 264 264 264 T T T 11 220-239 146 146 146 264 264 264 T T T 12 240-259 144 144 144 262 262 262 F F F 13 260-279 143 143 143 261 261 261 T T T 14 280-299 146 146 146 264 264 264 T T T 15 300-319 145 145 145 263 263 263 F F F 16 320-339 144 144 144 262 262 262 F F F 17 340-359 144 144 144 262 262 262 T T T 18 360-379 143 143 143 261 261 261 F F F 19 380-399 140 140 140 258 258 258 F F F 20 400-419 141 141 141 259 259 259 F F F 21 420-439 138 138 138 257 257 257 F F F 22 440-459 145 145 145 263 263 263 F F F 23 460-479 145 145 145 263 263 263 F F F 24 480-499 138 138 138 257 257 257 T T T 25 500-519 146 146 146 264 264 264 T T T 26 520-539 138 138 138 257 257 257 F F F 27 540-559 143 143 143 261 261 261 F F F 28 560-579 143 143 143 261 261 261 F F F 29 580-599 143 143 143 261 261 261 T T T Max gen: 146, Max flush: 264 A Difference in extent metadata is expected here On loop 73 Verifying data now No change after verify Loop: 73 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 145 145 145 263 263 263 F F F 1 020-039 144 144 144 262 262 262 F F F 2 040-059 147 147 147 265 265 265 F F F 3 060-079 147 147 147 265 265 265 F F F 4 080-099 146 146 146 264 264 264 F F F 5 100-119 146 146 146 264 264 264 F F F 6 120-139 147 147 147 265 265 265 F F F 7 140-159 146 146 146 264 264 264 F F F 8 160-179 138 138 138 257 257 257 F F F 9 180-199 144 144 144 262 262 262 F F F 10 200-219 147 147 147 265 265 265 F F F 11 220-239 147 147 147 265 265 265 F F F 12 240-259 144 144 144 262 262 262 F F F 13 260-279 147 147 147 265 265 265 F F F 14 280-299 147 147 147 265 265 265 F F F 15 300-319 145 145 145 263 263 263 F F F 16 320-339 144 144 144 262 262 262 F F F 17 340-359 147 147 147 265 265 265 F F F 18 360-379 143 143 143 261 261 261 F F F 19 380-399 140 140 140 258 258 258 F F F 20 400-419 141 141 141 259 259 259 F F F 21 420-439 138 138 138 257 257 257 F F F 22 440-459 145 145 145 263 263 263 F F F 23 460-479 145 145 145 263 263 263 F F F 24 480-499 147 147 147 265 265 265 F F F 25 500-519 147 147 147 265 265 265 F F F 26 520-539 138 138 138 257 257 257 F F F 27 540-559 143 143 143 261 261 261 F F F 28 560-579 143 143 143 261 261 261 F F F 29 580-599 147 147 147 265 265 265 F F F Max gen: 147, Max flush: 265 Begin loop 74 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:15.48448373Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1772,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:15.530634075Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1772,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:15.531284193Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1772} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"0fd98d0d5d909611f8b408ccda2559d35fe117bb\",\n git_commit_timestamp: \"2024-11-08T20:05:44.000000000Z\",\n git_branch: \"mkeeter/drop-skipped-backpressure-guard\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:15.531321011Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1772} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:15.5313354Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1772} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:15.53135012Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1772} {"msg":"Crucible f08ab346-1a17-4a30-9ea5-d2fee7957d56 has session id: 639479a5-c55d-4d33-bbf4-6aa90e9a228d","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:15.531789898Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1772,"session_id":"639479a5-c55d-4d33-bbf4-6aa90e9a228d"} {"msg":"Upstairs opts: Upstairs UUID: f08ab346-1a17-4a30-9ea5-d2fee7957d56, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:15.531847435Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1772,"session_id":"639479a5-c55d-4d33-bbf4-6aa90e9a228d"} {"msg":"Crucible stats registered with UUID: f08ab346-1a17-4a30-9ea5-d2fee7957d56","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:15.531878374Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1772,"session_id":"639479a5-c55d-4d33-bbf4-6aa90e9a228d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:15.565017204Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1772,"task":"crutest"} {"msg":"The guest has requested activation with gen:148","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:15.565052802Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1772} {"msg":"f08ab346-1a17-4a30-9ea5-d2fee7957d56 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:15.565070681Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1772,"session_id":"639479a5-c55d-4d33-bbf4-6aa90e9a228d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:15.565086851Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1772,"client":"0","":"downstairs","session_id":"639479a5-c55d-4d33-bbf4-6aa90e9a228d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:15.565116879Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1772,"client":"0","":"downstairs","session_id":"639479a5-c55d-4d33-bbf4-6aa90e9a228d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:15.565135469Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1772,"client":"1","":"downstairs","session_id":"639479a5-c55d-4d33-bbf4-6aa90e9a228d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:15.565152777Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1772,"client":"1","":"downstairs","session_id":"639479a5-c55d-4d33-bbf4-6aa90e9a228d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:15.565179406Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1772,"client":"2","":"downstairs","session_id":"639479a5-c55d-4d33-bbf4-6aa90e9a228d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:15.565280291Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1772,"client":"2","":"downstairs","session_id":"639479a5-c55d-4d33-bbf4-6aa90e9a228d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:15.565326369Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1772,"":"io task","client":"2","":"downstairs","session_id":"639479a5-c55d-4d33-bbf4-6aa90e9a228d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:15.565357907Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1772,"":"io task","client":"0","":"downstairs","session_id":"639479a5-c55d-4d33-bbf4-6aa90e9a228d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:15.565386036Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1772,"":"io task","client":"1","":"downstairs","session_id":"639479a5-c55d-4d33-bbf4-6aa90e9a228d"} {"msg":"ds_connection connected from Ok(127.0.0.1:34493)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:15.565416915Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1772,"":"io task","client":"2","":"downstairs","session_id":"639479a5-c55d-4d33-bbf4-6aa90e9a228d"} {"msg":"ds_connection connected from Ok(127.0.0.1:63651)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:15.56551118Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1772,"":"io task","client":"0","":"downstairs","session_id":"639479a5-c55d-4d33-bbf4-6aa90e9a228d"} {"msg":"ds_connection connected from Ok(127.0.0.1:47875)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:15.565561018Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1772,"":"io task","client":"1","":"downstairs","session_id":"639479a5-c55d-4d33-bbf4-6aa90e9a228d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:15.566250294Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1772,"client":"0","":"downstairs","session_id":"639479a5-c55d-4d33-bbf4-6aa90e9a228d"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:15.566291342Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1772,"client":"0","":"downstairs","session_id":"639479a5-c55d-4d33-bbf4-6aa90e9a228d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:15.566321331Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1772,"client":"2","":"downstairs","session_id":"639479a5-c55d-4d33-bbf4-6aa90e9a228d"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:15.566347829Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1772,"client":"2","":"downstairs","session_id":"639479a5-c55d-4d33-bbf4-6aa90e9a228d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:15.566410826Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1772,"client":"1","":"downstairs","session_id":"639479a5-c55d-4d33-bbf4-6aa90e9a228d"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:15.566448754Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1772,"client":"1","":"downstairs","session_id":"639479a5-c55d-4d33-bbf4-6aa90e9a228d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:15.566907642Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1772,"client":"2","":"downstairs","session_id":"639479a5-c55d-4d33-bbf4-6aa90e9a228d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:15.5669385Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1772,"client":"0","":"downstairs","session_id":"639479a5-c55d-4d33-bbf4-6aa90e9a228d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:15.567086913Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1772,"client":"1","":"downstairs","session_id":"639479a5-c55d-4d33-bbf4-6aa90e9a228d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:15.567533681Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1772,"client":"0","":"downstairs","session_id":"639479a5-c55d-4d33-bbf4-6aa90e9a228d"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:15.56756288Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1772,"client":"0","":"downstairs","session_id":"639479a5-c55d-4d33-bbf4-6aa90e9a228d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:15.567578229Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1772,"session_id":"639479a5-c55d-4d33-bbf4-6aa90e9a228d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:15.567591049Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1772,"client":"2","":"downstairs","session_id":"639479a5-c55d-4d33-bbf4-6aa90e9a228d"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:15.567604758Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1772,"client":"2","":"downstairs","session_id":"639479a5-c55d-4d33-bbf4-6aa90e9a228d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:15.567617987Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1772,"session_id":"639479a5-c55d-4d33-bbf4-6aa90e9a228d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:15.567630007Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1772,"client":"1","":"downstairs","session_id":"639479a5-c55d-4d33-bbf4-6aa90e9a228d"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:15.567647046Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1772,"client":"1","":"downstairs","session_id":"639479a5-c55d-4d33-bbf4-6aa90e9a228d"} {"msg":"[0]R flush_numbers[0..12]: [263, 262, 265, 265, 264, 264, 265, 264, 257, 262, 265, 265]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:15.567667485Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1772,"":"downstairs","":"downstairs","session_id":"639479a5-c55d-4d33-bbf4-6aa90e9a228d"} {"msg":"[0]R generation[0..12]: [145, 144, 147, 147, 146, 146, 147, 146, 138, 144, 147, 147]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:15.567681714Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1772,"":"downstairs","":"downstairs","session_id":"639479a5-c55d-4d33-bbf4-6aa90e9a228d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:15.567695793Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1772,"":"downstairs","":"downstairs","session_id":"639479a5-c55d-4d33-bbf4-6aa90e9a228d"} {"msg":"[1]R flush_numbers[0..12]: [263, 262, 265, 265, 264, 264, 265, 264, 257, 262, 265, 265]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:15.567709713Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1772,"":"downstairs","":"downstairs","session_id":"639479a5-c55d-4d33-bbf4-6aa90e9a228d"} {"msg":"[1]R generation[0..12]: [145, 144, 147, 147, 146, 146, 147, 146, 138, 144, 147, 147]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:15.567723352Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1772,"":"downstairs","":"downstairs","session_id":"639479a5-c55d-4d33-bbf4-6aa90e9a228d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:15.567737311Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1772,"":"downstairs","":"downstairs","session_id":"639479a5-c55d-4d33-bbf4-6aa90e9a228d"} {"msg":"[2]R flush_numbers[0..12]: [263, 262, 265, 265, 264, 264, 265, 264, 257, 262, 265, 265]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:15.567751041Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1772,"":"downstairs","":"downstairs","session_id":"639479a5-c55d-4d33-bbf4-6aa90e9a228d"} {"msg":"[2]R generation[0..12]: [145, 144, 147, 147, 146, 146, 147, 146, 138, 144, 147, 147]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:15.56776465Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1772,"":"downstairs","":"downstairs","session_id":"639479a5-c55d-4d33-bbf4-6aa90e9a228d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:15.567778539Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1772,"":"downstairs","":"downstairs","session_id":"639479a5-c55d-4d33-bbf4-6aa90e9a228d"} {"msg":"Max found gen is 148","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:15.567791439Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1772,"":"downstairs","":"downstairs","session_id":"639479a5-c55d-4d33-bbf4-6aa90e9a228d"} {"msg":"Generation requested: 148 >= found:148","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:15.567804658Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1772,"":"downstairs","":"downstairs","session_id":"639479a5-c55d-4d33-bbf4-6aa90e9a228d"} {"msg":"Next flush: 266","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:15.567818057Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1772,"":"downstairs","":"downstairs","session_id":"639479a5-c55d-4d33-bbf4-6aa90e9a228d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:15.567830987Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1772,"":"downstairs","":"downstairs","session_id":"639479a5-c55d-4d33-bbf4-6aa90e9a228d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:15.567845896Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1772,"session_id":"639479a5-c55d-4d33-bbf4-6aa90e9a228d"} {"msg":"Transition from WaitQuorum to Active",Wait for a query_work_queue command to finish before sending IO "v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:15.567866445Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1772,"client":"0","":"downstairs","session_id":"639479a5-c55d-4d33-bbf4-6aa90e9a228d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:15.567880314Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1772,"client":"1","":"downstairs","session_id":"639479a5-c55d-4d33-bbf4-6aa90e9a228d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:15.567893824Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1772,"client":"2","":"downstairs","session_id":"639479a5-c55d-4d33-bbf4-6aa90e9a228d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:15.567907433Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1772,"session_id":"639479a5-c55d-4d33-bbf4-6aa90e9a228d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:15.567919792Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1772,"session_id":"639479a5-c55d-4d33-bbf4-6aa90e9a228d"} {"msg":"f08ab346-1a17-4a30-9ea5-d2fee7957d56 is now active with session: 639479a5-c55d-4d33-bbf4-6aa90e9a228d","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:15.567932532Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1772,"session_id":"639479a5-c55d-4d33-bbf4-6aa90e9a228d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:15.567944611Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1772,"session_id":"639479a5-c55d-4d33-bbf4-6aa90e9a228d"} {"msg":"The guest has finished waiting for activation with:148","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:15.567956751Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1772} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 94 len 32768 data: 7 9 7 7 6 7 9 9 02/30 Read block 473 len 4096 03/30 Write block 337 len 36864 data: 13 15 15 14 15 17 15 13 14 04/30 Write block 535 len 8192 data: 12 12 05/30 Read block 521 len 12288 06/30 Write block 591 len 24576 data: 10 11 10 8 9 7 07/30 Write block 462 len 40960 data: 9 9 6 8 11 13 15 14 15 18 08/30 Read block 315 len 40960 09/30 Read block 287 len 40960 10/30 Read block 141 len 8192 11/30 Read block 210 len 16384 12/30 Read block 146 len 40960 13/30 Read block 334 len 4096 14/30 Read block 478 len 8192 15/30 Write block 430 len 36864 data: 16 15 13 12 11 14 16 15 15 16/30 Read block 490 len 32768 17/30 Write block 174 len 8192 data: 10 11 18/30 Flush 19/30 Write block 510 len 24576 data: 12 13 11 11 11 8 20/30 Read block 4 len 8192 21/30 Read block 454 len 24576 22/30 Read block 484 len 28672 23/30 Write block 4 len 20480 data: 7 10 10 10 12 24/30 Read block 198 len 40960 25/30 Write block 350 len 32768 data: 14 15 14 18 16 17 16 15 26/30 Read block 130 len 36864 27/30 Write block 105 len 8192 data: 10 11 28/30 Write block 172 len 8192 data: 8 8 29/30 Write block 193 len 16384 data: 11 11 10 11 30/30 Write block 34 len 28672 data: 10 9 9 10 12 13 14 ---------------------------------------------------------------- Crucible gen:148 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1018 Acked Write 6 Done Done Done false 1019 Acked Read 2 Done Done Done false 1020 Acked Read 6 Done Done Done false 1021 Acked Read 7 Done Done Done false 1022 Acked Write 5 Done Done Done false 1023 Acked Read 10 Done Done Done false 1024 Acked Write 8 Done Done Done false 1025 Acked Read 9 Done Done Done false 1026 Acked Write 2 Done Done Done false 1027 Acked Write 2 Done Done Done false 1028 Acked Write 4 Done Sent Done false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 2 1 4 Done 11 10 11 32 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1017 1017 1017 Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:28:15.593234195Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1772} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 145 145 145 263 263 263 T T T 1 020-039 144 144 144 262 262 262 T T T 2 040-059 147 147 147 265 265 265 T T T 3 060-079 147 147 147 265 265 265 F F F 4 080-099 148 148 148 266 266 266 F F F 5 100-119 148 148 148 266 266 266 T T T 6 120-139 147 147 147 265 265 265 F F F 7 140-159 146 146 146 264 264 264 F F F 8 160-179 148 148 148 266 266 266 T T T 9 180-199 144 144 144 262 262 262 T T T 10 200-219 147 147 147 265 265 265 F F F 11 220-239 147 147 147 265 265 265 F F F 12 240-259 144 144 144 262 262 262 F F F 13 260-279 147 147 147 265 265 265 F F F 14 280-299 147 147 147 265 265 265 F F F 15 300-319 145 145 145 263 263 263 F F F 16 320-339 148 148 148 266 266 266 F F F 17 340-359 148 148 148 266 266 266 T T T 18 360-379 143 143 143 261 261 261 F F F 19 380-399 140 140 140 258 258 258 F F F 20 400-419 141 141 141 259 259 259 F F F 21 420-439 148 148 148 266 266 266 F F F 22 440-459 145 145 145 263 263 263 F F F 23 460-479 148 148 148 266 266 266 F F F 24 480-499 147 147 147 265 265 265 F F F 25 500-519 147 147 147 265 265 265 T T T 26 520-539 148 148 148 266 266 266 F F F 27 540-559 143 143 143 261 261 261 F F F 28 560-579 143 143 143 261 261 261 F F F 29 580-599 148 148 148 266 266 266 F F F Max gen: 148, Max flush: 266 A Difference in extent metadata is expected here On loop 74 Verifying data now No change after verify Loop: 74 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 149 149 149 267 267 267 F F F 1 020-039 149 149 149 267 267 267 F F F 2 040-059 149 149 149 267 267 267 F F F 3 060-079 147 147 147 265 265 265 F F F 4 080-099 148 148 148 266 266 266 F F F 5 100-119 149 149 149 267 267 267 F F F 6 120-139 147 147 147 265 265 265 F F F 7 140-159 146 146 146 264 264 264 F F F 8 160-179 149 149 149 267 267 267 F F F 9 180-199 149 149 149 267 267 267 F F F 10 200-219 147 147 147 265 265 265 F F F 11 220-239 147 147 147 265 265 265 F F F 12 240-259 144 144 144 262 262 262 F F F 13 260-279 147 147 147 265 265 265 F F F 14 280-299 147 147 147 265 265 265 F F F 15 300-319 145 145 145 263 263 263 F F F 16 320-339 148 148 148 266 266 266 F F F 17 340-359 149 149 149 267 267 267 F F F 18 360-379 143 143 143 261 261 261 F F F 19 380-399 140 140 140 258 258 258 F F F 20 400-419 141 141 141 259 259 259 F F F 21 420-439 148 148 148 266 266 266 F F F 22 440-459 145 145 145 263 263 263 F F F 23 460-479 148 148 148 266 266 266 F F F 24 480-499 147 147 147 265 265 265 F F F 25 500-519 149 149 149 267 267 267 F F F 26 520-539 148 148 148 266 266 266 F F F 27 540-559 143 143 143 261 261 261 F F F 28 560-579 143 143 143 261 261 261 F F F 29 580-599 148 148 148 266 266 266 F F F Max gen: 149, Max flush: 267 Begin loop 75 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:19.962681101Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1780,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:20.007437303Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1780,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:20.008109321Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1780} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"0fd98d0d5d909611f8b408ccda2559d35fe117bb\",\n git_commit_timestamp: \"2024-11-08T20:05:44.000000000Z\",\n git_branch: \"mkeeter/drop-skipped-backpressure-guard\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:20.008155498Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1780} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:20.008173538Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1780} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:20.008191457Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1780} {"msg":"Crucible f3d65ba3-5fa0-4e9e-95c6-65763a224d59 has session id: 6f0f2c77-6203-499a-968a-a4ecfbf36dce","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:20.008695892Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1780,"session_id":"6f0f2c77-6203-499a-968a-a4ecfbf36dce"} {"msg":"Upstairs opts: Upstairs UUID: f3d65ba3-5fa0-4e9e-95c6-65763a224d59, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:20.008749759Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1780,"session_id":"6f0f2c77-6203-499a-968a-a4ecfbf36dce"} {"msg":"Crucible stats registered with UUID: f3d65ba3-5fa0-4e9e-95c6-65763a224d59","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:20.008771218Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1780,"session_id":"6f0f2c77-6203-499a-968a-a4ecfbf36dce"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:20.039984333Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1780,"task":"crutest"} {"msg":"The guest has requested activation with gen:150","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:20.04003165Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1780} {"msg":"f3d65ba3-5fa0-4e9e-95c6-65763a224d59 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:20.040054269Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1780,"session_id":"6f0f2c77-6203-499a-968a-a4ecfbf36dce"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:20.040071439Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1780,"client":"0","":"downstairs","session_id":"6f0f2c77-6203-499a-968a-a4ecfbf36dce"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:20.040099747Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1780,"client":"0","":"downstairs","session_id":"6f0f2c77-6203-499a-968a-a4ecfbf36dce"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:20.040117316Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1780,"client":"1","":"downstairs","session_id":"6f0f2c77-6203-499a-968a-a4ecfbf36dce"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:20.040134545Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1780,"client":"1","":"downstairs","session_id":"6f0f2c77-6203-499a-968a-a4ecfbf36dce"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:20.040151255Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1780,"client":"2","":"downstairs","session_id":"6f0f2c77-6203-499a-968a-a4ecfbf36dce"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:20.040168134Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1780,"client":"2","":"downstairs","session_id":"6f0f2c77-6203-499a-968a-a4ecfbf36dce"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:20.040184593Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1780,"":"io task","client":"2","":"downstairs","session_id":"6f0f2c77-6203-499a-968a-a4ecfbf36dce"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:20.040201892Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1780,"":"io task","client":"0","":"downstairs","session_id":"6f0f2c77-6203-499a-968a-a4ecfbf36dce"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:20.040219141Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1780,"":"io task","client":"1","":"downstairs","session_id":"6f0f2c77-6203-499a-968a-a4ecfbf36dce"} {"msg":"ds_connection connected from Ok(127.0.0.1:51201)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:20.040332926Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1780,"":"io task","client":"1","":"downstairs","session_id":"6f0f2c77-6203-499a-968a-a4ecfbf36dce"} {"msg":"ds_connection connected from Ok(127.0.0.1:35024)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:20.040364894Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1780,"":"io task","client":"2","":"downstairs","session_id":"6f0f2c77-6203-499a-968a-a4ecfbf36dce"} {"msg":"ds_connection connected from Ok(127.0.0.1:53605)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:20.040389613Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1780,"":"io task","client":"0","":"downstairs","session_id":"6f0f2c77-6203-499a-968a-a4ecfbf36dce"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:20.041117557Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1780,"client":"0","":"downstairs","session_id":"6f0f2c77-6203-499a-968a-a4ecfbf36dce"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:20.041145646Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1780,"client":"0","":"downstairs","session_id":"6f0f2c77-6203-499a-968a-a4ecfbf36dce"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:20.041293389Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1780,"client":"1","":"downstairs","session_id":"6f0f2c77-6203-499a-968a-a4ecfbf36dce"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:20.041339926Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1780,"client":"1","":"downstairs","session_id":"6f0f2c77-6203-499a-968a-a4ecfbf36dce"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:20.041371305Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1780,"client":"2","":"downstairs","session_id":"6f0f2c77-6203-499a-968a-a4ecfbf36dce"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:20.041399354Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1780,"client":"2","":"downstairs","session_id":"6f0f2c77-6203-499a-968a-a4ecfbf36dce"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:20.041777745Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1780,"client":"0","":"downstairs","session_id":"6f0f2c77-6203-499a-968a-a4ecfbf36dce"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:20.041807694Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1780,"client":"2","":"downstairs","session_id":"6f0f2c77-6203-499a-968a-a4ecfbf36dce"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:20.041942327Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1780,"client":"1","":"downstairs","session_id":"6f0f2c77-6203-499a-968a-a4ecfbf36dce"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:20.042404174Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1780,"client":"0","":"downstairs","session_id":"6f0f2c77-6203-499a-968a-a4ecfbf36dce"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:20.042463032Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1780,"client":"0","":"downstairs","session_id":"6f0f2c77-6203-499a-968a-a4ecfbf36dce"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:20.04249097Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1780,"session_id":"6f0f2c77-6203-499a-968a-a4ecfbf36dce"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:20.042512959Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1780,"client":"2","":"downstairs","session_id":"6f0f2c77-6203-499a-968a-a4ecfbf36dce"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:20.042536958Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1780,"client":"2","":"downstairs","session_id":"6f0f2c77-6203-499a-968a-a4ecfbf36dce"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:20.042561457Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1780,"session_id":"6f0f2c77-6203-499a-968a-a4ecfbf36dce"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:20.042583836Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1780,"client":"1","":"downstairs","session_id":"6f0f2c77-6203-499a-968a-a4ecfbf36dce"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:20.042606955Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1780,"client":"1","":"downstairs","session_id":"6f0f2c77-6203-499a-968a-a4ecfbf36dce"} {"msg":"[0]R flush_numbers[0..12]: [267, 267, 267, 265, 266, 267, 265, 264, 267, 267, 265, 265]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:20.042629894Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1780,"":"downstairs","":"downstairs","session_id":"6f0f2c77-6203-499a-968a-a4ecfbf36dce"} {"msg":"[0]R generation[0..12]: [149, 149, 149, 147, 148, 149, 147, 146, 149, 149, 147, 147]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:20.042658672Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1780,"":"downstairs","":"downstairs","session_id":"6f0f2c77-6203-499a-968a-a4ecfbf36dce"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:20.042686601Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1780,"":"downstairs","":"downstairs","session_id":"6f0f2c77-6203-499a-968a-a4ecfbf36dce"} {"msg":"[1]R flush_numbers[0..12]: [267, 267, 267, 265, 266, 267, 265, 264, 267, 267, 265, 265]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:20.042713129Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1780,"":"downstairs","":"downstairs","session_id":"6f0f2c77-6203-499a-968a-a4ecfbf36dce"} {"msg":"[1]R generation[0..12]: [149, 149, 149, 147, 148, 149, 147, 146, 149, 149, 147, 147]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:20.042739038Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1780,"":"downstairs","":"downstairs","session_id":"6f0f2c77-6203-499a-968a-a4ecfbf36dce"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:20.042768387Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1780,"":"downstairs","":"downstairs","session_id":"6f0f2c77-6203-499a-968a-a4ecfbf36dce"} Wait for a query_work_queue command to finish before sending IO {"msg":"[2]R flush_numbers[0..12]: [267, 267, 267, 265, 266, 267, 265, 264, 267, 267, 265, 265]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:20.042805795Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1780,"":"downstairs","":"downstairs","session_id":"6f0f2c77-6203-499a-968a-a4ecfbf36dce"} {"msg":"[2]R generation[0..12]: [149, 149, 149, 147, 148, 149, 147, 146, 149, 149, 147, 147]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:20.042837493Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1780,"":"downstairs","":"downstairs","session_id":"6f0f2c77-6203-499a-968a-a4ecfbf36dce"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:20.042867802Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1780,"":"downstairs","":"downstairs","session_id":"6f0f2c77-6203-499a-968a-a4ecfbf36dce"} {"msg":"Max found gen is 150","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:20.042895561Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1780,"":"downstairs","":"downstairs","session_id":"6f0f2c77-6203-499a-968a-a4ecfbf36dce"} {"msg":"Generation requested: 150 >= found:150","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:20.042923399Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1780,"":"downstairs","":"downstairs","session_id":"6f0f2c77-6203-499a-968a-a4ecfbf36dce"} {"msg":"Next flush: 268","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:20.042951708Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1780,"":"downstairs","":"downstairs","session_id":"6f0f2c77-6203-499a-968a-a4ecfbf36dce"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:20.042980936Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid"Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 :1780,"":"downstairs","":"downstairs","session_id":"6f0f2c77-6203-499a-968a-a4ecfbf36dce"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:20.043014615Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1780,"session_id":"6f0f2c77-6203-499a-968a-a4ecfbf36dce"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:20.043043383Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1780,"client":"0","":"downstairs","session_id":"6f0f2c77-6203-499a-968a-a4ecfbf36dce"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:20.043060133Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1780,"client":"1","":"downstairs","session_id":"6f0f2c77-6203-499a-968a-a4ecfbf36dce"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:20.043073892Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1780,"client":"2","":"downstairs","session_id":"6f0f2c77-6203-499a-968a-a4ecfbf36dce"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:20.043087381Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1780,"session_id":"6f0f2c77-6203-499a-968a-a4ecfbf36dce"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:20.043099711Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1780,"session_id":"6f0f2c77-6203-499a-968a-a4ecfbf36dce"} {"msg":"f3d65ba3-5fa0-4e9e-95c6-65763a224d59 is now active with session: 6f0f2c77-6203-499a-968a-a4ecfbf36dce","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:20.04311269Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1780,"session_id":"6f0f2c77-6203-499a-968a-a4ecfbf36dce"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:20.043124969Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1780,"session_id":"6f0f2c77-6203-499a-968a-a4ecfbf36dce"} {"msg":"The guest has finished waiting for activation with:150","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:20.043137369Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1780} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 377 len 20480 data: 12 12 12 11 10 02/30 Read block 12 len 32768 03/30 Read block 244 len 12288 04/30 Flush 05/30 Write block 571 len 4096 data: 13 06/30 Read block 458 len 36864 07/30 Read block 285 len 32768 08/30 Read block 232 len 12288 09/30 Write block 33 len 28672 data: 9 11 10 10 11 13 14 10/30 Read block 152 len 20480 11/30 Write block 270 len 32768 data: 13 12 13 14 16 14 15 13 12/30 Write block 369 len 8192 data: 12 11 13/30 Read block 26 len 24576 14/30 Flush 15/30 Write block 427 len 36864 data: 9 9 15 17 16 14 13 12 15 16/30 Write block 226 len 28672 data: 14 16 12 12 9 7 10 17/30 Read block 377 len 12288 18/30 Write block 515 len 4096 data: 9 19/30 Read block 71 len 4096 20/30 Read block 368 len 4096 21/30 Write block 364 len 8192 data: 15 12 22/30 Write block 544 len 28672 data: 16 12 13 10 12 9 8 23/30 Flush 24/30 Write block 275 len 12288 data: 15 16 14 25/30 Read block 441 len 24576 26/30 Write block 330 len 24576 data: 9 9 10 11 12 12 27/30 Write block 581 len 40960 data: 11 12 14 17 20 16 18 15 13 11 28/30 Write block 377 len 20480 data: 13 13 13 12 11 29/30 Read block 222 len 28672 30/30 Read block 420 len 20480 ---------------------------------------------------------------- Crucible gen:150 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 3 Done Done Done false 1024 Acked Read 6 Done Done Done false 1025 Acked Write 6 Done Done Done false 1026 Acked Write 10 Sent Done Sent false 1027 Acked Write 5 Sent Done Sent false 1028 Acked Read 7 Sent Done Sent false 1029 Acked Read 5 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 4 0 4 8 Done 3 7 3 13 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:28:20.06990967Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1780} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 149 149 149 267 267 267 F F F 1 020-039 150 150 150 269 269 269 F F F 2 040-059 149 149 149 267 267 267 F F F 3 060-079 147 147 147 265 265 265 F F F 4 080-099 148 148 148 266 266 266 F F F 5 100-119 149 149 149 267 267 267 F F F 6 120-139 147 147 147 265 265 265 F F F 7 140-159 146 146 146 264 264 264 F F F 8 160-179 149 149 149 267 267 267 F F F 9 180-199 149 149 149 267 267 267 F F F 10 200-219 147 147 147 265 265 265 F F F 11 220-239 150 150 150 270 270 270 F F F 12 240-259 144 144 144 262 262 262 F F F 13 260-279 150 150 150 269 269 269 T T T 14 280-299 147 147 147 265 265 265 F F F 15 300-319 145 145 145 263 263 263 F F F 16 320-339 148 148 148 266 266 266 T T T 17 340-359 149 149 149 267 267 267 F F F 18 360-379 150 150 150 270 270 270 T T T 19 380-399 150 150 150 268 268 268 T T T 20 400-419 141 141 141 259 259 259 F F F 21 420-439 150 150 150 270 270 270 F F F 22 440-459 145 145 145 263 263 263 F F F 23 460-479 148 148 148 266 266 266 F F F 24 480-499 147 147 147 265 265 265 F F F 25 500-519 150 150 150 270 270 270 F F F 26 520-539 148 148 148 266 266 266 F F F 27 540-559 150 150 150 270 270 270 F F F 28 560-579 150 150 150 269 269 269 F F F 29 580-599 148 148 148 266 266 266 T T T Max gen: 150, Max flush: 270 A Difference in extent metadata is expected here On loop 75 Verifying data now No change after verify Loop: 75 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 149 149 149 267 267 267 F F F 1 020-039 150 150 150 269 269 269 F F F 2 040-059 149 149 149 267 267 267 F F F 3 060-079 147 147 147 265 265 265 F F F 4 080-099 148 148 148 266 266 266 F F F 5 100-119 149 149 149 267 267 267 F F F 6 120-139 147 147 147 265 265 265 F F F 7 140-159 146 146 146 264 264 264 F F F 8 160-179 149 149 149 267 267 267 F F F 9 180-199 149 149 149 267 267 267 F F F 10 200-219 147 147 147 265 265 265 F F F 11 220-239 150 150 150 270 270 270 F F F 12 240-259 144 144 144 262 262 262 F F F 13 260-279 151 151 151 271 271 271 F F F 14 280-299 147 147 147 265 265 265 F F F 15 300-319 145 145 145 263 263 263 F F F 16 320-339 151 151 151 271 271 271 F F F 17 340-359 149 149 149 267 267 267 F F F 18 360-379 151 151 151 271 271 271 F F F 19 380-399 151 151 151 271 271 271 F F F 20 400-419 141 141 141 259 259 259 F F F 21 420-439 150 150 150 270 270 270 F F F 22 440-459 145 145 145 263 263 263 F F F 23 460-479 148 148 148 266 266 266 F F F 24 480-499 147 147 147 265 265 265 F F F 25 500-519 150 150 150 270 270 270 F F F 26 520-539 148 148 148 266 266 266 F F F 27 540-559 150 150 150 270 270 270 F F F 28 560-579 150 150 150 269 269 269 F F F 29 580-599 151 151 151 271 271 271 F F F Max gen: 151, Max flush: 271 Begin loop 76 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:24.38979271Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1788,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:24.434372811Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1788,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:24.435040198Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1788} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"0fd98d0d5d909611f8b408ccda2559d35fe117bb\",\n git_commit_timestamp: \"2024-11-08T20:05:44.000000000Z\",\n git_branch: \"mkeeter/drop-skipped-backpressure-guard\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:24.435085126Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1788} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:24.435103115Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1788} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:24.435121344Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1788} {"msg":"Crucible 36fd59de-3d6a-4734-a5df-d89f18658d98 has session id: 7a92f81b-fbf7-471e-915e-562e66a0e938","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:24.435653228Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1788,"session_id":"7a92f81b-fbf7-471e-915e-562e66a0e938"} {"msg":"Upstairs opts: Upstairs UUID: 36fd59de-3d6a-4734-a5df-d89f18658d98, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:24.435691376Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1788,"session_id":"7a92f81b-fbf7-471e-915e-562e66a0e938"} {"msg":"Crucible stats registered with UUID: 36fd59de-3d6a-4734-a5df-d89f18658d98","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:24.435708106Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1788,"session_id":"7a92f81b-fbf7-471e-915e-562e66a0e938"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:24.466700461Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1788,"task":"crutest"} {"msg":"The guest has requested activation with gen:152","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:24.466732049Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1788} {"msg":"36fd59de-3d6a-4734-a5df-d89f18658d98 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:24.466797596Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1788,"session_id":"7a92f81b-fbf7-471e-915e-562e66a0e938"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:24.466831864Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1788,"client":"0","":"downstairs","session_id":"7a92f81b-fbf7-471e-915e-562e66a0e938"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:24.466862873Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1788,"client":"0","":"downstairs","session_id":"7a92f81b-fbf7-471e-915e-562e66a0e938"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:24.466885992Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1788,"client":"1","":"downstairs","session_id":"7a92f81b-fbf7-471e-915e-562e66a0e938"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:24.466910901Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1788,"client":"1","":"downstairs","session_id":"7a92f81b-fbf7-471e-915e-562e66a0e938"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:24.466943319Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1788,"client":"2","":"downstairs","session_id":"7a92f81b-fbf7-471e-915e-562e66a0e938"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:24.466976357Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1788,"client":"2","":"downstairs","session_id":"7a92f81b-fbf7-471e-915e-562e66a0e938"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:24.467041734Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1788,"":"io task","client":"2","":"downstairs","session_id":"7a92f81b-fbf7-471e-915e-562e66a0e938"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:24.467083112Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1788,"":"io task","client":"0","":"downstairs","session_id":"7a92f81b-fbf7-471e-915e-562e66a0e938"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:24.467213656Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1788,"":"io task","client":"1","":"downstairs","session_id":"7a92f81b-fbf7-471e-915e-562e66a0e938"} {"msg":"ds_connection connected from Ok(127.0.0.1:44467)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:24.467239074Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1788,"":"io task","client":"2","":"downstairs","session_id":"7a92f81b-fbf7-471e-915e-562e66a0e938"} {"msg":"ds_connection connected from Ok(127.0.0.1:56722)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:24.467257194Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1788,"":"io task","client":"0","":"downstairs","session_id":"7a92f81b-fbf7-471e-915e-562e66a0e938"} {"msg":"ds_connection connected from Ok(127.0.0.1:45936)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:24.467274833Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1788,"":"io task","client":"1","":"downstairs","session_id":"7a92f81b-fbf7-471e-915e-562e66a0e938"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:24.467840825Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1788,"client":"2","":"downstairs","session_id":"7a92f81b-fbf7-471e-915e-562e66a0e938"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:24.467879593Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1788,"client":"2","":"downstairs","session_id":"7a92f81b-fbf7-471e-915e-562e66a0e938"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:24.467973729Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1788,"client":"0","":"downstairs","session_id":"7a92f81b-fbf7-471e-915e-562e66a0e938"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:24.468034756Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1788,"client":"0","":"downstairs","session_id":"7a92f81b-fbf7-471e-915e-562e66a0e938"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:24.468100872Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1788,"client":"1","":"downstairs","session_id":"7a92f81b-fbf7-471e-915e-562e66a0e938"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:24.468136031Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1788,"client":"1","":"downstairs","session_id":"7a92f81b-fbf7-471e-915e-562e66a0e938"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:24.468639796Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1788,"client":"0","":"downstairs","session_id":"7a92f81b-fbf7-471e-915e-562e66a0e938"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:24.468672135Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1788,"client":"2","":"downstairs","session_id":"7a92f81b-fbf7-471e-915e-562e66a0e938"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:24.468690354Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1788,"client":"1","":"downstairs","session_id":"7a92f81b-fbf7-471e-915e-562e66a0e938"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:24.469266175Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1788,"client":"0","":"downstairs","session_id":"7a92f81b-fbf7-471e-915e-562e66a0e938"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:24.469295834Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1788,"client":"0","":"downstairs","session_id":"7a92f81b-fbf7-471e-915e-562e66a0e938"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:24.469311473Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1788,"session_id":"7a92f81b-fbf7-471e-915e-562e66a0e938"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:24.469324312Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1788,"client":"2","":"downstairs","session_id":"7a92f81b-fbf7-471e-915e-562e66a0e938"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:24.469338442Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1788,"client":"2","":"downstairs","session_id":"7a92f81b-fbf7-471e-915e-562e66a0e938"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:24.469352251Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1788,"session_id":"7a92f81b-fbf7-471e-915e-562e66a0e938"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:24.469364761Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1788,"client":"1","":"downstairs","session_id":"7a92f81b-fbf7-471e-915e-562e66a0e938"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:24.46937835Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1788,"client":"1","":"downstairs","session_id":"7a92f81b-fbf7-471e-915e-562e66a0e938"} {"msg":"[0]R flush_numbers[0..12]: [267, 269, 267, 265, 266, 267, 265, 264, 267, 267, 265, 270]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:24.469392509Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1788,"":"downstairs","":"downstairs","session_id":"7a92f81b-fbf7-471e-915e-562e66a0e938"} {"msg":"[0]R generation[0..12]: [149, 150, 149, 147, 148, 149, 147, 146, 149, 149, 147, 150]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:24.469406628Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1788,"":"downstairs","":"downstairs","session_id":"7a92f81b-fbf7-471e-915e-562e66a0e938"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:24.469420718Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1788,"":"downstairs","":"downstairs","session_id":"7a92f81b-fbf7-471e-915e-562e66a0e938"} {"msg":"[1]R flush_numbers[0..12]: [267, 269, 267, 265, 266, 267, 265, 264, 267, 267, 265, 270]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:24.469438067Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1788,"":"downstairs","":"downstairs","session_id":"7a92f81b-fbf7-471e-915e-562e66a0e938"} {"msg":"[1]R generation[0..12]: [149, 150, 149, 147, 148, 149, 147, 146, 149, 149, 147, 150]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:24.469452376Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1788,"":"downstairs","":"downstairs","session_id":"7a92f81b-fbf7-471e-915e-562e66a0e938"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:24.469466496Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1788,"":"downstairs","":"downstairs","session_id":"7a92f81b-fbf7-471e-915e-562e66a0e938"} {"msg":"[2]R flush_numbers[0..12]: [267, 269, 267, 265, 266, 267, 265, 264, 267, 267, 265, 270]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:24.469480285Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1788,"":"downstairs","":"downstairs","session_id":"7a92f81b-fbf7-471e-915e-562e66a0e938"} {"msg":"[2]R generation[0..12]: [149, 150, 149, 147, 148, 149, 147, 146, 149, 149, 147, 150]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:24.469494014Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1788,"":"downstairs","":"downstairs","session_id":"7a92f81b-fbf7-471e-915e-562e66a0e938"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:24.469507944Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1788,"":"downstairs","":"downstairs","session_id":"7a92f81b-fbf7-471e-915e-562e66a0e938"} {"msg":"Max found gen is 152","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-08T20:28:24.469521043Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1788,"":"downstairs","":"downstairs","session_id":"7a92f81b-fbf7-471e-915e-562e66a0e938"} {"msg":"Generation requested: 152 >= found:152","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:24.469594119Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1788,"":"downstairs","":"downstairs","session_id":"7a92f81b-fbf7-471e-915e-562e66a0e938"} {"msg":"Next flush: 272","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:24.469618298Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1788,"":"downstairs","":"downstairs","session_id":"7a92f81b-fbf7-471e-915e-562e66a0e938"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:24.469633277Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1788,"":"downstairs","":"downstairs","session_id":"7a92f81b-fbf7-471e-915e-562e66a0e938"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:24.469646847Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1788,"session_id":"7a92f81b-fbf7-471e-915e-562e66a0e938"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:24.469659126Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1788,"client":"0","":"downstairs","session_id":"7a92f81b-fbf7-471e-915e-562e66a0e938"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:24.469672726Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1788,"client":"1","":"downstairs","session_id":"7a92f81b-fbf7-471e-915e-562e66a0e938"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:24.469686095Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1788,"client":"2","":"downstairs","session_id":"7a92f81b-fbf7-471e-915e-562e66a0e938"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:24.469699374Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1788,"session_id":"7a92f81b-fbf7-471e-915e-562e66a0e938"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:24.469715893Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1788,"session_id":"7a92f81b-fbf7-471e-915e-562e66a0e938"} {"msg":"36fd59de-3d6a-4734-a5df-d89f18658d98 is now active with session: 7a92f81b-fbf7-471e-915e-562e66a0e938","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:24.469729553Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1788,"session_id":"7a92f81b-fbf7-471e-915e-562e66a0e938"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:24.469741762Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1788,"session_id":"7a92f81b-fbf7-471e-915e-562e66a0e938"} {"msg":"The guest has finished waiting for activation with:152","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:24.469754222Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1788} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 100 len 12288 data: 10 10 9 02/30 Write block 221 len 4096 data: 16 03/30 Write block 425 len 24576 data: 11 10 10 10 16 18 04/30 Read block 284 len 12288 05/30 Read block 57 len 16384 06/30 Read block 97 len 36864 07/30 Read block 291 len 36864 08/30 Write block 517 len 40960 data: 7 11 13 10 10 9 10 5 7 6 09/30 Write block 447 len 24576 data: 13 11 11 9 9 11 10/30 Write block 429 len 28672 data: 17 19 17 15 14 13 16 11/30 Read block 191 len 8192 12/30 Read block 415 len 28672 13/30 Write block 323 len 40960 data: 13 10 10 13 8 9 9 10 10 11 14/30 Read block 312 len 28672 15/30 Read block 266 len 40960 16/30 Read block 211 len 20480 17/30 Write block 305 len 12288 data: 10 10 6 18/30 Flush 19/30 Write block 419 len 8192 data: 10 9 20/30 Write block 550 len 36864 data: 9 5 8 7 9 10 10 10 10 21/30 Read block 295 len 24576 22/30 Read block 110 len 24576 23/30 Flush 24/30 Write block 405 len 16384 data: 9 10 10 11 25/30 Read block 269 len 36864 26/30 Read block 517 len 20480 27/30 Write block 104 len 40960 data: 10 11 12 15 13 13 12 8 8 9 28/30 Write block 158 len 24576 data: 6 6 5 6 6 6 29/30 Write block 124 len 12288 data: 15 11 11 30/30 Write block 285 len 16384 data: 15 17 17 18 ---------------------------------------------------------------- Crucible gen:152 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 4 Done Done Done false 1024 Acked Read 9 Done Done Done false 1025 Acked Read 5 Done Done Done false 1026 Acked Write 10 Done Done Done false 1027 Acked Write 6 Sent Sent Sent false 1028 Acked Write 3 Sent Sent Sent false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 3 3 9 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:28:24.495813998Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1788} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 149 149 149 267 267 267 F F F 1 020-039 150 150 150 269 269 269 F F F 2 040-059 149 149 149 267 267 267 F F F 3 060-079 147 147 147 265 265 265 F F F 4 080-099 148 148 148 266 266 266 F F F 5 100-119 152 152 152 272 272 272 T T T 6 120-139 147 147 147 265 265 265 T T T 7 140-159 146 146 146 264 264 264 T T T 8 160-179 149 149 149 267 267 267 T T T 9 180-199 149 149 149 267 267 267 F F F 10 200-219 147 147 147 265 265 265 F F F 11 220-239 152 152 152 272 272 272 F F F 12 240-259 144 144 144 262 262 262 F F F 13 260-279 151 151 151 271 271 271 F F F 14 280-299 147 147 147 265 265 265 T T T 15 300-319 152 152 152 272 272 272 F F F 16 320-339 152 152 152 272 272 272 F F F 17 340-359 149 149 149 267 267 267 F F F 18 360-379 151 151 151 271 271 271 F F F 19 380-399 151 151 151 271 271 271 F F F 20 400-419 152 152 152 273 273 273 T T T 21 420-439 152 152 152 273 273 273 F F F 22 440-459 152 152 152 272 272 272 F F F 23 460-479 148 148 148 266 266 266 F F F 24 480-499 147 147 147 265 265 265 F F F 25 500-519 152 152 152 272 272 272 F F F 26 520-539 152 152 152 272 272 272 F F F 27 540-559 152 152 152 273 273 273 F F F 28 560-579 150 150 150 269 269 269 F F F 29 580-599 151 151 151 271 271 271 F F F Max gen: 152, Max flush: 273 A Difference in extent metadata is expected here On loop 76 Verifying data now No change after verify Loop: 76 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 149 149 149 267 267 267 F F F 1 020-039 150 150 150 269 269 269 F F F 2 040-059 149 149 149 267 267 267 F F F 3 060-079 147 147 147 265 265 265 F F F 4 080-099 148 148 148 266 266 266 F F F 5 100-119 153 153 153 274 274 274 F F F 6 120-139 153 153 153 274 274 274 F F F 7 140-159 153 153 153 274 274 274 F F F 8 160-179 153 153 153 274 274 274 F F F 9 180-199 149 149 149 267 267 267 F F F 10 200-219 147 147 147 265 265 265 F F F 11 220-239 152 152 152 272 272 272 F F F 12 240-259 144 144 144 262 262 262 F F F 13 260-279 151 151 151 271 271 271 F F F 14 280-299 153 153 153 274 274 274 F F F 15 300-319 152 152 152 272 272 272 F F F 16 320-339 152 152 152 272 272 272 F F F 17 340-359 149 149 149 267 267 267 F F F 18 360-379 151 151 151 271 271 271 F F F 19 380-399 151 151 151 271 271 271 F F F 20 400-419 153 153 153 274 274 274 F F F 21 420-439 152 152 152 273 273 273 F F F 22 440-459 152 152 152 272 272 272 F F F 23 460-479 148 148 148 266 266 266 F F F 24 480-499 147 147 147 265 265 265 F F F 25 500-519 152 152 152 272 272 272 F F F 26 520-539 152 152 152 272 272 272 F F F 27 540-559 152 152 152 273 273 273 F F F 28 560-579 150 150 150 269 269 269 F F F 29 580-599 151 151 151 271 271 271 F F F Max gen: 153, Max flush: 274 Begin loop 77 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:28.829414247Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1796,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:28.874655536Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1796,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:28.875319454Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1796} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"0fd98d0d5d909611f8b408ccda2559d35fe117bb\",\n git_commit_timestamp: \"2024-11-08T20:05:44.000000000Z\",\n git_branch: \"mkeeter/drop-skipped-backpressure-guard\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:28.875357732Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1796} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:28.875376261Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1796} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:28.87539128Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1796} {"msg":"Crucible 8d29a67e-cf45-490d-9525-95eb5d3d3453 has session id: 59b79dc1-e241-4493-9e0e-5d59eafbc992","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:28.875863217Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1796,"session_id":"59b79dc1-e241-4493-9e0e-5d59eafbc992"} {"msg":"Upstairs opts: Upstairs UUID: 8d29a67e-cf45-490d-9525-95eb5d3d3453, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:28.875909055Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1796,"session_id":"59b79dc1-e241-4493-9e0e-5d59eafbc992"} {"msg":"Crucible stats registered with UUID: 8d29a67e-cf45-490d-9525-95eb5d3d3453","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:28.875934443Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1796,"session_id":"59b79dc1-e241-4493-9e0e-5d59eafbc992"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:28.907455023Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1796,"task":"crutest"} {"msg":"The guest has requested activation with gen:154","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:28.907489001Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1796} {"msg":"8d29a67e-cf45-490d-9525-95eb5d3d3453 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:28.90752627Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1796,"session_id":"59b79dc1-e241-4493-9e0e-5d59eafbc992"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:28.907543509Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1796,"client":"0","":"downstairs","session_id":"59b79dc1-e241-4493-9e0e-5d59eafbc992"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:28.907571547Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1796,"client":"0","":"downstairs","session_id":"59b79dc1-e241-4493-9e0e-5d59eafbc992"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:28.907589556Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1796,"client":"1","":"downstairs","session_id":"59b79dc1-e241-4493-9e0e-5d59eafbc992"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:28.907606626Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1796,"client":"1","":"downstairs","session_id":"59b79dc1-e241-4493-9e0e-5d59eafbc992"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:28.907626955Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1796,"client":"2","":"downstairs","session_id":"59b79dc1-e241-4493-9e0e-5d59eafbc992"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:28.907656493Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1796,"client":"2","":"downstairs","session_id":"59b79dc1-e241-4493-9e0e-5d59eafbc992"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:28.907675282Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1796,"":"io task","client":"0","":"downstairs","session_id":"59b79dc1-e241-4493-9e0e-5d59eafbc992"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:28.907697531Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1796,"":"io task","client":"2","":"downstairs","session_id":"59b79dc1-e241-4493-9e0e-5d59eafbc992"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:28.9077296Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1796,"":"io task","client":"1","":"downstairs","session_id":"59b79dc1-e241-4493-9e0e-5d59eafbc992"} {"msg":"ds_connection connected from Ok(127.0.0.1:44858)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:28.907766508Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1796,"":"io task","client":"1","":"downstairs","session_id":"59b79dc1-e241-4493-9e0e-5d59eafbc992"} {"msg":"ds_connection connected from Ok(127.0.0.1:52107)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:28.907787307Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1796,"":"io task","client":"0","":"downstairs","session_id":"59b79dc1-e241-4493-9e0e-5d59eafbc992"} {"msg":"ds_connection connected from Ok(127.0.0.1:34590)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:28.908102451Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1796,"":"io task","client":"2","":"downstairs","session_id":"59b79dc1-e241-4493-9e0e-5d59eafbc992"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:28.908565119Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1796,"client":"1","":"downstairs","session_id":"59b79dc1-e241-4493-9e0e-5d59eafbc992"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:28.908602277Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1796,"client":"1","":"downstairs","session_id":"59b79dc1-e241-4493-9e0e-5d59eafbc992"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:28.908629366Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1796,"client":"0","":"downstairs","session_id":"59b79dc1-e241-4493-9e0e-5d59eafbc992"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:28.908657754Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1796,"client":"0","":"downstairs","session_id":"59b79dc1-e241-4493-9e0e-5d59eafbc992"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:28.908704202Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1796,"client":"2","":"downstairs","session_id":"59b79dc1-e241-4493-9e0e-5d59eafbc992"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:28.908725181Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1796,"client":"2","":"downstairs","session_id":"59b79dc1-e241-4493-9e0e-5d59eafbc992"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:28.909247486Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1796,"client":"0","":"downstairs","session_id":"59b79dc1-e241-4493-9e0e-5d59eafbc992"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:28.909287844Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1796,"client":"1","":"downstairs","session_id":"59b79dc1-e241-4493-9e0e-5d59eafbc992"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:28.909319472Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1796,"client":"2","":"downstairs","session_id":"59b79dc1-e241-4493-9e0e-5d59eafbc992"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:28.909798619Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1796,"client":"0","":"downstairs","session_id":"59b79dc1-e241-4493-9e0e-5d59eafbc992"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:28.909823527Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1796,"client":"0","":"downstairs","session_id":"59b79dc1-e241-4493-9e0e-5d59eafbc992"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:28.909851926Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1796,"session_id":"59b79dc1-e241-4493-9e0e-5d59eafbc992"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:28.909932152Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1796,"client":"1","":"downstairs","session_id":"59b79dc1-e241-4493-9e0e-5d59eafbc992"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:28.90996698Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1796,"client":"1","":"downstairs","session_id":"59b79dc1-e241-4493-9e0e-5d59eafbc992"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:28.909989189Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1796,"session_id":"59b79dc1-e241-4493-9e0e-5d59eafbc992"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:28.910002458Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1796,"client":"2","":"downstairs","session_id":"59b79dc1-e241-4493-9e0e-5d59eafbc992"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:28.910016078Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1796,"client":"2","":"downstairs","session_id":"59b79dc1-e241-4493-9e0e-5d59eafbc992"} {"msg":"[0]R flush_numbers[0..12]: [267, 269, 267, 265, 266, 274, 274, 274, 274, 267, 265, 272]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:28.910030207Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1796,"":"downstairs","":"downstairs","session_id":"59b79dc1-e241-4493-9e0e-5d59eafbc992"} {"msg":"[0]R generation[0..12]: [149, 150, 149, 147, 148, 153, 153, 153, 153, 149, 147, 152]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:28.910044027Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1796,"":"downstairs","":"downstairs","session_id":"59b79dc1-e241-4493-9e0e-5d59eafbc992"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:28.910058086Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1796,"":"downstairs","":"downstairs","session_id":"59b79dc1-e241-4493-9e0e-5d59eafbc992"} {"msg":"[1]R flush_numbers[0..12]: [267, 269, 267, 265, 266, 274, 274, 274, 274, 267, 265, 272]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:28.910071925Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1796,"":"downstairs","":"downstairs","session_id":"59b79dc1-e241-4493-9e0e-5d59eafbc992"} {"msg":"[1]R generation[0..12]: [149, 150, 149, 147, 148, 153, 153, 153, 153, 149, 147, 152]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:28.910085634Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1796,"":"downstairs","":"downstairs","session_id":"59b79dc1-e241-4493-9e0e-5d59eafbc992"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:28.910099484Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1796,"":"downstairs","":"downstairs","session_id":"59b79dc1-e241-4493-9e0e-5d59eafbc992"} {"msg":"[2]R flush_numbers[0..12]: [267, 269, 267, 265, 266, 274, 274, 274, 274, 267, 265, 272]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:28.910117453Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1796,"":"downstairs","":"downstairs","session_id":"59b79dc1-e241-4493-9e0e-5d59eafbc992"} {"msg":"[2]R generation[0..12]: [149, 150, 149, 147, 148, 153, 153, 153, 153, 149, 147, 152]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:28.910131672Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1796,"":"downstairs","":"downstairs","session_id":"59b79dc1-e241-4493-9e0e-5d59eafbc992"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:28.910147202Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1796,"":"downstairs","":"downstairs","session_id":"59b79dc1-e241-4493-9e0e-5d59eafbc992"} {"msg":"Max found gen is 154","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:28.91017061Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW"Wait for a query_work_queue command to finish before sending IO ,"pid":1796,"":"downstairs","":"downstairs","session_id":"59b79dc1-e241-4493-9e0e-5d59eafbc992"} {"msg":"Generation requested: 154 >= found:154","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:28.910205599Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1796,"":"downstairs","":"downstairs","session_id":"59b79dc1-e241-4493-9e0e-5d59eafbc992"} {"msg":"Next flush: 275","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:28.910225318Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1796,"":"downstairs","":"downstairs","session_id":"59b79dc1-e241-4493-9e0e-5d59eafbc992"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:28.910238937Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1796,"":"downstairs","":"downstairs","session_id":"59b79dc1-e241-4493-9e0e-5d59eafbc992"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:28.910252266Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1796,"session_id":"59b79dc1-e241-4493-9e0e-5d59eafbc992"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:28.910264306Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1796,"client":"0","":"downstairs","session_id":"59b79dc1-e241-4493-9e0e-5d59eafbc992"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:28.910277755Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1796,"client":"1","":"downstairs","session_id":"59b79dc1-e241-4493-9e0e-5d59eafbc992"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:28.910291084Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1796,"client":"2","":"downstairs","session_id":"59b79dc1-e241-4493-9e0e-5d59eafbc992"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:28.910304444Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1796,"session_id":"59b79dc1-e241-4493-9e0e-5d59eafbc992"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:28.910316743Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1796,"session_id":"59b79dc1-e241-4493-9e0e-5d59eafbc992Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 "} {"msg":"8d29a67e-cf45-490d-9525-95eb5d3d3453 is now active with session: 59b79dc1-e241-4493-9e0e-5d59eafbc992","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:28.910333252Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1796,"session_id":"59b79dc1-e241-4493-9e0e-5d59eafbc992"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:28.910345422Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1796,"session_id":"59b79dc1-e241-4493-9e0e-5d59eafbc992"} {"msg":"The guest has finished waiting for activation with:154","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:28.910360111Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1796} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 420 len 12288 data: 10 11 11 02/30 Flush 03/30 Write block 16 len 28672 data: 15 14 11 10 9 11 11 04/30 Read block 155 len 20480 05/30 Write block 99 len 40960 data: 8 11 11 10 10 11 12 13 16 14 06/30 Write block 104 len 36864 data: 12 13 14 17 15 14 13 9 9 07/30 Write block 102 len 20480 data: 11 11 13 14 15 08/30 Write block 372 len 32768 data: 13 10 10 10 10 14 14 14 09/30 Write block 135 len 28672 data: 15 13 13 11 10 8 10 10/30 Write block 268 len 24576 data: 13 13 14 13 14 15 11/30 Read block 313 len 12288 12/30 Write block 20 len 4096 data: 10 13/30 Write block 42 len 20480 data: 13 12 11 10 7 14/30 Read block 303 len 8192 15/30 Write block 48 len 20480 data: 6 7 9 8 7 16/30 Write block 239 len 24576 data: 12 13 12 14 12 12 17/30 Read block 120 len 20480 18/30 Write block 417 len 32768 data: 10 10 11 11 12 12 11 11 19/30 Read block 370 len 16384 20/30 Read block 578 len 4096 21/30 Read block 350 len 32768 22/30 Write block 223 len 4096 data: 12 23/30 Read block 516 len 24576 24/30 Write block 577 len 28672 data: 9 8 7 11 12 13 15 25/30 Write block 471 len 28672 data: 19 18 17 16 18 17 19 26/30 Read block 576 len 28672 27/30 Write block 523 len 20480 data: 11 6 8 7 9 28/30 Write block 65 len 16384 data: 11 10 9 9 29/30 Write block 42 len 36864 data: 14 13 12 11 8 4 7 8 10 30/30 Write block 7 len 8192 data: 11 13 ---------------------------------------------------------------- Crucible gen:154 GIO:true work queues: Upstairs:0 downstairs:28 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1002 Acked Write 7 Done Done Done false 1003 Acked Read 5 Done Done Done false 1004 Acked Write 10 Done Done Done false 1005 Acked Write 9 Done Done Done false 1006 Acked Write 5 Done Done Done false 1007 Acked Write 8 Done Done Done false 1008 Acked Write 7 Done Done Done false 1009 Acked Write 6 Done Done Done false 1010 Acked Read 3 Done Done Done false 1011 Acked Write 1 Done Done Done false 1012 Acked Write 5 Done Done Done false 1013 Acked Read 2 Done Done Done false 1014 Acked Write 5 Done Done Done false 1015 Acked Write 6 Done Done Done false 1016 Acked Read 5 Done Done Done false 1017 Acked Write 8 Done Done Done false 1018 Acked Read 4 Done Done Done false 1019 Acked Read 1 Done Done Done false 1020 Acked Read 8 Done Done Done false 1021 Acked Write 1 Done Done Done false 1022 Acked Read 6 Done Done Done false 1023 Acked Write 7 Done Done Done false 1024 Acked Write 7 Done Done Done false 1025 Acked Read 7 Done Done Done false 1026 Acked Write 5 Sent Done Done false 1027 Acked Write 4 Sent Done Done false 1028 Acked Write 9 Sent Sent Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 4 2 2 8 Done 24 26 26 76 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1001 1001 1001 Downstairs last five completed: 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:28:28.932100778Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1796} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 149 149 149 267 267 267 T T T 1 020-039 150 150 150 269 269 269 T T T 2 040-059 149 149 149 267 267 267 T T T 3 060-079 147 147 147 265 265 265 T T T 4 080-099 148 148 148 266 266 266 T T T 5 100-119 153 153 153 274 274 274 T T T 6 120-139 153 153 153 274 274 274 T T T 7 140-159 153 153 153 274 274 274 T T T 8 160-179 153 153 153 274 274 274 F F F 9 180-199 149 149 149 267 267 267 F F F 10 200-219 147 147 147 265 265 265 F F F 11 220-239 152 152 152 272 272 272 T T T 12 240-259 144 144 144 262 262 262 T T T 13 260-279 151 151 151 271 271 271 T T T 14 280-299 153 153 153 274 274 274 F F F 15 300-319 152 152 152 272 272 272 F F F 16 320-339 152 152 152 272 272 272 F F F 17 340-359 149 149 149 267 267 267 F F F 18 360-379 151 151 151 271 271 271 T T T 19 380-399 151 151 151 271 271 271 F F F 20 400-419 153 153 153 274 274 274 T T T 21 420-439 154 154 154 275 275 275 T T T 22 440-459 152 152 152 272 272 272 F F F 23 460-479 148 148 148 266 266 266 T T T 24 480-499 147 147 147 265 265 265 F F F 25 500-519 152 152 152 272 272 272 F F F 26 520-539 152 152 152 272 272 272 T T T 27 540-559 152 152 152 273 273 273 F F F 28 560-579 150 150 150 269 269 269 T T T 29 580-599 151 151 151 271 271 271 T T T Max gen: 154, Max flush: 275 A Difference in extent metadata is expected here On loop 77 Verifying data now No change after verify Loop: 77 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 155 155 155 276 276 276 F F F 1 020-039 155 155 155 276 276 276 F F F 2 040-059 155 155 155 276 276 276 F F F 3 060-079 155 155 155 276 276 276 F F F 4 080-099 155 155 155 276 276 276 F F F 5 100-119 155 155 155 276 276 276 F F F 6 120-139 155 155 155 276 276 276 F F F 7 140-159 155 155 155 276 276 276 F F F 8 160-179 153 153 153 274 274 274 F F F 9 180-199 149 149 149 267 267 267 F F F 10 200-219 147 147 147 265 265 265 F F F 11 220-239 155 155 155 276 276 276 F F F 12 240-259 155 155 155 276 276 276 F F F 13 260-279 155 155 155 276 276 276 F F F 14 280-299 153 153 153 274 274 274 F F F 15 300-319 152 152 152 272 272 272 F F F 16 320-339 152 152 152 272 272 272 F F F 17 340-359 149 149 149 267 267 267 F F F 18 360-379 155 155 155 276 276 276 F F F 19 380-399 151 151 151 271 271 271 F F F 20 400-419 155 155 155 276 276 276 F F F 21 420-439 155 155 155 276 276 276 F F F 22 440-459 152 152 152 272 272 272 F F F 23 460-479 155 155 155 276 276 276 F F F 24 480-499 147 147 147 265 265 265 F F F 25 500-519 152 152 152 272 272 272 F F F 26 520-539 155 155 155 276 276 276 F F F 27 540-559 152 152 152 273 273 273 F F F 28 560-579 155 155 155 276 276 276 F F F 29 580-599 155 155 155 276 276 276 F F F Max gen: 155, Max flush: 276 Begin loop 78 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:33.449300024Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1804,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:33.494910035Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1804,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:33.495599361Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1804} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"0fd98d0d5d909611f8b408ccda2559d35fe117bb\",\n git_commit_timestamp: \"2024-11-08T20:05:44.000000000Z\",\n git_branch: \"mkeeter/drop-skipped-backpressure-guard\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:33.49563467Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1804} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:33.495648629Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1804} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:33.495662788Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1804} {"msg":"Crucible 0b67cd5d-3c06-4b88-a2cc-fc98e1ed72a5 has session id: 0bce4a40-b023-446b-8f64-98b50caf101f","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:33.498268931Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1804,"session_id":"0bce4a40-b023-446b-8f64-98b50caf101f"} {"msg":"Upstairs opts: Upstairs UUID: 0b67cd5d-3c06-4b88-a2cc-fc98e1ed72a5, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:33.498305289Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1804,"session_id":"0bce4a40-b023-446b-8f64-98b50caf101f"} {"msg":"Crucible stats registered with UUID: 0b67cd5d-3c06-4b88-a2cc-fc98e1ed72a5","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:33.498320858Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1804,"session_id":"0bce4a40-b023-446b-8f64-98b50caf101f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:33.530007239Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1804,"task":"crutest"} {"msg":"The guest has requested activation with gen:156","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:33.530036778Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1804} {"msg":"0b67cd5d-3c06-4b88-a2cc-fc98e1ed72a5 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:33.530112505Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1804,"session_id":"0bce4a40-b023-446b-8f64-98b50caf101f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:33.530135243Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1804,"client":"0","":"downstairs","session_id":"0bce4a40-b023-446b-8f64-98b50caf101f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:33.530163412Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1804,"client":"0","":"downstairs","session_id":"0bce4a40-b023-446b-8f64-98b50caf101f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:33.530181531Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1804,"client":"1","":"downstairs","session_id":"0bce4a40-b023-446b-8f64-98b50caf101f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:33.53020222Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1804,"client":"1","":"downstairs","session_id":"0bce4a40-b023-446b-8f64-98b50caf101f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:33.530219059Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1804,"client":"2","":"downstairs","session_id":"0bce4a40-b023-446b-8f64-98b50caf101f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:33.530235688Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1804,"client":"2","":"downstairs","session_id":"0bce4a40-b023-446b-8f64-98b50caf101f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:33.530251868Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1804,"":"io task","client":"2","":"downstairs","session_id":"0bce4a40-b023-446b-8f64-98b50caf101f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:33.530268927Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1804,"":"io task","client":"0","":"downstairs","session_id":"0bce4a40-b023-446b-8f64-98b50caf101f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:33.530285736Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1804,"":"io task","client":"1","":"downstairs","session_id":"0bce4a40-b023-446b-8f64-98b50caf101f"} {"msg":"ds_connection connected from Ok(127.0.0.1:60034)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:33.53041302Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1804,"":"io task","client":"1","":"downstairs","session_id":"0bce4a40-b023-446b-8f64-98b50caf101f"} {"msg":"ds_connection connected from Ok(127.0.0.1:38699)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:33.530503415Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1804,"":"io task","client":"0","":"downstairs","session_id":"0bce4a40-b023-446b-8f64-98b50caf101f"} {"msg":"ds_connection connected from Ok(127.0.0.1:54025)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:33.530534014Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1804,"":"io task","client":"2","":"downstairs","session_id":"0bce4a40-b023-446b-8f64-98b50caf101f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:33.531210961Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1804,"client":"2","":"downstairs","session_id":"0bce4a40-b023-446b-8f64-98b50caf101f"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:33.531239479Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1804,"client":"2","":"downstairs","session_id":"0bce4a40-b023-446b-8f64-98b50caf101f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:33.531254969Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1804,"client":"1","":"downstairs","session_id":"0bce4a40-b023-446b-8f64-98b50caf101f"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:33.531269298Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1804,"client":"1","":"downstairs","session_id":"0bce4a40-b023-446b-8f64-98b50caf101f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:33.531282887Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1804,"client":"0","":"downstairs","session_id":"0bce4a40-b023-446b-8f64-98b50caf101f"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:33.531304956Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1804,"client":"0","":"downstairs","session_id":"0bce4a40-b023-446b-8f64-98b50caf101f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:33.531877588Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1804,"client":"0","":"downstairs","session_id":"0bce4a40-b023-446b-8f64-98b50caf101f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:33.531918196Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1804,"client":"2","":"downstairs","session_id":"0bce4a40-b023-446b-8f64-98b50caf101f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:33.531938655Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1804,"client":"1","":"downstairs","session_id":"0bce4a40-b023-446b-8f64-98b50caf101f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:33.532433181Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1804,"client":"0","":"downstairs","session_id":"0bce4a40-b023-446b-8f64-98b50caf101f"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:33.532462919Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1804,"client":"0","":"downstairs","session_id":"0bce4a40-b023-446b-8f64-98b50caf101f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:33.532480209Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1804,"session_id":"0bce4a40-b023-446b-8f64-98b50caf101f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:33.532495508Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1804,"client":"2","":"downstairs","session_id":"0bce4a40-b023-446b-8f64-98b50caf101f"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:33.532512067Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1804,"client":"2","":"downstairs","session_id":"0bce4a40-b023-446b-8f64-98b50caf101f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:33.532528246Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1804,"session_id":"0bce4a40-b023-446b-8f64-98b50caf101f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:33.532543306Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1804,"client":"1","":"downstairs","session_id":"0bce4a40-b023-446b-8f64-98b50caf101f"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:33.532559905Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1804,"client":"1","":"downstairs","session_id":"0bce4a40-b023-446b-8f64-98b50caf101f"} {"msg":"[0]R flush_numbers[0..12]: [276, 276, 276, 276, 276, 276, 276, 276, 274, 267, 265, 276]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:33.532577054Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1804,"":"downstairs","":"downstairs","session_id":"0bce4a40-b023-446b-8f64-98b50caf101f"} {"msg":"[0]R generation[0..12]: [155, 155, 155, 155, 155, 155, 155, 155, 153, 149, 147, 155]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:33.532594363Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1804,"":"downstairs","":"downstairs","session_id":"0bce4a40-b023-446b-8f64-98b50caf101f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:33.532611572Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1804,"":"downstairs","":"downstairs","session_id":"0bce4a40-b023-446b-8f64-98b50caf101f"} {"msg":"[1]R flush_numbers[0..12]: [276, 276, 276, 276, 276, 276, 276, 276, 274, 267, 265, 276]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:33.532628962Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1804,"":"downstairs","":"downstairs","session_id":"0bce4a40-b023-446b-8f64-98b50caf101f"} {"msg":"[1]R generation[0..12]: [155, 155, 155, 155, 155, 155, 155, 155, 153, 149, 147, 155]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:33.53264941Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1804,"":"downstairs","":"downstairs","session_id":"0bce4a40-b023-446b-8f64-98b50caf101f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:33.532667249Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1804,"":"downstairs","":"downstairs","session_id":"0bce4a40-b023-446b-8f64-98b50caf101f"} {"msg":"[2]R flush_numbers[0..12]: [276, 276, 276, 276, 276, 276, 276, 276, 274, 267, 265, 276]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:33.532684399Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1804,"":"downstairs","":"downstairs","session_id":"0bce4a40-b023-446b-8f64-98b50caf101f"} {"msg":"[2]R generation[0..12]: [155, 155, 155, 155, 155, 155, 155, 155, 153, 149, 147, 155]"Wait for a query_work_queue command to finish before sending IO ,"v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:33.532710438Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1804,"":"downstairs","":"downstairs","session_id":"0bce4a40-b023-446b-8f64-98b50caf101f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:33.532728167Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1804,"":"downstairs","":"downstairs","session_id":"0bce4a40-b023-446b-8f64-98b50caf101f"} {"msg":"Max found gen is 156","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:33.532744536Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1804,"":"downstairs","":"downstairs","session_id":"0bce4a40-b023-446b-8f64-98b50caf101f"} {"msg":"Generation requested: 156 >= found:156","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:33.532760895Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1804,"":"downstairs","":"downstairs","session_id":"0bce4a40-b023-446b-8f64-98b50caf101f"} {"msg":"Next flush: 277","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:33.532776914Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1804,"":"downstairs","":"downstairs","session_id":"0bce4a40-b023-446b-8f64-98b50caf101f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:33.532792963Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1804,"":"downstairs","":"downstairs","session_id":"0bce4a40-b023-446b-8f64-98b50caf101f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:33.532809243Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1804,"session_id":"0bce4a40-b023-446b-8f64-98b50caf101f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:33.532824102Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1804,"client":"0","":"downstairs","session_id":"0bce4a40-b023-446b-8f64-98b50caf101f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:33.532840421Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1804,"client":"1","":"downstairs","session_id":"0bce4a40-b023-446b-8f64-98b50caf101f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:33.53285659Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1804,"client":"2","":"downstairs","session_id":"0bce4a40-b023-446b-8f64-98b50caf101f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:33.53287305Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1804,"session_id":"0bce4a40-b023-446b-8f64-98b50caf101f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:33.532888349Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1804,"session_id":"0bce4a40-b023-446b-8f64-98b50caf101f"} {"msg":"0b67cd5d-3c06-4b88-a2cc-fc98e1ed72a5 is now active with session: 0bce4a40-b023-446b-8f64-98b50caf101f","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:33.532906168Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1804,"session_id":"0bce4a40-b023-446b-8f64-98b50caf101f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:33.532921137Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1804,"session_id":"0bce4a40-b023-446b-8f64-98b50caf101f"} {"msg":"The guest has finished waiting for activation with:156","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:33.532936167Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1804} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 254 len 8192 data: 15 14 02/30 Write block 536 len 8192 data: 13 11 03/30 Read block 11 len 4096 04/30 Read block 31 len 36864 05/30 Read block 71 len 16384 06/30 Write block 353 len 16384 data: 19 17 18 17 07/30 Read block 216 len 8192 08/30 Write block 100 len 32768 data: 12 12 12 12 14 15 16 18 09/30 Write block 246 len 32768 data: 11 15 14 15 17 14 17 16 10/30 Write block 271 len 4096 data: 14 11/30 Read block 212 len 12288 12/30 Write block 431 len 28672 data: 18 16 15 14 17 17 16 13/30 Read block 82 len 40960 14/30 Read block 201 len 24576 15/30 Write block 219 len 8192 data: 14 13 16/30 Write block 106 len 8192 data: 17 19 17/30 Flush 18/30 Write block 540 len 36864 data: 13 14 14 14 17 13 14 11 13 19/30 Write block 587 len 4096 data: 19 20/30 Write block 284 len 8192 data: 13 16 21/30 Flush 22/30 Write block 286 len 16384 data: 18 18 19 15 23/30 Read block 92 len 4096 24/30 Read block 297 len 12288 25/30 Read block 196 len 12288 26/30 Read block 450 len 20480 27/30 Write block 387 len 12288 data: 7 8 8 28/30 Write block 145 len 16384 data: 13 12 13 12 29/30 Write block 361 len 36864 data: 20 18 16 16 13 12 15 14 13 30/30 Write block 419 len 40960 data: 12 12 13 13 12 12 12 11 11 11 ---------------------------------------------------------------- Crucible gen:156 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 4 Done Done Done false 1022 Acked Read 1 Done Done Done false 1023 Acked Read 3 Done Done Done false 1024 Acked Read 3 Done Done Done false 1025 Acked Read 5 Done Done Done false 1026 Acked Write 3 Done Done Done false 1027 Acked Write 4 Done Done Done false 1028 Acked Write 9 Sent Sent Sent false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 7 7 7 21 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1020 1020 1020 Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:28:33.556126103Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1804} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 155 155 155 276 276 276 F F F 1 020-039 155 155 155 276 276 276 F F F 2 040-059 155 155 155 276 276 276 F F F 3 060-079 155 155 155 276 276 276 F F F 4 080-099 155 155 155 276 276 276 F F F 5 100-119 156 156 156 277 277 277 F F F 6 120-139 155 155 155 276 276 276 F F F 7 140-159 155 155 155 276 276 276 T T T 8 160-179 153 153 153 274 274 274 F F F 9 180-199 149 149 149 267 267 267 F F F 10 200-219 156 156 156 277 277 277 F F F 11 220-239 156 156 156 277 277 277 F F F 12 240-259 156 156 156 277 277 277 F F F 13 260-279 156 156 156 277 277 277 F F F 14 280-299 156 156 156 278 278 278 T T T 15 300-319 152 152 152 272 272 272 F F F 16 320-339 152 152 152 272 272 272 F F F 17 340-359 156 156 156 277 277 277 F F F 18 360-379 155 155 155 276 276 276 T T T 19 380-399 151 151 151 271 271 271 T T T 20 400-419 155 155 155 276 276 276 T T T 21 420-439 156 156 156 277 277 277 T T T 22 440-459 152 152 152 272 272 272 F F F 23 460-479 155 155 155 276 276 276 F F F 24 480-499 147 147 147 265 265 265 F F F 25 500-519 152 152 152 272 272 272 F F F 26 520-539 156 156 156 277 277 277 F F F 27 540-559 156 156 156 278 278 278 F F F 28 560-579 155 155 155 276 276 276 F F F 29 580-599 156 156 156 278 278 278 F F F Max gen: 156, Max flush: 278 A Difference in extent metadata is expected here On loop 78 Verifying data now No change after verify Loop: 78 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 155 155 155 276 276 276 F F F 1 020-039 155 155 155 276 276 276 F F F 2 040-059 155 155 155 276 276 276 F F F 3 060-079 155 155 155 276 276 276 F F F 4 080-099 155 155 155 276 276 276 F F F 5 100-119 156 156 156 277 277 277 F F F 6 120-139 155 155 155 276 276 276 F F F 7 140-159 157 157 157 279 279 279 F F F 8 160-179 153 153 153 274 274 274 F F F 9 180-199 149 149 149 267 267 267 F F F 10 200-219 156 156 156 277 277 277 F F F 11 220-239 156 156 156 277 277 277 F F F 12 240-259 156 156 156 277 277 277 F F F 13 260-279 156 156 156 277 277 277 F F F 14 280-299 157 157 157 279 279 279 F F F 15 300-319 152 152 152 272 272 272 F F F 16 320-339 152 152 152 272 272 272 F F F 17 340-359 156 156 156 277 277 277 F F F 18 360-379 157 157 157 279 279 279 F F F 19 380-399 157 157 157 279 279 279 F F F 20 400-419 157 157 157 279 279 279 F F F 21 420-439 157 157 157 279 279 279 F F F 22 440-459 152 152 152 272 272 272 F F F 23 460-479 155 155 155 276 276 276 F F F 24 480-499 147 147 147 265 265 265 F F F 25 500-519 152 152 152 272 272 272 F F F 26 520-539 156 156 156 277 277 277 F F F 27 540-559 156 156 156 278 278 278 F F F 28 560-579 155 155 155 276 276 276 F F F 29 580-599 156 156 156 278 278 278 F F F Max gen: 157, Max flush: 279 Begin loop 79 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:37.894796294Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1812,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:28:37.937454859Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1812,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:37.93785122Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1812,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:37.969740471Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1812,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:37.970330762Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1812} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"0fd98d0d5d909611f8b408ccda2559d35fe117bb\",\n git_commit_timestamp: \"2024-11-08T20:05:44.000000000Z\",\n git_branch: \"mkeeter/drop-skipped-backpressure-guard\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:37.97036903Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1812} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:37.97038501Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1812} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:37.970402329Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1812} {"msg":"Crucible 22981986-f6c4-428e-a385-fa2d26c06329 has session id: 63e97124-a9ee-4f51-a883-9a37d7bee879","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:37.970906974Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1812,"session_id":"63e97124-a9ee-4f51-a883-9a37d7bee879"} {"msg":"Upstairs opts: Upstairs UUID: 22981986-f6c4-428e-a385-fa2d26c06329, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:37.970942502Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1812,"session_id":"63e97124-a9ee-4f51-a883-9a37d7bee879"} {"msg":"Crucible stats registered with UUID: 22981986-f6c4-428e-a385-fa2d26c06329","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:37.970958731Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1812,"session_id":"63e97124-a9ee-4f51-a883-9a37d7bee879"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:38.002043142Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1812,"task":"crutest"} {"msg":"The guest has requested activation with gen:158","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:38.002075571Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1812} {"msg":"22981986-f6c4-428e-a385-fa2d26c06329 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:38.002125528Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1812,"session_id":"63e97124-a9ee-4f51-a883-9a37d7bee879"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:38.002145397Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1812,"client":"0","":"downstairs","session_id":"63e97124-a9ee-4f51-a883-9a37d7bee879"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:38.002173306Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1812,"client":"0","":"downstairs","session_id":"63e97124-a9ee-4f51-a883-9a37d7bee879"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:38.002191525Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1812,"client":"1","":"downstairs","session_id":"63e97124-a9ee-4f51-a883-9a37d7bee879"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:38.002208744Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1812,"client":"1","":"downstairs","session_id":"63e97124-a9ee-4f51-a883-9a37d7bee879"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:38.002228743Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1812,"client":"2","":"downstairs","session_id":"63e97124-a9ee-4f51-a883-9a37d7bee879"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:38.002245342Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1812,"client":"2","":"downstairs","session_id":"63e97124-a9ee-4f51-a883-9a37d7bee879"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:38.002313919Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1812,"":"io task","client":"2","":"downstairs","session_id":"63e97124-a9ee-4f51-a883-9a37d7bee879"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:38.002343618Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1812,"":"io task","client":"0","":"downstairs","session_id":"63e97124-a9ee-4f51-a883-9a37d7bee879"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:38.002370296Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1812,"":"io task","client":"1","":"downstairs","session_id":"63e97124-a9ee-4f51-a883-9a37d7bee879"} {"msg":"ds_connection connected from Ok(127.0.0.1:38846)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:38.002408185Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1812,"":"io task","client":"1","":"downstairs","session_id":"63e97124-a9ee-4f51-a883-9a37d7bee879"} {"msg":"ds_connection connected from Ok(127.0.0.1:49480)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:38.002620844Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1812,"":"io task","client":"2","":"downstairs","session_id":"63e97124-a9ee-4f51-a883-9a37d7bee879"} {"msg":"ds_connection connected from Ok(127.0.0.1:37698)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:38.002673761Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1812,"":"io task","client":"0","":"downstairs","session_id":"63e97124-a9ee-4f51-a883-9a37d7bee879"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:38.003285602Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1812,"client":"2","":"downstairs","session_id":"63e97124-a9ee-4f51-a883-9a37d7bee879"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:38.00331339Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1812,"client":"2","":"downstairs","session_id":"63e97124-a9ee-4f51-a883-9a37d7bee879"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:38.003334709Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1812,"client":"1","":"downstairs","session_id":"63e97124-a9ee-4f51-a883-9a37d7bee879"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:38.003349508Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1812,"client":"1","":"downstairs","session_id":"63e97124-a9ee-4f51-a883-9a37d7bee879"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:38.003379847Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1812,"client":"0","":"downstairs","session_id":"63e97124-a9ee-4f51-a883-9a37d7bee879"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:38.003400646Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1812,"client":"0","":"downstairs","session_id":"63e97124-a9ee-4f51-a883-9a37d7bee879"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:38.003867873Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1812,"client":"1","":"downstairs","session_id":"63e97124-a9ee-4f51-a883-9a37d7bee879"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:38.003901001Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1812,"client":"2","":"downstairs","session_id":"63e97124-a9ee-4f51-a883-9a37d7bee879"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:38.004060624Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1812,"client":"0","":"downstairs","session_id":"63e97124-a9ee-4f51-a883-9a37d7bee879"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:38.004453494Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1812,"client":"1","":"downstairs","session_id":"63e97124-a9ee-4f51-a883-9a37d7bee879"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:38.004479893Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1812,"client":"1","":"downstairs","session_id":"63e97124-a9ee-4f51-a883-9a37d7bee879"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:38.004494312Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1812,"session_id":"63e97124-a9ee-4f51-a883-9a37d7bee879"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:38.004578399Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1812,"client":"0","":"downstairs","session_id":"63e97124-a9ee-4f51-a883-9a37d7bee879"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:38.004598758Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1812,"client":"0","":"downstairs","session_id":"63e97124-a9ee-4f51-a883-9a37d7bee879"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:38.004612467Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1812,"session_id":"63e97124-a9ee-4f51-a883-9a37d7bee879"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:38.004624686Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1812,"client":"2","":"downstairs","session_id":"63e97124-a9ee-4f51-a883-9a37d7bee879"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:38.004638016Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1812,"client":"2","":"downstairs","session_id":"63e97124-a9ee-4f51-a883-9a37d7bee879"} {"msg":"[0]R flush_numbers[0..12]: [276, 276, 276, 276, 276, 277, 276, 279, 274, 267, 277, 277]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:38.004651995Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1812,"":"downstairs","":"downstairs","session_id":"63e97124-a9ee-4f51-a883-9a37d7bee879"} {"msg":"[0]R generation[0..12]: [155, 155, 155, 155, 155, 156, 155, 157, 153, 149, 156, 156]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:38.004666104Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1812,"":"downstairs","":"downstairs","session_id":"63e97124-a9ee-4f51-a883-9a37d7bee879"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:38.004680243Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1812,"":"downstairs","":"downstairs","session_id":"63e97124-a9ee-4f51-a883-9a37d7bee879"} {"msg":"[1]R flush_numbers[0..12]: [276, 276, 276, 276, 276, 277, 276, 279, 274, 267, 277, 277]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:38.004694193Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1812,"":"downstairs","":"downstairs","session_id":"63e97124-a9ee-4f51-a883-9a37d7bee879"} {"msg":"[1]R generation[0..12]: [155, 155, 155, 155, 155, 156, 155, 157, 153, 149, 156, 156]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:38.004707782Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1812,"":"downstairs","":"downstairs","session_id":"63e97124-a9ee-4f51-a883-9a37d7bee879"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:38.004724471Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1812,"":"downstairs","":"downstairs","session_id":"63e97124-a9ee-4f51-a883-9a37d7bee879"} {"msg":"[2]R flush_numbers[0..12]: [276, 276, 276, 276, 276, 277, 276, 279, 274, 267, 277, 277]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:38.004738541Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1812,"":"downstairs","":"downstairs","session_id":"63e97124-a9ee-4f51-a883-9a37d7bee879"} {"msg":"[2]R generation[0..12]: [155, 155, 155, 155, 155, 156, 155, 157, 153, 149, 156, 156]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:38.00475234Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1812,"":"downstairs","":"downstairs","session_id":"63e97124-a9ee-4f51-a883-9a37d7bee879"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:38.004766589Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1812,"":"downstairs","":"downstairs","session_id":"63e97124-a9ee-4f51-a883-9a37d7bee879"} {"msg":"Max found gen is 158","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:38.004779659Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1812,"":"downstairs","":"downstairs","session_id":"63e97124-a9ee-4f51-a883-9a37d7bee879"} {"msg":"Generation requested: 158 >= found:158","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:38.004792798Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1812,"":"downstairs","":"downstairs","session_id":"63e97124-a9ee-4f51-a883-9a37d7bee879"} {"msg":"Next flush: 280","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:38.004805847Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1812,"":"downstairs","":"downstairs","session_id":"63e97124-a9ee-4f51-a883-9a37d7bee879"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:38.004819117Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1812,"":"downstairs","":"downstairs","session_id"Wait for a query_work_queue command to finish before sending IO :"63e97124-a9ee-4f51-a883-9a37d7bee879"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:38.004897843Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1812,"session_id":"63e97124-a9ee-4f51-a883-9a37d7bee879"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:38.004923072Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1812,"client":"0","":"downstairs","session_id":"63e97124-a9ee-4f51-a883-9a37d7bee879"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:38.004939171Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1812,"client":"1","":"downstairs","session_id":"63e97124-a9ee-4f51-a883-9a37d7bee879"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:38.00495316Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1812,"client":"2","":"downstairs","session_id":"63e97124-a9ee-4f51-a883-9a37d7bee879"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:38.004967029Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1812,"session_id":"63e97124-a9ee-4f51-a883-9a37d7bee879"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:38.004979559Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1812,"session_id":"63e97124-a9ee-4f51-a883-9a37d7bee879"} {"msg":"22981986-f6c4-428e-a385-fa2d26c06329 is now active with session: 63e97124-a9ee-4f51-a883-9a37d7bee879","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:38.004992708Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1812,"session_id":"63e97124-a9ee-4f51-a883-9a37d7bee879"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:38.005009557Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1812,"session_id":"63e97124-a9ee-4f51-a883-9a37d7bee879"} {"msg":"The guest has finished waiting for activation with:158","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:38.005022967Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1812} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 257 len 32768 data: 10 10 10 8 8 9 10 11 02/30 Read block 99 len 4096 03/30 Read block 418 len 28672 04/30 Read block 18 len 28672 05/30 Write block 52 len 8192 data: 8 9 06/30 Flush 07/30 Write block 70 len 36864 data: 11 10 11 12 12 11 12 12 11 08/30 Read block 400 len 16384 09/30 Write block 412 len 4096 data: 14 10/30 Write block 242 len 20480 data: 15 13 13 11 12 11/30 Flush 12/30 Write block 539 len 24576 data: 12 14 15 15 15 18 13/30 Read block 351 len 20480 14/30 Read block 372 len 24576 15/30 Write block 211 len 24576 data: 9 11 12 11 12 13 16/30 Flush 17/30 Write block 105 len 36864 data: 16 18 20 16 15 14 10 10 10 18/30 Read block 583 len 40960 19/30 Read block 399 len 24576 20/30 Write block 58 len 32768 data: 10 7 9 10 10 11 11 12 21/30 Write block 246 len 28672 data: 13 16 15 16 18 15 18 22/30 Read block 508 len 12288 23/30 Read block 2 len 36864 24/30 Read block 284 len 36864 25/30 Read block 438 len 16384 26/30 Read block 375 len 36864 27/30 Read block 110 len 12288 28/30 Read block 391 len 16384 29/30 Read block 31 len 12288 30/30 Read block 420 len 8192 ---------------------------------------------------------------- Crucible gen:158 GIO:true work queues: Upstairs:0 downstairs:14 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1016 Acked Write 9 Done Done Done false 1017 Acked Read 10 Done Done Done false 1018 Acked Read 6 Done Done Done false 1019 Acked Write 8 Done Done Done false 1020 Acked Write 7 Done Done Done false 1021 Acked Read 3 Done Done Done false 1022 Acked Read 9 Done Done Done false 1023 Acked Read 9 Done Done Done false 1024 Acked Read 4 Done Done Done false 1025 Acked Read 9 Done Done Done false 1026 Acked Read 3 Done Done Done false 1027 Acked Read 4 Done Done Done false 1028 Acked Read 3 Done Done Done false 1029 Acked Read 2 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 14 14 14 42 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1015 1015 1015 Downstairs last five completed: 1015 1014 1013 1012 1011 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:28:38.03258071Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1812} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 155 155 155 276 276 276 F F F 1 020-039 155 155 155 276 276 276 F F F 2 040-059 158 158 158 280 280 280 T T T 3 060-079 158 158 158 281 281 281 T T T 4 080-099 155 155 155 276 276 276 F F F 5 100-119 156 156 156 277 277 277 T T T 6 120-139 155 155 155 276 276 276 F F F 7 140-159 157 157 157 279 279 279 F F F 8 160-179 153 153 153 274 274 274 F F F 9 180-199 149 149 149 267 267 267 F F F 10 200-219 158 158 158 282 282 282 F F F 11 220-239 156 156 156 277 277 277 F F F 12 240-259 158 158 158 281 281 281 T T T 13 260-279 158 158 158 280 280 280 F F F 14 280-299 157 157 157 279 279 279 F F F 15 300-319 152 152 152 272 272 272 F F F 16 320-339 152 152 152 272 272 272 F F F 17 340-359 156 156 156 277 277 277 F F F 18 360-379 157 157 157 279 279 279 F F F 19 380-399 157 157 157 279 279 279 F F F 20 400-419 158 158 158 281 281 281 F F F 21 420-439 157 157 157 279 279 279 F F F 22 440-459 152 152 152 272 272 272 F F F 23 460-479 155 155 155 276 276 276 F F F 24 480-499 147 147 147 265 265 265 F F F 25 500-519 152 152 152 272 272 272 F F F 26 520-539 158 158 158 282 282 282 F F F 27 540-559 158 158 158 282 282 282 F F F 28 560-579 155 155 155 276 276 276 F F F 29 580-599 156 156 156 278 278 278 F F F Max gen: 158, Max flush: 282 A Difference in extent metadata is expected here On loop 79 Verifying data now No change after verify Loop: 79 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 155 155 155 276 276 276 F F F 1 020-039 155 155 155 276 276 276 F F F 2 040-059 159 159 159 283 283 283 F F F 3 060-079 159 159 159 283 283 283 F F F 4 080-099 155 155 155 276 276 276 F F F 5 100-119 159 159 159 283 283 283 F F F 6 120-139 155 155 155 276 276 276 F F F 7 140-159 157 157 157 279 279 279 F F F 8 160-179 153 153 153 274 274 274 F F F 9 180-199 149 149 149 267 267 267 F F F 10 200-219 158 158 158 282 282 282 F F F 11 220-239 156 156 156 277 277 277 F F F 12 240-259 159 159 159 283 283 283 F F F 13 260-279 158 158 158 280 280 280 F F F 14 280-299 157 157 157 279 279 279 F F F 15 300-319 152 152 152 272 272 272 F F F 16 320-339 152 152 152 272 272 272 F F F 17 340-359 156 156 156 277 277 277 F F F 18 360-379 157 157 157 279 279 279 F F F 19 380-399 157 157 157 279 279 279 F F F 20 400-419 158 158 158 281 281 281 F F F 21 420-439 157 157 157 279 279 279 F F F 22 440-459 152 152 152 272 272 272 F F F 23 460-479 155 155 155 276 276 276 F F F 24 480-499 147 147 147 265 265 265 F F F 25 500-519 152 152 152 272 272 272 F F F 26 520-539 158 158 158 282 282 282 F F F 27 540-559 158 158 158 282 282 282 F F F 28 560-579 155 155 155 276 276 276 F F F 29 580-599 156 156 156 278 278 278 F F F Max gen: 159, Max flush: 283 Begin loop 80 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:42.367827849Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1820,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:42.412150802Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1820,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:42.412753193Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1820} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"0fd98d0d5d909611f8b408ccda2559d35fe117bb\",\n git_commit_timestamp: \"2024-11-08T20:05:44.000000000Z\",\n git_branch: \"mkeeter/drop-skipped-backpressure-guard\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:42.412820459Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1820} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:42.412854798Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1820} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:42.412875647Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1820} {"msg":"Crucible 849d6cb3-ddff-48f2-9694-c9dd3e9288a7 has session id: c467b210-36c4-4881-a64a-cc9195a880fc","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:42.413284597Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1820,"session_id":"c467b210-36c4-4881-a64a-cc9195a880fc"} {"msg":"Upstairs opts: Upstairs UUID: 849d6cb3-ddff-48f2-9694-c9dd3e9288a7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:42.413331574Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1820,"session_id":"c467b210-36c4-4881-a64a-cc9195a880fc"} {"msg":"Crucible stats registered with UUID: 849d6cb3-ddff-48f2-9694-c9dd3e9288a7","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:42.413352984Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1820,"session_id":"c467b210-36c4-4881-a64a-cc9195a880fc"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:42.444264763Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1820,"task":"crutest"} {"msg":"The guest has requested activation with gen:160","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:42.444294231Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1820} {"msg":"849d6cb3-ddff-48f2-9694-c9dd3e9288a7 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:42.444369788Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1820,"session_id":"c467b210-36c4-4881-a64a-cc9195a880fc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:42.444391647Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1820,"client":"0","":"downstairs","session_id":"c467b210-36c4-4881-a64a-cc9195a880fc"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:42.444420275Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1820,"client":"0","":"downstairs","session_id":"c467b210-36c4-4881-a64a-cc9195a880fc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:42.444437824Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1820,"client":"1","":"downstairs","session_id":"c467b210-36c4-4881-a64a-cc9195a880fc"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:42.444454993Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1820,"client":"1","":"downstairs","session_id":"c467b210-36c4-4881-a64a-cc9195a880fc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:42.444476083Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1820,"client":"2","":"downstairs","session_id":"c467b210-36c4-4881-a64a-cc9195a880fc"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:42.444507021Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1820,"client":"2","":"downstairs","session_id":"c467b210-36c4-4881-a64a-cc9195a880fc"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:42.444544979Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1820,"":"io task","client":"2","":"downstairs","session_id":"c467b210-36c4-4881-a64a-cc9195a880fc"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:42.444580477Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1820,"":"io task","client":"0","":"downstairs","session_id":"c467b210-36c4-4881-a64a-cc9195a880fc"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:42.444612396Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1820,"":"io task","client":"1","":"downstairs","session_id":"c467b210-36c4-4881-a64a-cc9195a880fc"} {"msg":"ds_connection connected from Ok(127.0.0.1:46806)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:42.444647634Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1820,"":"io task","client":"2","":"downstairs","session_id":"c467b210-36c4-4881-a64a-cc9195a880fc"} {"msg":"ds_connection connected from Ok(127.0.0.1:59927)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:42.444743049Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1820,"":"io task","client":"1","":"downstairs","session_id":"c467b210-36c4-4881-a64a-cc9195a880fc"} {"msg":"ds_connection connected from Ok(127.0.0.1:55439)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:42.444779788Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1820,"":"io task","client":"0","":"downstairs","session_id":"c467b210-36c4-4881-a64a-cc9195a880fc"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:42.445483913Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1820,"client":"1","":"downstairs","session_id":"c467b210-36c4-4881-a64a-cc9195a880fc"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:42.445513002Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1820,"client":"1","":"downstairs","session_id":"c467b210-36c4-4881-a64a-cc9195a880fc"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:42.445659885Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1820,"client":"0","":"downstairs","session_id":"c467b210-36c4-4881-a64a-cc9195a880fc"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:42.445694763Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1820,"client":"0","":"downstairs","session_id":"c467b210-36c4-4881-a64a-cc9195a880fc"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:42.445770739Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1820,"client":"2","":"downstairs","session_id":"c467b210-36c4-4881-a64a-cc9195a880fc"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:42.445808387Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1820,"client":"2","":"downstairs","session_id":"c467b210-36c4-4881-a64a-cc9195a880fc"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:42.446091493Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1820,"client":"1","":"downstairs","session_id":"c467b210-36c4-4881-a64a-cc9195a880fc"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:42.446233186Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1820,"client":"0","":"downstairs","session_id":"c467b210-36c4-4881-a64a-cc9195a880fc"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:42.44657626Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1820,"client":"2","":"downstairs","session_id":"c467b210-36c4-4881-a64a-cc9195a880fc"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:42.446710473Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1820,"client":"1","":"downstairs","session_id":"c467b210-36c4-4881-a64a-cc9195a880fc"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:42.446740672Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1820,"client":"1","":"downstairs","session_id":"c467b210-36c4-4881-a64a-cc9195a880fc"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:42.446756161Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1820,"session_id":"c467b210-36c4-4881-a64a-cc9195a880fc"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:42.44677292Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1820,"client":"0","":"downstairs","session_id":"c467b210-36c4-4881-a64a-cc9195a880fc"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:42.446786559Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1820,"client":"0","":"downstairs","session_id":"c467b210-36c4-4881-a64a-cc9195a880fc"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:42.446799869Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1820,"session_id":"c467b210-36c4-4881-a64a-cc9195a880fc"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:42.447058296Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1820,"client":"2","":"downstairs","session_id":"c467b210-36c4-4881-a64a-cc9195a880fc"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:42.447078065Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1820,"client":"2","":"downstairs","session_id":"c467b210-36c4-4881-a64a-cc9195a880fc"} {"msg":"[0]R flush_numbers[0..12]: [276, 276, 283, 283, 276, 283, 276, 279, 274, 267, 282, 277]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:42.447092135Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1820,"":"downstairs","":"downstairs","session_id":"c467b210-36c4-4881-a64a-cc9195a880fc"} {"msg":"[0]R generation[0..12]: [155, 155, 159, 159, 155, 159, 155, 157, 153, 149, 158, 156]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:42.447105924Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1820,"":"downstairs","":"downstairs","session_id":"c467b210-36c4-4881-a64a-cc9195a880fc"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:42.447119793Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1820,"":"downstairs","":"downstairs","session_id":"c467b210-36c4-4881-a64a-cc9195a880fc"} {"msg":"[1]R flush_numbers[0..12]: [276, 276, 283, 283, 276, 283, 276, 279, 274, 267, 282, 277]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:42.447133553Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1820,"":"downstairs","":"downstairs","session_id":"c467b210-36c4-4881-a64a-cc9195a880fc"} {"msg":"[1]R generation[0..12]: [155, 155, 159, 159, 155, 159, 155, 157, 153, 149, 158, 156]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:42.447147192Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1820,"":"downstairs","":"downstairs","session_id":"c467b210-36c4-4881-a64a-cc9195a880fc"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:42.447161051Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1820,"":"downstairs","":"downstairs","session_id":"c467b210-36c4-4881-a64a-cc9195a880fc"} {"msg":"[2]R flush_numbers[0..12]: [276, 276, 283, 283, 276, 283, 276, 279, 274, 267, 282, 277]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:42.44717486Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1820,"":"downstairs","":"downstairs","session_id":"c467b210-36c4-4881-a64a-cc9195a880fc"} {"msg":"[2]R generation[0..12]: [155, 155, 159, 159, 155, 159, 155, 157, 153, 149, 158, 156]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:42.44719184Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1820,"":"downstairs","":"downstairs","session_id":"c467b210-36c4-4881-a64a-cc9195a880fc"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:42.447205949Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1820,"":"downstairs","":"downstairs","session_id":"c467b210-36c4-4881-a64a-cc9195a880fc"} {"msg":"Max found gen is 160","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:42.447218988Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1820,"":"downstairs","":"downstairs","session_id":"c467b210-36c4-4881-a64a-cc9195a880fc"} {"msg":"Generation requested: 160 >= found:160","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:42.447232088Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1820,"":"downstairs","":"downstairs","session_id":"c467b210-36c4-4881-a64a-cc9195a880fc"} {"msg":"Next flush: 284","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:42.447244937Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1820,"":"downstairs","":"downstairs","session_id":"c467b210-36c4-4881-a64a-cc9195a880fc"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:42.447257676Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1820,"":"downstairs","":"downstairs","session_id":"c467b210-36c4-4881-a64a-cc9195a880fc"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-08T20:28:42.447270686Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1820,"session_id":"c467b210-36c4-4881-a64a-cc9195a880fc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:42.447288345Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1820,"client":"0","":"downstairs","session_id":"c467b210-36c4-4881-a64a-cc9195a880fc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:42.447301444Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1820,"client":"1","":"downstairs","session_id":"c467b210-36c4-4881-a64a-cc9195a880fc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:42.447314454Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1820,"client":"2","":"downstairs","session_id":"c467b210-36c4-4881-a64a-cc9195a880fc"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:42.447327533Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1820,"session_id":"c467b210-36c4-4881-a64a-cc9195a880fc"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:42.447339743Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1820,"session_id":"c467b210-36c4-4881-a64a-cc9195a880fc"} {"msg":"849d6cb3-ddff-48f2-9694-c9dd3e9288a7 is now active with session: c467b210-36c4-4881-a64a-cc9195a880fc","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:42.447352612Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1820,"session_id":"c467b210-36c4-4881-a64a-cc9195a880fc"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:42.447364791Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1820,"session_id":"c467b210-36c4-4881-a64a-cc9195a880fc"} {"msg":"The guest has finished waiting for activation with:160","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:42.447377001Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1820} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 215 len 20480 data: 13 14 12 15 15 02/30 Write block 434 len 12288 data: 15 18 18 03/30 Read block 585 len 32768 04/30 Write block 365 len 36864 data: 14 13 16 15 14 12 12 14 11 05/30 Read block 298 len 8192 06/30 Write block 317 len 4096 data: 11 07/30 Read block 350 len 40960 08/30 Write block 320 len 20480 data: 16 12 13 14 11 09/30 Flush 10/30 Write block 407 len 28672 data: 11 12 12 16 15 15 13 11/30 Flush 12/30 Write block 265 len 36864 data: 14 13 10 14 14 15 15 15 16 13/30 Read block 26 len 24576 14/30 Write block 297 len 16384 data: 12 12 16 15 15/30 Flush 16/30 Write block 474 len 8192 data: 17 19 17/30 Read block 424 len 20480 18/30 Read block 36 len 8192 19/30 Flush 20/30 Write block 526 len 40960 data: 8 10 11 9 9 14 12 12 11 13 21/30 Write block 387 len 4096 data: 8 22/30 Read block 20 len 40960 23/30 Read block 535 len 40960 24/30 Write block 256 len 16384 data: 9 11 11 11 25/30 Write block 233 len 24576 data: 10 8 10 11 10 13 26/30 Write block 511 len 12288 data: 14 12 12 27/30 Write block 533 len 32768 data: 13 12 14 14 12 13 13 15 28/30 Read block 306 len 24576 29/30 Write block 461 len 32768 data: 10 10 10 7 9 12 14 16 30/30 Read block 94 len 32768 ---------------------------------------------------------------- Crucible gen:160 GIO:true work queues: Upstairs:0 downstairs:11 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1019 Acked Write 10 Done Done Done false 1020 Acked Write 1 Done Done Done false 1021 Acked Read 10 Done Done Done false 1022 Acked Read 10 Done Done Done false 1023 Acked Write 4 Done Done Done false 1024 Acked Write 6 Done Done Done false 1025 Acked Write 3 Done Done Done false 1026 Acked Write 8 Done Done Done false 1027 Acked Read 6 Done Done Done false 1028 Acked Write 8 Done Done Sent false 1029 Acked Read 8 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 2 3 Done 11 10 9 30 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1018 1018 1018 Downstairs last five completed: 1018 1017 1016 1015 1014 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:28:42.473075085Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1820} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 155 155 155 276 276 276 F F F 1 020-039 155 155 155 276 276 276 F F F 2 040-059 159 159 159 283 283 283 F F F 3 060-079 159 159 159 283 283 283 F F F 4 080-099 155 155 155 276 276 276 F F F 5 100-119 159 159 159 283 283 283 F F F 6 120-139 155 155 155 276 276 276 F F F 7 140-159 157 157 157 279 279 279 F F F 8 160-179 153 153 153 274 274 274 F F F 9 180-199 149 149 149 267 267 267 F F F 10 200-219 160 160 160 284 284 284 F F F 11 220-239 156 156 156 277 277 277 T T T 12 240-259 159 159 159 283 283 283 T T T 13 260-279 160 160 160 286 286 286 F F F 14 280-299 160 160 160 286 286 286 F F F 15 300-319 160 160 160 286 286 286 F F F 16 320-339 160 160 160 284 284 284 F F F 17 340-359 156 156 156 277 277 277 F F F 18 360-379 160 160 160 284 284 284 F F F 19 380-399 157 157 157 279 279 279 T T T 20 400-419 160 160 160 285 285 285 F F F 21 420-439 160 160 160 284 284 284 F F F 22 440-459 152 152 152 272 272 272 F F F 23 460-479 160 160 160 287 287 287 T T T 24 480-499 147 147 147 265 265 265 F F F 25 500-519 152 152 152 272 272 272 T T T 26 520-539 158 158 158 282 282 282 T T T 27 540-559 158 158 158 282 282 282 T T T 28 560-579 155 155 155 276 276 276 F F F 29 580-599 156 156 156 278 278 278 F F F Max gen: 160, Max flush: 287 A Difference in extent metadata is expected here On loop 80 Verifying data now No change after verify Loop: 80 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 155 155 155 276 276 276 F F F 1 020-039 155 155 155 276 276 276 F F F 2 040-059 159 159 159 283 283 283 F F F 3 060-079 159 159 159 283 283 283 F F F 4 080-099 155 155 155 276 276 276 F F F 5 100-119 159 159 159 283 283 283 F F F 6 120-139 155 155 155 276 276 276 F F F 7 140-159 157 157 157 279 279 279 F F F 8 160-179 153 153 153 274 274 274 F F F 9 180-199 149 149 149 267 267 267 F F F 10 200-219 160 160 160 284 284 284 F F F 11 220-239 161 161 161 288 288 288 F F F 12 240-259 161 161 161 288 288 288 F F F 13 260-279 160 160 160 286 286 286 F F F 14 280-299 160 160 160 286 286 286 F F F 15 300-319 160 160 160 286 286 286 F F F 16 320-339 160 160 160 284 284 284 F F F 17 340-359 156 156 156 277 277 277 F F F 18 360-379 160 160 160 284 284 284 F F F 19 380-399 161 161 161 288 288 288 F F F 20 400-419 160 160 160 285 285 285 F F F 21 420-439 160 160 160 284 284 284 F F F 22 440-459 152 152 152 272 272 272 F F F 23 460-479 161 161 161 288 288 288 F F F 24 480-499 147 147 147 265 265 265 F F F 25 500-519 161 161 161 288 288 288 F F F 26 520-539 161 161 161 288 288 288 F F F 27 540-559 161 161 161 288 288 288 F F F 28 560-579 155 155 155 276 276 276 F F F 29 580-599 156 156 156 278 278 278 F F F Max gen: 161, Max flush: 288 Begin loop 81 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:46.828972292Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1828,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:28:46.872953062Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1828,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:46.873355012Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1828,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:46.906291043Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1828,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:46.906862075Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1828} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"0fd98d0d5d909611f8b408ccda2559d35fe117bb\",\n git_commit_timestamp: \"2024-11-08T20:05:44.000000000Z\",\n git_branch: \"mkeeter/drop-skipped-backpressure-guard\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:46.906892933Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1828} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:46.906910472Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1828} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:46.906925082Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1828} {"msg":"Crucible 7ee3905e-f6e0-4426-98dc-3770c7a00e2a has session id: e3a86a50-b5a7-4628-8cbb-a1a3cdecac8d","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:46.907398408Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1828,"session_id":"e3a86a50-b5a7-4628-8cbb-a1a3cdecac8d"} {"msg":"Upstairs opts: Upstairs UUID: 7ee3905e-f6e0-4426-98dc-3770c7a00e2a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:46.907432207Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1828,"session_id":"e3a86a50-b5a7-4628-8cbb-a1a3cdecac8d"} {"msg":"Crucible stats registered with UUID: 7ee3905e-f6e0-4426-98dc-3770c7a00e2a","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:46.907447586Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1828,"session_id":"e3a86a50-b5a7-4628-8cbb-a1a3cdecac8d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:46.93908448Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1828,"task":"crutest"} {"msg":"The guest has requested activation with gen:162","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:46.939119438Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1828} {"msg":"7ee3905e-f6e0-4426-98dc-3770c7a00e2a active request set","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:46.939194494Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1828,"session_id":"e3a86a50-b5a7-4628-8cbb-a1a3cdecac8d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:46.939217284Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1828,"client":"0","":"downstairs","session_id":"e3a86a50-b5a7-4628-8cbb-a1a3cdecac8d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:46.939245572Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1828,"client":"0","":"downstairs","session_id":"e3a86a50-b5a7-4628-8cbb-a1a3cdecac8d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:46.939263621Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1828,"client":"1","":"downstairs","session_id":"e3a86a50-b5a7-4628-8cbb-a1a3cdecac8d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:46.93928063Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1828,"client":"1","":"downstairs","session_id":"e3a86a50-b5a7-4628-8cbb-a1a3cdecac8d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:46.939297229Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1828,"client":"2","":"downstairs","session_id":"e3a86a50-b5a7-4628-8cbb-a1a3cdecac8d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:46.939313929Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1828,"client":"2","":"downstairs","session_id":"e3a86a50-b5a7-4628-8cbb-a1a3cdecac8d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:46.939330218Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1828,"":"io task","client":"2","":"downstairs","session_id":"e3a86a50-b5a7-4628-8cbb-a1a3cdecac8d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:46.939351487Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1828,"":"io task","client":"0","":"downstairs","session_id":"e3a86a50-b5a7-4628-8cbb-a1a3cdecac8d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:46.939368936Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1828,"":"io task","client":"1","":"downstairs","session_id":"e3a86a50-b5a7-4628-8cbb-a1a3cdecac8d"} {"msg":"ds_connection connected from Ok(127.0.0.1:33581)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:46.939576906Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1828,"":"io task","client":"0","":"downstairs","session_id":"e3a86a50-b5a7-4628-8cbb-a1a3cdecac8d"} {"msg":"ds_connection connected from Ok(127.0.0.1:46725)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:46.939626104Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1828,"":"io task","client":"2","":"downstairs","session_id":"e3a86a50-b5a7-4628-8cbb-a1a3cdecac8d"} {"msg":"ds_connection connected from Ok(127.0.0.1:34449)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:46.939661092Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1828,"":"io task","client":"1","":"downstairs","session_id":"e3a86a50-b5a7-4628-8cbb-a1a3cdecac8d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:46.940480182Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1828,"client":"1","":"downstairs","session_id":"e3a86a50-b5a7-4628-8cbb-a1a3cdecac8d"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:46.94051157Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1828,"client":"1","":"downstairs","session_id":"e3a86a50-b5a7-4628-8cbb-a1a3cdecac8d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:46.940526829Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1828,"client":"2","":"downstairs","session_id":"e3a86a50-b5a7-4628-8cbb-a1a3cdecac8d"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:46.940540629Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1828,"client":"2","":"downstairs","session_id":"e3a86a50-b5a7-4628-8cbb-a1a3cdecac8d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:46.940554248Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1828,"client":"0","":"downstairs","session_id":"e3a86a50-b5a7-4628-8cbb-a1a3cdecac8d"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:46.940595126Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1828,"client":"0","":"downstairs","session_id":"e3a86a50-b5a7-4628-8cbb-a1a3cdecac8d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:46.941161738Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1828,"client":"2","":"downstairs","session_id":"e3a86a50-b5a7-4628-8cbb-a1a3cdecac8d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:46.941305761Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1828,"client":"1","":"downstairs","session_id":"e3a86a50-b5a7-4628-8cbb-a1a3cdecac8d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:46.941385267Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1828,"client":"0","":"downstairs","session_id":"e3a86a50-b5a7-4628-8cbb-a1a3cdecac8d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:46.941700872Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1828,"client":"2","":"downstairs","session_id":"e3a86a50-b5a7-4628-8cbb-a1a3cdecac8d"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:46.94173746Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1828,"client":"2","":"downstairs","session_id":"e3a86a50-b5a7-4628-8cbb-a1a3cdecac8d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:46.941754629Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1828,"session_id":"e3a86a50-b5a7-4628-8cbb-a1a3cdecac8d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:46.941831516Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1828,"client":"1","":"downstairs","session_id":"e3a86a50-b5a7-4628-8cbb-a1a3cdecac8d"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:46.941852745Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1828,"client":"1","":"downstairs","session_id":"e3a86a50-b5a7-4628-8cbb-a1a3cdecac8d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:46.941866404Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1828,"session_id":"e3a86a50-b5a7-4628-8cbb-a1a3cdecac8d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:46.941903982Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1828,"client":"0","":"downstairs","session_id":"e3a86a50-b5a7-4628-8cbb-a1a3cdecac8d"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:46.941921321Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1828,"client":"0","":"downstairs","session_id":"e3a86a50-b5a7-4628-8cbb-a1a3cdecac8d"} {"msg":"[0]R flush_numbers[0..12]: [276, 276, 283, 283, 276, 283, 276, 279, 274, 267, 284, 288]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:46.94194451Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1828,"":"downstairs","":"downstairs","session_id":"e3a86a50-b5a7-4628-8cbb-a1a3cdecac8d"} {"msg":"[0]R generation[0..12]: [155, 155, 159, 159, 155, 159, 155, 157, 153, 149, 160, 161]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:46.94195917Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1828,"":"downstairs","":"downstairs","session_id":"e3a86a50-b5a7-4628-8cbb-a1a3cdecac8d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:46.941973219Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1828,"":"downstairs","":"downstairs","session_id":"e3a86a50-b5a7-4628-8cbb-a1a3cdecac8d"} {"msg":"[1]R flush_numbers[0..12]: [276, 276, 283, 283, 276, 283, 276, 279, 274, 267, 284, 288]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:46.941986998Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1828,"":"downstairs","":"downstairs","session_id":"e3a86a50-b5a7-4628-8cbb-a1a3cdecac8d"} {"msg":"[1]R generation[0..12]: [155, 155, 159, 159, 155, 159, 155, 157, 153, 149, 160, 161]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:46.942007077Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1828,"":"downstairs","":"downstairs","session_id":"e3a86a50-b5a7-4628-8cbb-a1a3cdecac8d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:46.942028416Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1828,"":"downstairs","":"downstairs","session_id":"e3a86a50-b5a7-4628-8cbb-a1a3cdecac8d"} {"msg":"[2]R flush_numbers[0..12]: [276, 276, 283, 283, 276, 283, 276, 279, 274, 267, 284, 288]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:46.942042975Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1828,"":"downstairs","":"downstairs","session_id":"e3a86a50-b5a7-4628-8cbb-a1a3cdecac8d"} {"msg":"[2]R generation[0..12]: [155, 155, 159, 159, 155, 159, 155, 157, 153, 149, 160, 161]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:46.942059274Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1828,"":"downstairs","":"downstairs","session_id":"e3a86a50-b5a7-4628-8cbb-a1a3cdecac8d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:46.942073244Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1828,"":"downstairs","":"downstairs","session_id":"e3a86a50-b5a7-4628-8cbb-a1a3cdecac8d"} {"msg":"Max found gen is 162","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:46.942086133Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1828,"":"downstairs","":"downstairs","session_id":"e3a86a50-b5a7-4628-8cbb-a1a3cdecac8d"} {"msg":"Generation requested: 162 >= found:162","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:46.942099463Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1828,"":"downstairs","":"downstairs","session_id":"e3a86a50-b5a7-4628-8cbb-a1a3cdecac8d"} {"msg":"Next flush: 289","Wait for a query_work_queue command to finish before sending IO v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:46.942120622Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1828,"":"downstairs","":"downstairs","session_id":"e3a86a50-b5a7-4628-8cbb-a1a3cdecac8d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:46.942133811Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1828,"":"downstairs","":"downstairs","session_id":"e3a86a50-b5a7-4628-8cbb-a1a3cdecac8d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:46.9421471Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1828,"session_id":"e3a86a50-b5a7-4628-8cbb-a1a3cdecac8d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:46.9421593Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1828,"client":"0","":"downstairs","session_id":"e3a86a50-b5a7-4628-8cbb-a1a3cdecac8d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:46.942179159Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1828,"client":"1","":"downstairs","session_id":"e3a86a50-b5a7-4628-8cbb-a1a3cdecac8d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:46.942194618Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1828,"client":"2","":"downstairs","session_id":"e3a86a50-b5a7-4628-8cbb-a1a3cdecac8d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:46.942208237Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1828,"session_id":"e3a86a50-b5a7-4628-8cbb-a1a3cdecac8d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:46.942221297Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1828,"session_id":"e3a86a50-b5a7-4628-8cbb-a1a3cdecac8d"} {"msg":"7ee3905e-f6e0-4426-98dc-3770c7a00e2a is now active with session: e3a86a50-b5a7-4628-8cbb-a1a3cdecac8d","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:46.942234086Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1828,"session_id":"e3a86a50-b5a7-4628-8cbb-a1a3cdecac8d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:46.942245916Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1828,"session_id":"e3a86a50-b5a7-4628-8cbb-a1a3cdecac8d"} {"msg":"The guest has finished waiting for activation with:162","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:46.942258195Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1828} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 490 len 20480 data: 14 11 9 10 9 02/30 Write block 353 len 8192 data: 20 18 03/30 Write block 65 len 36864 data: 13 11 10 10 8 12 11 12 13 04/30 Read block 257 len 8192 05/30 Write block 252 len 28672 data: 19 17 16 15 10 12 12 06/30 Read block 40 len 36864 07/30 Write block 449 len 16384 data: 12 10 10 12 08/30 Write block 50 len 40960 data: 11 9 9 10 12 13 11 12 11 8 09/30 Read block 185 len 12288 10/30 Read block 207 len 28672 11/30 Read block 144 len 32768 12/30 Flush 13/30 Write block 531 len 20480 data: 15 13 14 13 15 14/30 Read block 467 len 12288 15/30 Write block 207 len 8192 data: 12 10 16/30 Write block 77 len 4096 data: 13 17/30 Read block 49 len 16384 18/30 Read block 518 len 12288 19/30 Write block 426 len 40960 data: 12 12 12 18 20 19 17 16 16 19 20/30 Write block 58 len 20480 data: 12 9 10 11 11 21/30 Read block 556 len 4096 22/30 Read block 69 len 24576 23/30 Read block 196 len 16384 24/30 Read block 219 len 8192 25/30 Write block 427 len 12288 data: 13 13 19 26/30 Write block 581 len 20480 data: 13 14 16 18 21 27/30 Write block 580 len 32768 data: 12 14 15 17 19 22 17 20 28/30 Write block 555 len 40960 data: 11 11 11 11 11 10 10 9 8 9 29/30 Write block 19 len 16384 data: 11 11 12 12 30/30 Read block 355 len 4096 ---------------------------------------------------------------- Crucible gen:162 GIO:true work queues: Upstairs:0 downstairs:18 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1012 Acked Write 5 Done Done Done false 1013 Acked Read 3 Done Done Done false 1014 Acked Write 2 Done Done Done false 1015 Acked Write 1 Done Done Done false 1016 Acked Read 4 Done Done Done false 1017 Acked Read 3 Done Done Done false 1018 Acked Write 10 Done Done Done false 1019 Acked Write 5 Done Done Done false 1020 Acked Read 1 Done Done Done false 1021 Acked Read 6 Done Done Done false 1022 Acked Read 4 Done Done Done false 1023 Acked Read 2 Done Done Done false 1024 Acked Write 3 Done Done Done false 1025 Acked Write 5 Done Done Done false 1026 Acked Write 8 Done Done Done false 1027 Acked Write 10 Done Done Done false 1028 Acked Write 4 Sent Sent Done false 1029 Acked Read 1 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 0 4 Done 16 16 18 50 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1011 1011 1011 Downstairs last five completed: 1011 1010 1009 1008 1007 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:28:46.962791971Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1828} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 155 155 155 276 276 276 T T T 1 020-039 155 155 155 276 276 276 T T T 2 040-059 162 162 162 289 289 289 T T T 3 060-079 162 162 162 289 289 289 T T T 4 080-099 155 155 155 276 276 276 F F F 5 100-119 159 159 159 283 283 283 F F F 6 120-139 155 155 155 276 276 276 F F F 7 140-159 157 157 157 279 279 279 F F F 8 160-179 153 153 153 274 274 274 F F F 9 180-199 149 149 149 267 267 267 F F F 10 200-219 160 160 160 284 284 284 T T T 11 220-239 161 161 161 288 288 288 F F F 12 240-259 162 162 162 289 289 289 F F F 13 260-279 160 160 160 286 286 286 F F F 14 280-299 160 160 160 286 286 286 F F F 15 300-319 160 160 160 286 286 286 F F F 16 320-339 160 160 160 284 284 284 F F F 17 340-359 162 162 162 289 289 289 F F F 18 360-379 160 160 160 284 284 284 F F F 19 380-399 161 161 161 288 288 288 F F F 20 400-419 160 160 160 285 285 285 F F F 21 420-439 160 160 160 284 284 284 T T T 22 440-459 162 162 162 289 289 289 F F F 23 460-479 161 161 161 288 288 288 F F F 24 480-499 162 162 162 289 289 289 F F F 25 500-519 161 161 161 288 288 288 F F F 26 520-539 161 161 161 288 288 288 T T T 27 540-559 161 161 161 288 288 288 T T T 28 560-579 155 155 155 276 276 276 T T T 29 580-599 156 156 156 278 278 278 T T T Max gen: 162, Max flush: 289 A Difference in extent metadata is expected here On loop 81 Verifying data now No change after verify Loop: 81 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 163 163 163 290 290 290 F F F 1 020-039 163 163 163 290 290 290 F F F 2 040-059 163 163 163 290 290 290 F F F 3 060-079 163 163 163 290 290 290 F F F 4 080-099 155 155 155 276 276 276 F F F 5 100-119 159 159 159 283 283 283 F F F 6 120-139 155 155 155 276 276 276 F F F 7 140-159 157 157 157 279 279 279 F F F 8 160-179 153 153 153 274 274 274 F F F 9 180-199 149 149 149 267 267 267 F F F 10 200-219 163 163 163 290 290 290 F F F 11 220-239 161 161 161 288 288 288 F F F 12 240-259 162 162 162 289 289 289 F F F 13 260-279 160 160 160 286 286 286 F F F 14 280-299 160 160 160 286 286 286 F F F 15 300-319 160 160 160 286 286 286 F F F 16 320-339 160 160 160 284 284 284 F F F 17 340-359 162 162 162 289 289 289 F F F 18 360-379 160 160 160 284 284 284 F F F 19 380-399 161 161 161 288 288 288 F F F 20 400-419 160 160 160 285 285 285 F F F 21 420-439 163 163 163 290 290 290 F F F 22 440-459 162 162 162 289 289 289 F F F 23 460-479 161 161 161 288 288 288 F F F 24 480-499 162 162 162 289 289 289 F F F 25 500-519 161 161 161 288 288 288 F F F 26 520-539 163 163 163 290 290 290 F F F 27 540-559 163 163 163 290 290 290 F F F 28 560-579 163 163 163 290 290 290 F F F 29 580-599 163 163 163 290 290 290 F F F Max gen: 163, Max flush: 290 Begin loop 82 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:51.390134099Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1836,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:51.434358628Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1836,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:51.434938849Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1836} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"0fd98d0d5d909611f8b408ccda2559d35fe117bb\",\n git_commit_timestamp: \"2024-11-08T20:05:44.000000000Z\",\n git_branch: \"mkeeter/drop-skipped-backpressure-guard\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:51.434989787Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1836} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:51.435033195Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1836} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:51.435058823Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1836} {"msg":"Crucible b0754e63-e9b1-4bde-9308-ed73e8c36092 has session id: 7113183f-2d2b-49bf-b985-f196c05ae967","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:51.435489152Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1836,"session_id":"7113183f-2d2b-49bf-b985-f196c05ae967"} {"msg":"Upstairs opts: Upstairs UUID: b0754e63-e9b1-4bde-9308-ed73e8c36092, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:51.43553559Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1836,"session_id":"7113183f-2d2b-49bf-b985-f196c05ae967"} {"msg":"Crucible stats registered with UUID: b0754e63-e9b1-4bde-9308-ed73e8c36092","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:51.435560439Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1836,"session_id":"7113183f-2d2b-49bf-b985-f196c05ae967"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:51.46664321Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1836,"task":"crutest"} {"msg":"The guest has requested activation with gen:164","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:51.466677068Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1836} {"msg":"b0754e63-e9b1-4bde-9308-ed73e8c36092 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:51.466748374Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1836,"session_id":"7113183f-2d2b-49bf-b985-f196c05ae967"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:51.466771793Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1836,"client":"0","":"downstairs","session_id":"7113183f-2d2b-49bf-b985-f196c05ae967"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:51.466800752Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1836,"client":"0","":"downstairs","session_id":"7113183f-2d2b-49bf-b985-f196c05ae967"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:51.466818681Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1836,"client":"1","":"downstairs","session_id":"7113183f-2d2b-49bf-b985-f196c05ae967"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:51.46684248Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1836,"client":"1","":"downstairs","session_id":"7113183f-2d2b-49bf-b985-f196c05ae967"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:51.466962584Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1836,"client":"2","":"downstairs","session_id":"7113183f-2d2b-49bf-b985-f196c05ae967"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:51.467001892Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1836,"client":"2","":"downstairs","session_id":"7113183f-2d2b-49bf-b985-f196c05ae967"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:51.467031931Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1836,"":"io task","client":"2","":"downstairs","session_id":"7113183f-2d2b-49bf-b985-f196c05ae967"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:51.467064549Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1836,"":"io task","client":"0","":"downstairs","session_id":"7113183f-2d2b-49bf-b985-f196c05ae967"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:51.467094128Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1836,"":"io task","client":"1","":"downstairs","session_id":"7113183f-2d2b-49bf-b985-f196c05ae967"} {"msg":"ds_connection connected from Ok(127.0.0.1:37535)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:51.467123796Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1836,"":"io task","client":"1","":"downstairs","session_id":"7113183f-2d2b-49bf-b985-f196c05ae967"} {"msg":"ds_connection connected from Ok(127.0.0.1:52211)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:51.467154615Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1836,"":"io task","client":"2","":"downstairs","session_id":"7113183f-2d2b-49bf-b985-f196c05ae967"} {"msg":"ds_connection connected from Ok(127.0.0.1:46257)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:51.467235431Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1836,"":"io task","client":"0","":"downstairs","session_id":"7113183f-2d2b-49bf-b985-f196c05ae967"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:51.467905488Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1836,"client":"0","":"downstairs","session_id":"7113183f-2d2b-49bf-b985-f196c05ae967"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:51.467944806Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1836,"client":"0","":"downstairs","session_id":"7113183f-2d2b-49bf-b985-f196c05ae967"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:51.467972535Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1836,"client":"1","":"downstairs","session_id":"7113183f-2d2b-49bf-b985-f196c05ae967"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:51.467999183Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1836,"client":"1","":"downstairs","session_id":"7113183f-2d2b-49bf-b985-f196c05ae967"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:51.468028812Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1836,"client":"2","":"downstairs","session_id":"7113183f-2d2b-49bf-b985-f196c05ae967"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:51.468056061Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1836,"client":"2","":"downstairs","session_id":"7113183f-2d2b-49bf-b985-f196c05ae967"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:51.468667971Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1836,"client":"1","":"downstairs","session_id":"7113183f-2d2b-49bf-b985-f196c05ae967"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:51.468697779Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1836,"client":"0","":"downstairs","session_id":"7113183f-2d2b-49bf-b985-f196c05ae967"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:51.468713449Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1836,"client":"2","":"downstairs","session_id":"7113183f-2d2b-49bf-b985-f196c05ae967"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:51.469260062Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1836,"client":"1","":"downstairs","session_id":"7113183f-2d2b-49bf-b985-f196c05ae967"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:51.4692925Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1836,"client":"1","":"downstairs","session_id":"7113183f-2d2b-49bf-b985-f196c05ae967"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:51.469307929Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1836,"session_id":"7113183f-2d2b-49bf-b985-f196c05ae967"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:51.469320559Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1836,"client":"0","":"downstairs","session_id":"7113183f-2d2b-49bf-b985-f196c05ae967"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:51.469334168Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1836,"client":"0","":"downstairs","session_id":"7113183f-2d2b-49bf-b985-f196c05ae967"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:51.469347687Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1836,"session_id":"7113183f-2d2b-49bf-b985-f196c05ae967"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:51.469359827Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1836,"client":"2","":"downstairs","session_id":"7113183f-2d2b-49bf-b985-f196c05ae967"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:51.469373026Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1836,"client":"2","":"downstairs","session_id":"7113183f-2d2b-49bf-b985-f196c05ae967"} {"msg":"[0]R flush_numbers[0..12]: [290, 290, 290, 290, 276, 283, 276, 279, 274, 267, 290, 288]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:51.469386956Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1836,"":"downstairs","":"downstairs","session_id":"7113183f-2d2b-49bf-b985-f196c05ae967"} {"msg":"[0]R generation[0..12]: [163, 163, 163, 163, 155, 159, 155, 157, 153, 149, 163, 161]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:51.469400985Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1836,"":"downstairs","":"downstairs","session_id":"7113183f-2d2b-49bf-b985-f196c05ae967"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:51.469414944Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1836,"":"downstairs","":"downstairs","session_id":"7113183f-2d2b-49bf-b985-f196c05ae967"} {"msg":"[1]R flush_numbers[0..12]: [290, 290, 290, 290, 276, 283, 276, 279, 274, 267, 290, 288]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:51.469428754Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1836,"":"downstairs","":"downstairs","session_id":"7113183f-2d2b-49bf-b985-f196c05ae967"} {"msg":"[1]R generation[0..12]: [163, 163, 163, 163, 155, 159, 155, 157, 153, 149, 163, 161]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:51.469442573Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1836,"":"downstairs","":"downstairs","session_id":"7113183f-2d2b-49bf-b985-f196c05ae967"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:51.469456602Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1836,"":"downstairs","":"downstairs","session_id":"7113183f-2d2b-49bf-b985-f196c05ae967"} {"msg":"[2]R flush_numbers[0..12]: [290, 290, 290, 290, 276, 283, 276, 279, 274, 267, 290, 288]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:51.469470312Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1836,"":"downstairs","":"downstairs","session_id":"7113183f-2d2b-49bf-b985-f196c05ae967"} {"msg":"[2]R generation[0..12]: [163, 163, 163, 163, 155, 159, 155, 157, 153, 149, 163, 161]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:51.469484101Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1836,"":"downstairs","":"downstairs","session_id":"7113183f-2d2b-49bf-b985-f196c05ae967"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:51.46950031Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1836,"":"downstairs","":"downstairs","session_id":"7113183f-2d2b-49bf-b985-f196c05ae967"} {"msg":"Max found gen is 164","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:51.469513629Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1836,"":"downstairs","":"downstairs","session_id":"7113183f-2d2b-49bf-b985-f196c05ae967"} {"msg":"Generation requested: 164 >= found:164","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:51.469526959Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1836,"":"downstairs","":"downstairs","session_id":"7113183f-2d2b-49bf-b985-f196c05ae967"} {"msg":"Next flush: 291","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:51.469540048Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1836,"":"downstairs","":"downstairs","session_id":"7113183f-2d2b-49bf-b985-f196c05ae967"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:51.469552968Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1836,"":"downstairs","":"downstairs","session_id":"7113183f-2d2b-49bf-b985-f196c05ae967"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:51.469566597Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":Wait for a query_work_queue command to finish before sending IO 1836,"session_id":"7113183f-2d2b-49bf-b985-f196c05ae967"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:51.469591175Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1836,"client":"0","":"downstairs","session_id":"7113183f-2d2b-49bf-b985-f196c05ae967"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:51.469604965Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1836,"client":"1","":"downstairs","session_id":"7113183f-2d2b-49bf-b985-f196c05ae967"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:51.469618124Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1836,"client":"2","":"downstairs","session_id":"7113183f-2d2b-49bf-b985-f196c05ae967"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:51.469631384Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1836,"session_id":"7113183f-2d2b-49bf-b985-f196c05ae967"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:51.469643733Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1836,"session_id":"7113183f-2d2b-49bf-b985-f196c05ae967"} {"msg":"b0754e63-e9b1-4bde-9308-ed73e8c36092 is now active with session: 7113183f-2d2b-49bf-b985-f196c05ae967","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:51.469656532Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1836,"session_id":"7113183f-2d2b-49bf-b985-f196c05ae967"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:51.469668792Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1836,"session_id":"7113183f-2d2b-49bf-b985-f196c05ae967"} {"msg":"The guest has finished waiting for activation with:164","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:51.469681151Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1836} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 435 len 12288 data: 20 19 17 02/30 Write block 92 len 28672 data: 9 9 8 10 8 8 7 03/30 Flush 04/30 Write block 505 len 28672 data: 13 13 12 15 13 13 15 05/30 Flush 06/30 Write block 488 len 20480 data: 12 14 15 12 10 07/30 Read block 135 len 28672 08/30 Write block 583 len 12288 data: 18 20 23 09/30 Write block 352 len 40960 data: 15 21 19 19 18 16 16 17 18 21 10/30 Write block 384 len 32768 data: 8 7 7 9 9 9 10 9 11/30 Read block 396 len 12288 12/30 Read block 329 len 4096 13/30 Read block 249 len 28672 14/30 Read block 572 len 16384 15/30 Read block 416 len 32768 16/30 Read block 557 len 4096 17/30 Write block 575 len 36864 data: 11 10 10 9 8 13 15 16 19 18/30 Write block 521 len 28672 data: 11 10 12 7 9 9 11 19/30 Write block 281 len 28672 data: 9 11 13 14 17 19 19 20/30 Read block 470 len 32768 21/30 Read block 105 len 20480 22/30 Write block 402 len 4096 data: 10 23/30 Flush 24/30 Flush 25/30 Write block 331 len 12288 data: 11 12 12 26/30 Read block 263 len 32768 27/30 Write block 415 len 28672 data: 11 11 11 11 13 13 14 28/30 Read block 131 len 8192 29/30 Write block 445 len 32768 data: 11 14 14 12 13 11 11 13 30/30 Read block 282 len 24576 ---------------------------------------------------------------- Crucible gen:164 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 3 Done Done Done false 1025 Acked Read 8 Done Done Done false 1026 Acked Write 7 Done Done Done false 1027 Acked Read 2 Done Done Done false 1028 Acked Write 8 Done Sent Sent false 1029 Acked Read 6 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 2 2 4 Done 6 4 4 14 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:28:51.499403549Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1836} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 163 163 163 290 290 290 F F F 1 020-039 163 163 163 290 290 290 F F F 2 040-059 163 163 163 290 290 290 F F F 3 060-079 163 163 163 290 290 290 F F F 4 080-099 164 164 164 291 291 291 F F F 5 100-119 159 159 159 283 283 283 F F F 6 120-139 155 155 155 276 276 276 F F F 7 140-159 157 157 157 279 279 279 F F F 8 160-179 153 153 153 274 274 274 F F F 9 180-199 149 149 149 267 267 267 F F F 10 200-219 163 163 163 290 290 290 F F F 11 220-239 161 161 161 288 288 288 F F F 12 240-259 162 162 162 289 289 289 F F F 13 260-279 160 160 160 286 286 286 F F F 14 280-299 164 164 164 293 293 293 F F F 15 300-319 160 160 160 286 286 286 F F F 16 320-339 160 160 160 284 284 284 T T T 17 340-359 164 164 164 293 293 293 F F F 18 360-379 164 164 164 293 293 293 F F F 19 380-399 164 164 164 293 293 293 F F F 20 400-419 164 164 164 293 293 293 T T T 21 420-439 164 164 164 291 291 291 T T T 22 440-459 162 162 162 289 289 289 T T T 23 460-479 161 161 161 288 288 288 F F F 24 480-499 164 164 164 293 293 293 F F F 25 500-519 164 164 164 292 292 292 F F F 26 520-539 164 164 164 293 293 293 F F F 27 540-559 163 163 163 290 290 290 F F F 28 560-579 164 164 164 293 293 293 F F F 29 580-599 164 164 164 293 293 293 F F F Max gen: 164, Max flush: 293 A Difference in extent metadata is expected here On loop 82 Verifying data now No change after verify Loop: 82 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 163 163 163 290 290 290 F F F 1 020-039 163 163 163 290 290 290 F F F 2 040-059 163 163 163 290 290 290 F F F 3 060-079 163 163 163 290 290 290 F F F 4 080-099 164 164 164 291 291 291 F F F 5 100-119 159 159 159 283 283 283 F F F 6 120-139 155 155 155 276 276 276 F F F 7 140-159 157 157 157 279 279 279 F F F 8 160-179 153 153 153 274 274 274 F F F 9 180-199 149 149 149 267 267 267 F F F 10 200-219 163 163 163 290 290 290 F F F 11 220-239 161 161 161 288 288 288 F F F 12 240-259 162 162 162 289 289 289 F F F 13 260-279 160 160 160 286 286 286 F F F 14 280-299 164 164 164 293 293 293 F F F 15 300-319 160 160 160 286 286 286 F F F 16 320-339 165 165 165 294 294 294 F F F 17 340-359 164 164 164 293 293 293 F F F 18 360-379 164 164 164 293 293 293 F F F 19 380-399 164 164 164 293 293 293 F F F 20 400-419 165 165 165 294 294 294 F F F 21 420-439 165 165 165 294 294 294 F F F 22 440-459 165 165 165 294 294 294 F F F 23 460-479 161 161 161 288 288 288 F F F 24 480-499 164 164 164 293 293 293 F F F 25 500-519 164 164 164 292 292 292 F F F 26 520-539 164 164 164 293 293 293 F F F 27 540-559 163 163 163 290 290 290 F F F 28 560-579 164 164 164 293 293 293 F F F 29 580-599 164 164 164 293 293 293 F F F Max gen: 165, Max flush: 294 Begin loop 83 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:55.805292802Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1844,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:55.849902662Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1844,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:55.850531891Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1844} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"0fd98d0d5d909611f8b408ccda2559d35fe117bb\",\n git_commit_timestamp: \"2024-11-08T20:05:44.000000000Z\",\n git_branch: \"mkeeter/drop-skipped-backpressure-guard\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:55.850567399Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1844} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:55.850582038Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1844} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:55.850596268Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1844} {"msg":"Crucible 363b1ad2-b2e8-431a-b45a-787d66f315bc has session id: 11eb6597-03fb-4ab0-bf8c-a4df255cb76c","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:55.851055275Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1844,"session_id":"11eb6597-03fb-4ab0-bf8c-a4df255cb76c"} {"msg":"Upstairs opts: Upstairs UUID: 363b1ad2-b2e8-431a-b45a-787d66f315bc, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:55.851107803Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1844,"session_id":"11eb6597-03fb-4ab0-bf8c-a4df255cb76c"} {"msg":"Crucible stats registered with UUID: 363b1ad2-b2e8-431a-b45a-787d66f315bc","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:55.851133821Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1844,"session_id":"11eb6597-03fb-4ab0-bf8c-a4df255cb76c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:55.88288051Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1844,"task":"crutest"} {"msg":"The guest has requested activation with gen:166","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:55.882913678Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1844} {"msg":"363b1ad2-b2e8-431a-b45a-787d66f315bc active request set","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:55.882948176Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1844,"session_id":"11eb6597-03fb-4ab0-bf8c-a4df255cb76c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:55.882964766Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1844,"client":"0","":"downstairs","session_id":"11eb6597-03fb-4ab0-bf8c-a4df255cb76c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:55.882992364Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1844,"client":"0","":"downstairs","session_id":"11eb6597-03fb-4ab0-bf8c-a4df255cb76c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:55.883009933Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1844,"client":"1","":"downstairs","session_id":"11eb6597-03fb-4ab0-bf8c-a4df255cb76c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:55.883027013Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1844,"client":"1","":"downstairs","session_id":"11eb6597-03fb-4ab0-bf8c-a4df255cb76c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:55.883043612Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1844,"client":"2","":"downstairs","session_id":"11eb6597-03fb-4ab0-bf8c-a4df255cb76c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:55.883060041Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1844,"client":"2","":"downstairs","session_id":"11eb6597-03fb-4ab0-bf8c-a4df255cb76c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:55.88307626Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1844,"":"io task","client":"2","":"downstairs","session_id":"11eb6597-03fb-4ab0-bf8c-a4df255cb76c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:55.883093549Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1844,"":"io task","client":"0","":"downstairs","session_id":"11eb6597-03fb-4ab0-bf8c-a4df255cb76c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:55.883110539Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1844,"":"io task","client":"1","":"downstairs","session_id":"11eb6597-03fb-4ab0-bf8c-a4df255cb76c"} {"msg":"ds_connection connected from Ok(127.0.0.1:43761)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:55.883211304Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1844,"":"io task","client":"2","":"downstairs","session_id":"11eb6597-03fb-4ab0-bf8c-a4df255cb76c"} {"msg":"ds_connection connected from Ok(127.0.0.1:48714)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:55.883265981Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1844,"":"io task","client":"0","":"downstairs","session_id":"11eb6597-03fb-4ab0-bf8c-a4df255cb76c"} {"msg":"ds_connection connected from Ok(127.0.0.1:39238)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:55.883312309Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1844,"":"io task","client":"1","":"downstairs","session_id":"11eb6597-03fb-4ab0-bf8c-a4df255cb76c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:55.884033903Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1844,"client":"2","":"downstairs","session_id":"11eb6597-03fb-4ab0-bf8c-a4df255cb76c"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:55.884063962Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1844,"client":"2","":"downstairs","session_id":"11eb6597-03fb-4ab0-bf8c-a4df255cb76c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:55.884079031Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1844,"client":"0","":"downstairs","session_id":"11eb6597-03fb-4ab0-bf8c-a4df255cb76c"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:55.88409288Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1844,"client":"0","":"downstairs","session_id":"11eb6597-03fb-4ab0-bf8c-a4df255cb76c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:55.88410655Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1844,"client":"1","":"downstairs","session_id":"11eb6597-03fb-4ab0-bf8c-a4df255cb76c"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:55.884130729Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1844,"client":"1","":"downstairs","session_id":"11eb6597-03fb-4ab0-bf8c-a4df255cb76c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:55.884655723Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1844,"client":"2","":"downstairs","session_id":"11eb6597-03fb-4ab0-bf8c-a4df255cb76c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:55.884762358Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1844,"client":"0","":"downstairs","session_id":"11eb6597-03fb-4ab0-bf8c-a4df255cb76c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:55.884853073Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1844,"client":"1","":"downstairs","session_id":"11eb6597-03fb-4ab0-bf8c-a4df255cb76c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:55.88513854Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1844,"client":"2","":"downstairs","session_id":"11eb6597-03fb-4ab0-bf8c-a4df255cb76c"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:55.885159248Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1844,"client":"2","":"downstairs","session_id":"11eb6597-03fb-4ab0-bf8c-a4df255cb76c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:55.885173398Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1844,"session_id":"11eb6597-03fb-4ab0-bf8c-a4df255cb76c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:55.88533429Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1844,"client":"0","":"downstairs","session_id":"11eb6597-03fb-4ab0-bf8c-a4df255cb76c"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:55.885358249Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1844,"client":"0","":"downstairs","session_id":"11eb6597-03fb-4ab0-bf8c-a4df255cb76c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:55.885375248Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1844,"session_id":"11eb6597-03fb-4ab0-bf8c-a4df255cb76c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:55.885391717Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1844,"client":"1","":"downstairs","session_id":"11eb6597-03fb-4ab0-bf8c-a4df255cb76c"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:55.885405216Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1844,"client":"1","":"downstairs","session_id":"11eb6597-03fb-4ab0-bf8c-a4df255cb76c"} {"msg":"[0]R flush_numbers[0..12]: [290, 290, 290, 290, 291, 283, 276, 279, 274, 267, 290, 288]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:55.885463024Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1844,"":"downstairs","":"downstairs","session_id":"11eb6597-03fb-4ab0-bf8c-a4df255cb76c"} {"msg":"[0]R generation[0..12]: [163, 163, 163, 163, 164, 159, 155, 157, 153, 149, 163, 161]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:55.885482563Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1844,"":"downstairs","":"downstairs","session_id":"11eb6597-03fb-4ab0-bf8c-a4df255cb76c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:55.885496862Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1844,"":"downstairs","":"downstairs","session_id":"11eb6597-03fb-4ab0-bf8c-a4df255cb76c"} {"msg":"[1]R flush_numbers[0..12]: [290, 290, 290, 290, 291, 283, 276, 279, 274, 267, 290, 288]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:55.885510751Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1844,"":"downstairs","":"downstairs","session_id":"11eb6597-03fb-4ab0-bf8c-a4df255cb76c"} {"msg":"[1]R generation[0..12]: [163, 163, 163, 163, 164, 159, 155, 157, 153, 149, 163, 161]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:55.885524441Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1844,"":"downstairs","":"downstairs","session_id":"11eb6597-03fb-4ab0-bf8c-a4df255cb76c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:55.88553843Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1844,"":"downstairs","":"downstairs","session_id":"11eb6597-03fb-4ab0-bf8c-a4df255cb76c"} {"msg":"[2]R flush_numbers[0..12]: [290, 290, 290, 290, 291, 283, 276, 279, 274, 267, 290, 288]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:55.885552109Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1844,"":"downstairs","":"downstairs","session_id":"11eb6597-03fb-4ab0-bf8c-a4df255cb76c"} {"msg":"[2]R generation[0..12]: [163, 163, 163, 163, 164, 159, 155, 157, 153, 149, 163, 161]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:55.885565759Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1844,"":"downstairs","":"downstairs","session_id":"11eb6597-03fb-4ab0-bf8c-a4df255cb76c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:55.885579628Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1844,"":"downstairs","":"downstairs","session_id":"11eb6597-03fb-4ab0-bf8c-a4df255cb76c"} {"msg":"Max found gen is 166","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:55.885592467Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1844,"":"downstairs","":"downstairs","session_id":"11eb6597-03fb-4ab0-bf8c-a4df255cb76c"} {"msg":"Generation requested: 166 >= found:166","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:55.885605857Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1844,"":"downstairs","":"downstairs","session_id":"11eb6597-03fb-4ab0-bf8c-a4df255cb76c"} Wait for a query_work_queue command to finish before sending IO {"msg":"Next flush: 295","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:55.885630415Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1844,"":"downstairs","":"downstairs","session_id":"11eb6597-03fb-4ab0-bf8c-a4df255cb76c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:55.885643995Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1844,"":"downstairs","":"downstairs","session_id":"11eb6597-03fb-4ab0-bf8c-a4df255cb76c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:55.885657124Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1844,"session_id":"11eb6597-03fb-4ab0-bf8c-a4df255cb76c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:55.885669083Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1844,"client":"0","":"downstairs","session_id":"11eb6597-03fb-4ab0-bf8c-a4df255cb76c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:55.885682213Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1844,"client":"1","":"downstairs","session_id":"11eb6597-03fb-4ab0-bf8c-a4df255cb76c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:55.885695322Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1844,"client":"2","":"downstairs","session_id":"11eb6597-03fb-4ab0-bf8c-a4df255cb76c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:55.885708361Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1844,"session_id":"11eb6597-03fb-4ab0-bf8c-a4df255cb76c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:55.885720681Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1844,"session_id":"11eb6597-03fb-4ab0-bf8c-a4df255cb76c"} {"msg":"363b1ad2-b2e8-431a-b45a-787d66f315bc is now active with session: 11eb6597-03fb-4ab0-bf8c-a4df255cb76c","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:55.8857334Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1844,"session_id":"11eb6597-03fb-4ab0-bf8c-a4df255cb76c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:55.88574553Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1844,"session_id":"11eb6597-03fb-4ab0-bf8c-a4df255cb76c"} {"msg":"The guest has finished waiting for activation with:166","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:28:55.885757859Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1844} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 126 len 24576 data: 12 9 9 7 10 10 02/30 Read block 28 len 36864 03/30 Read block 358 len 4096 04/30 Flush 05/30 Write block 224 len 4096 data: 13 06/30 Write block 160 len 40960 data: 6 7 7 7 8 9 9 9 14 13 07/30 Read block 534 len 32768 08/30 Read block 243 len 32768 09/30 Write block 189 len 40960 data: 11 14 11 11 12 12 11 12 11 11 10/30 Read block 503 len 16384 11/30 Flush 12/30 Write block 477 len 12288 data: 20 19 18 13/30 Read block 2 len 4096 14/30 Read block 17 len 12288 15/30 Write block 163 len 8192 data: 8 9 16/30 Read block 466 len 32768 17/30 Read block 305 len 4096 18/30 Read block 469 len 8192 19/30 Write block 399 len 40960 data: 10 10 9 11 10 9 10 11 12 13 20/30 Flush 21/30 Write block 144 len 36864 data: 12 14 13 14 13 14 13 14 11 22/30 Read block 343 len 12288 23/30 Read block 120 len 32768 24/30 Read block 389 len 28672 25/30 Read block 42 len 28672 26/30 Write block 197 len 20480 data: 12 12 10 10 9 27/30 Write block 422 len 4096 data: 14 28/30 Read block 326 len 24576 29/30 Read block 590 len 32768 30/30 Read block 330 len 24576 ---------------------------------------------------------------- Crucible gen:166 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1020 Acked Write 9 Done Done Done false 1021 Acked Read 3 Done Done Done false 1022 Acked Read 8 Done Done Done false 1023 Acked Read 7 Done Done Done false 1024 Acked Read 7 Done Done Done false 1025 Acked Write 5 Done Done Done false 1026 Acked Write 1 Done Done Done false 1027 Acked Read 6 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Read 6 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 0 1 Done 10 9 10 29 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1019 1019 1019 Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:28:55.912706292Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1844} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 163 163 163 290 290 290 F F F 1 020-039 163 163 163 290 290 290 F F F 2 040-059 163 163 163 290 290 290 F F F 3 060-079 163 163 163 290 290 290 F F F 4 080-099 164 164 164 291 291 291 F F F 5 100-119 159 159 159 283 283 283 F F F 6 120-139 166 166 166 295 295 295 F F F 7 140-159 157 157 157 279 279 279 T T T 8 160-179 166 166 166 297 297 297 F F F 9 180-199 166 166 166 296 296 296 T T T 10 200-219 163 163 163 290 290 290 T T T 11 220-239 166 166 166 296 296 296 F F F 12 240-259 162 162 162 289 289 289 F F F 13 260-279 160 160 160 286 286 286 F F F 14 280-299 164 164 164 293 293 293 F F F 15 300-319 160 160 160 286 286 286 F F F 16 320-339 165 165 165 294 294 294 F F F 17 340-359 164 164 164 293 293 293 F F F 18 360-379 164 164 164 293 293 293 F F F 19 380-399 166 166 166 297 297 297 F F F 20 400-419 166 166 166 297 297 297 F F F 21 420-439 165 165 165 294 294 294 T T T 22 440-459 165 165 165 294 294 294 F F F 23 460-479 166 166 166 297 297 297 F F F 24 480-499 164 164 164 293 293 293 F F F 25 500-519 164 164 164 292 292 292 F F F 26 520-539 164 164 164 293 293 293 F F F 27 540-559 163 163 163 290 290 290 F F F 28 560-579 164 164 164 293 293 293 F F F 29 580-599 164 164 164 293 293 293 F F F Max gen: 166, Max flush: 297 A Difference in extent metadata is expected here On loop 83 Verifying data now No change after verify Loop: 83 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 163 163 163 290 290 290 F F F 1 020-039 163 163 163 290 290 290 F F F 2 040-059 163 163 163 290 290 290 F F F 3 060-079 163 163 163 290 290 290 F F F 4 080-099 164 164 164 291 291 291 F F F 5 100-119 159 159 159 283 283 283 F F F 6 120-139 166 166 166 295 295 295 F F F 7 140-159 167 167 167 298 298 298 F F F 8 160-179 166 166 166 297 297 297 F F F 9 180-199 167 167 167 298 298 298 F F F 10 200-219 167 167 167 298 298 298 F F F 11 220-239 166 166 166 296 296 296 F F F 12 240-259 162 162 162 289 289 289 F F F 13 260-279 160 160 160 286 286 286 F F F 14 280-299 164 164 164 293 293 293 F F F 15 300-319 160 160 160 286 286 286 F F F 16 320-339 165 165 165 294 294 294 F F F 17 340-359 164 164 164 293 293 293 F F F 18 360-379 164 164 164 293 293 293 F F F 19 380-399 166 166 166 297 297 297 F F F 20 400-419 166 166 166 297 297 297 F F F 21 420-439 167 167 167 298 298 298 F F F 22 440-459 165 165 165 294 294 294 F F F 23 460-479 166 166 166 297 297 297 F F F 24 480-499 164 164 164 293 293 293 F F F 25 500-519 164 164 164 292 292 292 F F F 26 520-539 164 164 164 293 293 293 F F F 27 540-559 163 163 163 290 290 290 F F F 28 560-579 164 164 164 293 293 293 F F F 29 580-599 164 164 164 293 293 293 F F F Max gen: 167, Max flush: 298 Begin loop 84 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:00.218736099Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1852,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:00.263260103Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1852,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:00.263897972Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1852} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"0fd98d0d5d909611f8b408ccda2559d35fe117bb\",\n git_commit_timestamp: \"2024-11-08T20:05:44.000000000Z\",\n git_branch: \"mkeeter/drop-skipped-backpressure-guard\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:00.263943999Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1852} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:00.263962048Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1852} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:00.263979848Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1852} {"msg":"Crucible 307f2a83-2313-4218-8c63-1f499125f917 has session id: 490acb17-6ec4-41bd-b68a-c47f68d3db88","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:00.264444195Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1852,"session_id":"490acb17-6ec4-41bd-b68a-c47f68d3db88"} {"msg":"Upstairs opts: Upstairs UUID: 307f2a83-2313-4218-8c63-1f499125f917, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:00.264489812Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1852,"session_id":"490acb17-6ec4-41bd-b68a-c47f68d3db88"} {"msg":"Crucible stats registered with UUID: 307f2a83-2313-4218-8c63-1f499125f917","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:00.264515161Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1852,"session_id":"490acb17-6ec4-41bd-b68a-c47f68d3db88"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:00.295791073Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1852,"task":"crutest"} {"msg":"The guest has requested activation with gen:168","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:00.295831291Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1852} {"msg":"307f2a83-2313-4218-8c63-1f499125f917 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:00.295877138Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1852,"session_id":"490acb17-6ec4-41bd-b68a-c47f68d3db88"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:00.295894328Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1852,"client":"0","":"downstairs","session_id":"490acb17-6ec4-41bd-b68a-c47f68d3db88"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:00.295915957Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1852,"client":"0","":"downstairs","session_id":"490acb17-6ec4-41bd-b68a-c47f68d3db88"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:00.295930236Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1852,"client":"1","":"downstairs","session_id":"490acb17-6ec4-41bd-b68a-c47f68d3db88"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:00.295944175Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1852,"client":"1","":"downstairs","session_id":"490acb17-6ec4-41bd-b68a-c47f68d3db88"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:00.295957545Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1852,"client":"2","":"downstairs","session_id":"490acb17-6ec4-41bd-b68a-c47f68d3db88"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:00.295970824Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1852,"client":"2","":"downstairs","session_id":"490acb17-6ec4-41bd-b68a-c47f68d3db88"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:00.295983983Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1852,"":"io task","client":"2","":"downstairs","session_id":"490acb17-6ec4-41bd-b68a-c47f68d3db88"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:00.295997643Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1852,"":"io task","client":"0","":"downstairs","session_id":"490acb17-6ec4-41bd-b68a-c47f68d3db88"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:00.296013082Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1852,"":"io task","client":"1","":"downstairs","session_id":"490acb17-6ec4-41bd-b68a-c47f68d3db88"} {"msg":"ds_connection connected from Ok(127.0.0.1:42069)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:00.296290548Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1852,"":"io task","client":"2","":"downstairs","session_id":"490acb17-6ec4-41bd-b68a-c47f68d3db88"} {"msg":"ds_connection connected from Ok(127.0.0.1:51921)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:00.296332596Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1852,"":"io task","client":"0","":"downstairs","session_id":"490acb17-6ec4-41bd-b68a-c47f68d3db88"} {"msg":"ds_connection connected from Ok(127.0.0.1:47880)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:00.296361775Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1852,"":"io task","client":"1","":"downstairs","session_id":"490acb17-6ec4-41bd-b68a-c47f68d3db88"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:00.296895609Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1852,"client":"2","":"downstairs","session_id":"490acb17-6ec4-41bd-b68a-c47f68d3db88"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:00.296924377Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1852,"client":"2","":"downstairs","session_id":"490acb17-6ec4-41bd-b68a-c47f68d3db88"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:00.296943246Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1852,"client":"0","":"downstairs","session_id":"490acb17-6ec4-41bd-b68a-c47f68d3db88"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:00.296995554Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1852,"client":"0","":"downstairs","session_id":"490acb17-6ec4-41bd-b68a-c47f68d3db88"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:00.297027412Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1852,"client":"1","":"downstairs","session_id":"490acb17-6ec4-41bd-b68a-c47f68d3db88"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:00.297054251Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1852,"client":"1","":"downstairs","session_id":"490acb17-6ec4-41bd-b68a-c47f68d3db88"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:00.29769794Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1852,"client":"2","":"downstairs","session_id":"490acb17-6ec4-41bd-b68a-c47f68d3db88"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:00.297739347Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1852,"client":"0","":"downstairs","session_id":"490acb17-6ec4-41bd-b68a-c47f68d3db88"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:00.297772146Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1852,"client":"1","":"downstairs","session_id":"490acb17-6ec4-41bd-b68a-c47f68d3db88"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:00.29831343Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1852,"client":"1","":"downstairs","session_id":"490acb17-6ec4-41bd-b68a-c47f68d3db88"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:00.298354857Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1852,"client":"1","":"downstairs","session_id":"490acb17-6ec4-41bd-b68a-c47f68d3db88"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:00.298385126Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1852,"session_id":"490acb17-6ec4-41bd-b68a-c47f68d3db88"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:00.298401965Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1852,"client":"2","":"downstairs","session_id":"490acb17-6ec4-41bd-b68a-c47f68d3db88"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:00.298416405Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1852,"client":"2","":"downstairs","session_id":"490acb17-6ec4-41bd-b68a-c47f68d3db88"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:00.298429974Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1852,"session_id":"490acb17-6ec4-41bd-b68a-c47f68d3db88"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:00.298442533Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1852,"client":"0","":"downstairs","session_id":"490acb17-6ec4-41bd-b68a-c47f68d3db88"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:00.298455883Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1852,"client":"0","":"downstairs","session_id":"490acb17-6ec4-41bd-b68a-c47f68d3db88"} {"msg":"[0]R flush_numbers[0..12]: [290, 290, 290, 290, 291, 283, 295, 298, 297, 298, 298, 296]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:00.298472852Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1852,"":"downstairs","":"downstairs","session_id":"490acb17-6ec4-41bd-b68a-c47f68d3db88"} {"msg":"[0]R generation[0..12]: [163, 163, 163, 163, 164, 159, 166, 167, 166, 167, 167, 166]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:00.298487001Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1852,"":"downstairs","":"downstairs","session_id":"490acb17-6ec4-41bd-b68a-c47f68d3db88"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:00.29850089Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1852,"":"downstairs","":"downstairs","session_id":"490acb17-6ec4-41bd-b68a-c47f68d3db88"} {"msg":"[1]R flush_numbers[0..12]: [290, 290, 290, 290, 291, 283, 295, 298, 297, 298, 298, 296]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:00.29851473Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1852,"":"downstairs","":"downstairs","session_id":"490acb17-6ec4-41bd-b68a-c47f68d3db88"} {"msg":"[1]R generation[0..12]: [163, 163, 163, 163, 164, 159, 166, 167, 166, 167, 167, 166]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:00.298528799Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1852,"":"downstairs","":"downstairs","session_id":"490acb17-6ec4-41bd-b68a-c47f68d3db88"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:00.298542958Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1852,"":"downstairs","":"downstairs","session_id":"490acb17-6ec4-41bd-b68a-c47f68d3db88"} {"msg":"[2]R flush_numbers[0..12]: [290, 290, 290, 290, 291, 283, 295, 298, 297, 298, 298, 296]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:00.298558498Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1852,"":"downstairs","":"downstairs","session_id":"490acb17-6ec4-41bd-b68a-c47f68d3db88"} {"msg":"[2]R generation[0..12]: [163, 163, 163, 163, 164, 159, 166, 167, 166, 167, 167, 166]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:00.298572437Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1852,"":"downstairs","":"downstairs","session_id":"490acb17-6ec4-41bd-b68a-c47f68d3db88"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:00.298586506Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1852,"":"downstairs","":"downstairs","session_id":"490acb17-6ec4-41bd-b68a-c47f68d3db88"} {"msg":"Max found gen is 168","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:00.298599476Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1852,"":"downstairs","":"downstairs","session_id":"490acb17-6ec4-41bd-b68a-c47f68d3db88"} {"msg":"Generation requested: 168 >= found:168","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:00.298612765Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1852,"":"downstairs","":"downstairs","session_id":"490acb17-6ec4-41bd-b68a-c47f68d3db88"} {"msg":"Next flush: 299","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:00.298625634Z","hostname":"Wait for a query_work_queue command to finish before sending IO w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1852,"":"downstairs","":"downstairs","session_id":"490acb17-6ec4-41bd-b68a-c47f68d3db88"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:00.298648983Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1852,"":"downstairs","":"downstairs","session_id":"490acb17-6ec4-41bd-b68a-c47f68d3db88"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:00.298662333Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1852,"session_id":"490acb17-6ec4-41bd-b68a-c47f68d3db88"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:00.298676232Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1852,"client":"0","":"downstairs","session_id":"490acb17-6ec4-41bd-b68a-c47f68d3db88"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:00.298689811Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1852,"client":"1","":"downstairs","session_id":"490acb17-6ec4-41bd-b68a-c47f68d3db88"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:00.298703271Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1852,"client":"2","":"downstairs","session_id":"490acb17-6ec4-41bd-b68a-c47f68d3db88"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:00.29871651Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1852,"session_id":"490acb17-6ec4-41bd-b68a-c47f68d3db88"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:00.298728799Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1852,"session_id":"490acb17-6ec4-41bd-b68a-c47f68d3db88"} {"msg":"307f2a83-2313-4218-8c63-1f499125f917 is now active with session: 490acb17-6ec4-41bd-b68a-c47f68d3db88","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:00.298741589Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1852,"session_id":"490acb17-6ec4-41bd-b68a-c47f68d3db88"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:00.298753478Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1852,"session_id":"490acb17-6ec4-41bd-b68a-c47f68d3db88"} {"msg":"The guest has finished waiting for activation with:168","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:00.298765777Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1852} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 188 len 32768 data: 11 12 15 12 12 13 13 12 02/30 Write block 234 len 36864 data: 9 11 12 11 14 13 14 13 16 03/30 Read block 51 len 16384 04/30 Write block 590 len 12288 data: 12 11 12 05/30 Write block 5 len 12288 data: 11 11 12 06/30 Write block 495 len 40960 data: 10 10 10 11 10 10 13 13 11 12 07/30 Read block 389 len 32768 08/30 Write block 383 len 8192 data: 7 9 09/30 Read block 451 len 20480 10/30 Read block 462 len 16384 11/30 Read block 584 len 32768 12/30 Flush 13/30 Write block 425 len 16384 data: 13 13 14 14 14/30 Read block 168 len 8192 15/30 Read block 21 len 36864 16/30 Read block 558 len 28672 17/30 Write block 57 len 4096 data: 13 18/30 Write block 141 len 12288 data: 11 10 12 19/30 Read block 163 len 40960 20/30 Read block 528 len 24576 21/30 Read block 132 len 40960 22/30 Read block 358 len 16384 23/30 Write block 46 len 20480 data: 9 5 8 9 12 24/30 Write block 546 len 32768 data: 15 12 14 10 10 6 9 8 25/30 Read block 100 len 32768 26/30 Read block 90 len 12288 27/30 Read block 257 len 8192 28/30 Read block 478 len 24576 29/30 Read block 279 len 32768 30/30 Read block 144 len 16384 ---------------------------------------------------------------- Crucible gen:168 GIO:true work queues: Upstairs:0 downstairs:18 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1012 Acked Write 4 Done Done Done false 1013 Acked Read 2 Done Done Done false 1014 Acked Read 9 Done Done Done false 1015 Acked Read 7 Done Done Done false 1016 Acked Write 1 Done Done Done false 1017 Acked Write 3 Done Done Done false 1018 Acked Read 10 Done Done Done false 1019 Acked Read 6 Done Done Done false 1020 Acked Read 10 Done Done Done false 1021 Acked Read 4 Done Done Done false 1022 Acked Write 5 Done Done Done false 1023 Acked Write 8 Done Done Done false 1024 Acked Read 8 Done Done Done false 1025 Acked Read 3 Done Done Done false 1026 Acked Read 2 Done Done Done false 1027 Acked Read 6 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Read 4 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 0 1 Done 18 17 18 53 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1011 1011 1011 Downstairs last five completed: 1011 1010 1009 1008 1007 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:29:00.323903919Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1852} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 168 168 168 299 299 299 F F F 1 020-039 163 163 163 290 290 290 F F F 2 040-059 163 163 163 290 290 290 T T T 3 060-079 163 163 163 290 290 290 F F F 4 080-099 164 164 164 291 291 291 F F F 5 100-119 159 159 159 283 283 283 F F F 6 120-139 166 166 166 295 295 295 F F F 7 140-159 167 167 167 298 298 298 T T T 8 160-179 166 166 166 297 297 297 F F F 9 180-199 168 168 168 299 299 299 F F F 10 200-219 167 167 167 298 298 298 F F F 11 220-239 168 168 168 299 299 299 F F F 12 240-259 168 168 168 299 299 299 F F F 13 260-279 160 160 160 286 286 286 F F F 14 280-299 164 164 164 293 293 293 F F F 15 300-319 160 160 160 286 286 286 F F F 16 320-339 165 165 165 294 294 294 F F F 17 340-359 164 164 164 293 293 293 F F F 18 360-379 164 164 164 293 293 293 F F F 19 380-399 168 168 168 299 299 299 F F F 20 400-419 166 166 166 297 297 297 F F F 21 420-439 167 167 167 298 298 298 T T T 22 440-459 165 165 165 294 294 294 F F F 23 460-479 166 166 166 297 297 297 F F F 24 480-499 168 168 168 299 299 299 F F F 25 500-519 168 168 168 299 299 299 F F F 26 520-539 164 164 164 293 293 293 F F F 27 540-559 163 163 163 290 290 290 T T T 28 560-579 164 164 164 293 293 293 F F F 29 580-599 168 168 168 299 299 299 F F F Max gen: 168, Max flush: 299 A Difference in extent metadata is expected here On loop 84 Verifying data now No change after verify Loop: 84 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 168 168 168 299 299 299 F F F 1 020-039 163 163 163 290 290 290 F F F 2 040-059 169 169 169 300 300 300 F F F 3 060-079 163 163 163 290 290 290 F F F 4 080-099 164 164 164 291 291 291 F F F 5 100-119 159 159 159 283 283 283 F F F 6 120-139 166 166 166 295 295 295 F F F 7 140-159 169 169 169 300 300 300 F F F 8 160-179 166 166 166 297 297 297 F F F 9 180-199 168 168 168 299 299 299 F F F 10 200-219 167 167 167 298 298 298 F F F 11 220-239 168 168 168 299 299 299 F F F 12 240-259 168 168 168 299 299 299 F F F 13 260-279 160 160 160 286 286 286 F F F 14 280-299 164 164 164 293 293 293 F F F 15 300-319 160 160 160 286 286 286 F F F 16 320-339 165 165 165 294 294 294 F F F 17 340-359 164 164 164 293 293 293 F F F 18 360-379 164 164 164 293 293 293 F F F 19 380-399 168 168 168 299 299 299 F F F 20 400-419 166 166 166 297 297 297 F F F 21 420-439 169 169 169 300 300 300 F F F 22 440-459 165 165 165 294 294 294 F F F 23 460-479 166 166 166 297 297 297 F F F 24 480-499 168 168 168 299 299 299 F F F 25 500-519 168 168 168 299 299 299 F F F 26 520-539 164 164 164 293 293 293 F F F 27 540-559 169 169 169 300 300 300 F F F 28 560-579 164 164 164 293 293 293 F F F 29 580-599 168 168 168 299 299 299 F F F Max gen: 169, Max flush: 300 Begin loop 85 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:04.631357996Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1860,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:29:04.674782064Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1860,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:04.675184964Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1860,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:04.707829329Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1860,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:04.708381732Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1860} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"0fd98d0d5d909611f8b408ccda2559d35fe117bb\",\n git_commit_timestamp: \"2024-11-08T20:05:44.000000000Z\",\n git_branch: \"mkeeter/drop-skipped-backpressure-guard\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:04.70842102Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1860} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:04.708436569Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1860} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:04.708451048Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1860} {"msg":"Crucible 8a1482d9-5e10-4f5e-9757-44d5d40fff59 has session id: d949b0b9-dfd5-4511-8964-25905f69682e","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:04.708924615Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1860,"session_id":"d949b0b9-dfd5-4511-8964-25905f69682e"} {"msg":"Upstairs opts: Upstairs UUID: 8a1482d9-5e10-4f5e-9757-44d5d40fff59, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:04.708970383Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1860,"session_id":"d949b0b9-dfd5-4511-8964-25905f69682e"} {"msg":"Crucible stats registered with UUID: 8a1482d9-5e10-4f5e-9757-44d5d40fff59","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:04.708989132Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1860,"session_id":"d949b0b9-dfd5-4511-8964-25905f69682e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:04.739776078Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1860,"task":"crutest"} {"msg":"The guest has requested activation with gen:170","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:04.739803326Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1860} {"msg":"8a1482d9-5e10-4f5e-9757-44d5d40fff59 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:04.739875702Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1860,"session_id":"d949b0b9-dfd5-4511-8964-25905f69682e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:04.739894492Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1860,"client":"0","":"downstairs","session_id":"d949b0b9-dfd5-4511-8964-25905f69682e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:04.73991902Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1860,"client":"0","":"downstairs","session_id":"d949b0b9-dfd5-4511-8964-25905f69682e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:04.739944479Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1860,"client":"1","":"downstairs","session_id":"d949b0b9-dfd5-4511-8964-25905f69682e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:04.739971448Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1860,"client":"1","":"downstairs","session_id":"d949b0b9-dfd5-4511-8964-25905f69682e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:04.740001366Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1860,"client":"2","":"downstairs","session_id":"d949b0b9-dfd5-4511-8964-25905f69682e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:04.740029995Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1860,"client":"2","":"downstairs","session_id":"d949b0b9-dfd5-4511-8964-25905f69682e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:04.740058833Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1860,"":"io task","client":"2","":"downstairs","session_id":"d949b0b9-dfd5-4511-8964-25905f69682e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:04.740086872Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1860,"":"io task","client":"0","":"downstairs","session_id":"d949b0b9-dfd5-4511-8964-25905f69682e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:04.74012472Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1860,"":"io task","client":"1","":"downstairs","session_id":"d949b0b9-dfd5-4511-8964-25905f69682e"} {"msg":"ds_connection connected from Ok(127.0.0.1:45061)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:04.740326091Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1860,"":"io task","client":"2","":"downstairs","session_id":"d949b0b9-dfd5-4511-8964-25905f69682e"} {"msg":"ds_connection connected from Ok(127.0.0.1:58161)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:04.740369638Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1860,"":"io task","client":"0","":"downstairs","session_id":"d949b0b9-dfd5-4511-8964-25905f69682e"} {"msg":"ds_connection connected from Ok(127.0.0.1:61163)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:04.740394847Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1860,"":"io task","client":"1","":"downstairs","session_id":"d949b0b9-dfd5-4511-8964-25905f69682e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:04.740902452Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1860,"client":"1","":"downstairs","session_id":"d949b0b9-dfd5-4511-8964-25905f69682e"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:04.740937811Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1860,"client":"1","":"downstairs","session_id":"d949b0b9-dfd5-4511-8964-25905f69682e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:04.74095391Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1860,"client":"2","":"downstairs","session_id":"d949b0b9-dfd5-4511-8964-25905f69682e"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:04.740968249Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1860,"client":"2","":"downstairs","session_id":"d949b0b9-dfd5-4511-8964-25905f69682e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:04.740982299Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1860,"client":"0","":"downstairs","session_id":"d949b0b9-dfd5-4511-8964-25905f69682e"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:04.741034446Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1860,"client":"0","":"downstairs","session_id":"d949b0b9-dfd5-4511-8964-25905f69682e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:04.741550281Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1860,"client":"1","":"downstairs","session_id":"d949b0b9-dfd5-4511-8964-25905f69682e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:04.741695094Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1860,"client":"0","":"downstairs","session_id":"d949b0b9-dfd5-4511-8964-25905f69682e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:04.741727572Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1860,"client":"2","":"downstairs","session_id":"d949b0b9-dfd5-4511-8964-25905f69682e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:04.742136542Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1860,"client":"1","":"downstairs","session_id":"d949b0b9-dfd5-4511-8964-25905f69682e"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:04.74216625Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1860,"client":"1","":"downstairs","session_id":"d949b0b9-dfd5-4511-8964-25905f69682e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:04.74218192Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1860,"session_id":"d949b0b9-dfd5-4511-8964-25905f69682e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:04.742288275Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1860,"client":"0","":"downstairs","session_id":"d949b0b9-dfd5-4511-8964-25905f69682e"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:04.742317303Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1860,"client":"0","":"downstairs","session_id":"d949b0b9-dfd5-4511-8964-25905f69682e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:04.742332622Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1860,"session_id":"d949b0b9-dfd5-4511-8964-25905f69682e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:04.742345342Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1860,"client":"2","":"downstairs","session_id":"d949b0b9-dfd5-4511-8964-25905f69682e"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:04.742362021Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1860,"client":"2","":"downstairs","session_id":"d949b0b9-dfd5-4511-8964-25905f69682e"} {"msg":"[0]R flush_numbers[0..12]: [299, 290, 300, 290, 291, 283, 295, 300, 297, 299, 298, 299]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:04.7423763Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1860,"":"downstairs","":"downstairs","session_id":"d949b0b9-dfd5-4511-8964-25905f69682e"} {"msg":"[0]R generation[0..12]: [168, 163, 169, 163, 164, 159, 166, 169, 166, 168, 167, 168]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:04.7423909Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1860,"":"downstairs","":"downstairs","session_id":"d949b0b9-dfd5-4511-8964-25905f69682e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:04.742405249Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1860,"":"downstairs","":"downstairs","session_id":"d949b0b9-dfd5-4511-8964-25905f69682e"} {"msg":"[1]R flush_numbers[0..12]: [299, 290, 300, 290, 291, 283, 295, 300, 297, 299, 298, 299]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:04.742419388Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1860,"":"downstairs","":"downstairs","session_id":"d949b0b9-dfd5-4511-8964-25905f69682e"} {"msg":"[1]R generation[0..12]: [168, 163, 169, 163, 164, 159, 166, 169, 166, 168, 167, 168]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:04.742433408Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1860,"":"downstairs","":"downstairs","session_id":"d949b0b9-dfd5-4511-8964-25905f69682e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:04.742447397Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1860,"":"downstairs","":"downstairs","session_id":"d949b0b9-dfd5-4511-8964-25905f69682e"} {"msg":"[2]R flush_numbers[0..12]: [299, 290, 300, 290, 291, 283, 295, 300, 297, 299, 298, 299]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:04.742461356Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1860,"":"downstairs","":"downstairs","session_id":"d949b0b9-dfd5-4511-8964-25905f69682e"} {"msg":"[2]R generation[0..12]: [168, 163, 169, 163, 164, 159, 166, 169, 166, 168, 167, 168]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:04.742475175Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1860,"":"downstairs","":"downstairs","session_id":"d949b0b9-dfd5-4511-8964-25905f69682e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:04.742489285Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1860,"":"downstairs","":"downstairs","session_id":"d949b0b9-dfd5-4511-8964-25905f69682e"} {"msg":"Max found gen is 170","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:04.742502274Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1860,"":"downstairs","":"downstairs","session_id":"d949b0b9-dfd5-4511-8964-25905f69682e"} {"msg":"Generation requested: 170 >= found:170","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:04.742515584Z","hostname":"Wait for a query_work_queue command to finish before sending IO w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1860,"":"downstairs","":"downstairs","session_id":"d949b0b9-dfd5-4511-8964-25905f69682e"} {"msg":"Next flush: 301","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:04.742598189Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1860,"":"downstairs","":"downstairs","session_id":"d949b0b9-dfd5-4511-8964-25905f69682e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:04.742614629Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1860,"":"downstairs","":"downstairs","session_id":"d949b0b9-dfd5-4511-8964-25905f69682e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:04.742631078Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1860,"session_id":"d949b0b9-dfd5-4511-8964-25905f69682e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:04.742643617Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1860,"client":"0","":"downstairs","session_id":"d949b0b9-dfd5-4511-8964-25905f69682e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:04.742657107Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1860,"client":"1","":"downstairs","session_id":"d949b0b9-dfd5-4511-8964-25905f69682e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:04.742670346Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1860,"client":"2","":"downstairs","session_id":"d949b0b9-dfd5-4511-8964-25905f69682e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:04.742683745Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1860,"session_id":"d949b0b9-dfd5-4511-8964-25905f69682e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:04.742696125Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1860,"session_id":"d949b0b9-dfd5-4511-8964-25905f69682e"} {"msg":"8a1482d9-5e10-4f5e-9757-44d5d40fff59 is now active with session: d949b0b9-dfd5-4511-8964-25905f69682e","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:04.742708914Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1860,"session_id":"d949b0b9-dfd5-4511-8964-25905f69682e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:04.742721014Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1860,"session_id":"d949b0b9-dfd5-4511-8964-25905f69682e"} {"msg":"The guest has finished waiting for activation with:170","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:04.742733453Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1860} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 50 len 16384 data: 13 10 10 11 02/30 Flush 03/30 Write block 85 len 28672 data: 12 12 11 12 11 10 8 04/30 Write block 206 len 4096 data: 12 05/30 Read block 435 len 28672 06/30 Write block 361 len 32768 data: 22 19 17 17 15 14 17 16 07/30 Read block 526 len 28672 08/30 Write block 415 len 24576 data: 12 12 12 12 14 14 09/30 Read block 354 len 8192 10/30 Read block 539 len 32768 11/30 Read block 17 len 8192 12/30 Flush 13/30 Flush 14/30 Write block 35 len 36864 data: 11 11 12 14 15 15 11 15 14 15/30 Write block 33 len 16384 data: 10 12 12 12 16/30 Write block 222 len 36864 data: 15 13 14 12 15 17 13 13 10 17/30 Write block 19 len 16384 data: 12 12 13 13 18/30 Write block 531 len 28672 data: 16 14 15 14 16 15 13 19/30 Read block 487 len 20480 20/30 Flush 21/30 Write block 267 len 40960 data: 11 15 15 16 16 16 17 17 16 17 22/30 Read block 559 len 36864 23/30 Flush 24/30 Write block 466 len 12288 data: 13 15 17 25/30 Read block 255 len 4096 26/30 Write block 54 len 24576 data: 13 14 12 14 13 10 27/30 Read block 537 len 4096 28/30 Read block 581 len 8192 29/30 Read block 434 len 4096 30/30 Read block 546 len 8192 ---------------------------------------------------------------- Crucible gen:170 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 3 Done Done Done false 1024 Acked Read 1 Done Done Done false 1025 Acked Write 6 Done Done Done false 1026 Acked Read 1 Done Done Done false 1027 Acked Read 2 Done Done Done false 1028 Acked Read 1 Done Done Done false 1029 Acked Read 2 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 0 1 Done 6 7 7 20 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:29:04.772180834Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1860} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 170 170 170 304 304 304 F F F 1 020-039 170 170 170 304 304 304 F F F 2 040-059 170 170 170 304 304 304 T T T 3 060-079 163 163 163 290 290 290 F F F 4 080-099 170 170 170 302 302 302 F F F 5 100-119 159 159 159 283 283 283 F F F 6 120-139 166 166 166 295 295 295 F F F 7 140-159 169 169 169 300 300 300 F F F 8 160-179 166 166 166 297 297 297 F F F 9 180-199 168 168 168 299 299 299 F F F 10 200-219 170 170 170 302 302 302 F F F 11 220-239 170 170 170 304 304 304 F F F 12 240-259 168 168 168 299 299 299 F F F 13 260-279 170 170 170 305 305 305 F F F 14 280-299 164 164 164 293 293 293 F F F 15 300-319 160 160 160 286 286 286 F F F 16 320-339 165 165 165 294 294 294 F F F 17 340-359 164 164 164 293 293 293 F F F 18 360-379 170 170 170 302 302 302 F F F 19 380-399 168 168 168 299 299 299 F F F 20 400-419 170 170 170 302 302 302 F F F 21 420-439 170 170 170 302 302 302 F F F 22 440-459 165 165 165 294 294 294 F F F 23 460-479 166 166 166 297 297 297 T T T 24 480-499 168 168 168 299 299 299 F F F 25 500-519 168 168 168 299 299 299 F F F 26 520-539 170 170 170 304 304 304 F F F 27 540-559 169 169 169 300 300 300 F F F 28 560-579 164 164 164 293 293 293 F F F 29 580-599 168 168 168 299 299 299 F F F Max gen: 170, Max flush: 305 A Difference in extent metadata is expected here On loop 85 Verifying data now No change after verify Loop: 85 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 170 170 170 304 304 304 F F F 1 020-039 170 170 170 304 304 304 F F F 2 040-059 171 171 171 306 306 306 F F F 3 060-079 163 163 163 290 290 290 F F F 4 080-099 170 170 170 302 302 302 F F F 5 100-119 159 159 159 283 283 283 F F F 6 120-139 166 166 166 295 295 295 F F F 7 140-159 169 169 169 300 300 300 F F F 8 160-179 166 166 166 297 297 297 F F F 9 180-199 168 168 168 299 299 299 F F F 10 200-219 170 170 170 302 302 302 F F F 11 220-239 170 170 170 304 304 304 F F F 12 240-259 168 168 168 299 299 299 F F F 13 260-279 170 170 170 305 305 305 F F F 14 280-299 164 164 164 293 293 293 F F F 15 300-319 160 160 160 286 286 286 F F F 16 320-339 165 165 165 294 294 294 F F F 17 340-359 164 164 164 293 293 293 F F F 18 360-379 170 170 170 302 302 302 F F F 19 380-399 168 168 168 299 299 299 F F F 20 400-419 170 170 170 302 302 302 F F F 21 420-439 170 170 170 302 302 302 F F F 22 440-459 165 165 165 294 294 294 F F F 23 460-479 171 171 171 306 306 306 F F F 24 480-499 168 168 168 299 299 299 F F F 25 500-519 168 168 168 299 299 299 F F F 26 520-539 170 170 170 304 304 304 F F F 27 540-559 169 169 169 300 300 300 F F F 28 560-579 164 164 164 293 293 293 F F F 29 580-599 168 168 168 299 299 299 F F F Max gen: 171, Max flush: 306 Begin loop 86 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:09.076450585Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1868,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:29:09.119206315Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1868,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:09.119617005Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1868,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:09.151686358Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1868,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:09.152220472Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1868} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"0fd98d0d5d909611f8b408ccda2559d35fe117bb\",\n git_commit_timestamp: \"2024-11-08T20:05:44.000000000Z\",\n git_branch: \"mkeeter/drop-skipped-backpressure-guard\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:09.15225747Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1868} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:09.152271959Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1868} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:09.152288398Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1868} {"msg":"Crucible 01db189c-d978-4cbe-bd41-8512cc3cc174 has session id: 9b6dfdbd-8ca7-43ea-82b1-095a0b84c0f0","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:09.152779814Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1868,"session_id":"9b6dfdbd-8ca7-43ea-82b1-095a0b84c0f0"} {"msg":"Upstairs opts: Upstairs UUID: 01db189c-d978-4cbe-bd41-8512cc3cc174, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:09.152822622Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1868,"session_id":"9b6dfdbd-8ca7-43ea-82b1-095a0b84c0f0"} {"msg":"Crucible stats registered with UUID: 01db189c-d978-4cbe-bd41-8512cc3cc174","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:09.152848501Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1868,"session_id":"9b6dfdbd-8ca7-43ea-82b1-095a0b84c0f0"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:09.184753442Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1868,"task":"crutest"} {"msg":"The guest has requested activation with gen:172","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:09.18478822Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1868} {"msg":"01db189c-d978-4cbe-bd41-8512cc3cc174 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:09.184820068Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1868,"session_id":"9b6dfdbd-8ca7-43ea-82b1-095a0b84c0f0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:09.184838137Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1868,"client":"0","":"downstairs","session_id":"9b6dfdbd-8ca7-43ea-82b1-095a0b84c0f0"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:09.184865726Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1868,"client":"0","":"downstairs","session_id":"9b6dfdbd-8ca7-43ea-82b1-095a0b84c0f0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:09.184883685Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1868,"client":"1","":"downstairs","session_id":"9b6dfdbd-8ca7-43ea-82b1-095a0b84c0f0"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:09.184901094Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1868,"client":"1","":"downstairs","session_id":"9b6dfdbd-8ca7-43ea-82b1-095a0b84c0f0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:09.184930803Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1868,"client":"2","":"downstairs","session_id":"9b6dfdbd-8ca7-43ea-82b1-095a0b84c0f0"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:09.184964961Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1868,"client":"2","":"downstairs","session_id":"9b6dfdbd-8ca7-43ea-82b1-095a0b84c0f0"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:09.18499733Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1868,"":"io task","client":"2","":"downstairs","session_id":"9b6dfdbd-8ca7-43ea-82b1-095a0b84c0f0"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:09.185030058Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1868,"":"io task","client":"0","":"downstairs","session_id":"9b6dfdbd-8ca7-43ea-82b1-095a0b84c0f0"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:09.185060637Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1868,"":"io task","client":"1","":"downstairs","session_id":"9b6dfdbd-8ca7-43ea-82b1-095a0b84c0f0"} {"msg":"ds_connection connected from Ok(127.0.0.1:48756)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:09.185091515Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1868,"":"io task","client":"2","":"downstairs","session_id":"9b6dfdbd-8ca7-43ea-82b1-095a0b84c0f0"} {"msg":"ds_connection connected from Ok(127.0.0.1:39062)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:09.185125063Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1868,"":"io task","client":"1","":"downstairs","session_id":"9b6dfdbd-8ca7-43ea-82b1-095a0b84c0f0"} {"msg":"ds_connection connected from Ok(127.0.0.1:52510)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:09.185356822Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1868,"":"io task","client":"0","":"downstairs","session_id":"9b6dfdbd-8ca7-43ea-82b1-095a0b84c0f0"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:09.185973002Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1868,"client":"1","":"downstairs","session_id":"9b6dfdbd-8ca7-43ea-82b1-095a0b84c0f0"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:09.18600723Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1868,"client":"1","":"downstairs","session_id":"9b6dfdbd-8ca7-43ea-82b1-095a0b84c0f0"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:09.18602655Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1868,"client":"2","":"downstairs","session_id":"9b6dfdbd-8ca7-43ea-82b1-095a0b84c0f0"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:09.186044489Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1868,"client":"2","":"downstairs","session_id":"9b6dfdbd-8ca7-43ea-82b1-095a0b84c0f0"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:09.186058738Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1868,"client":"0","":"downstairs","session_id":"9b6dfdbd-8ca7-43ea-82b1-095a0b84c0f0"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:09.186072837Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1868,"client":"0","":"downstairs","session_id":"9b6dfdbd-8ca7-43ea-82b1-095a0b84c0f0"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:09.186608901Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1868,"client":"1","":"downstairs","session_id":"9b6dfdbd-8ca7-43ea-82b1-095a0b84c0f0"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:09.186751674Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1868,"client":"0","":"downstairs","session_id":"9b6dfdbd-8ca7-43ea-82b1-095a0b84c0f0"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:09.186778883Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1868,"client":"2","":"downstairs","session_id":"9b6dfdbd-8ca7-43ea-82b1-095a0b84c0f0"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:09.187217041Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1868,"client":"1","":"downstairs","session_id":"9b6dfdbd-8ca7-43ea-82b1-095a0b84c0f0"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:09.18724871Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1868,"client":"1","":"downstairs","session_id":"9b6dfdbd-8ca7-43ea-82b1-095a0b84c0f0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:09.187265269Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1868,"session_id":"9b6dfdbd-8ca7-43ea-82b1-095a0b84c0f0"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:09.187322306Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1868,"client":"0","":"downstairs","session_id":"9b6dfdbd-8ca7-43ea-82b1-095a0b84c0f0"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:09.187363104Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1868,"client":"0","":"downstairs","session_id":"9b6dfdbd-8ca7-43ea-82b1-095a0b84c0f0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:09.187380723Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1868,"session_id":"9b6dfdbd-8ca7-43ea-82b1-095a0b84c0f0"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:09.187393843Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1868,"client":"2","":"downstairs","session_id":"9b6dfdbd-8ca7-43ea-82b1-095a0b84c0f0"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:09.187407392Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1868,"client":"2","":"downstairs","session_id":"9b6dfdbd-8ca7-43ea-82b1-095a0b84c0f0"} {"msg":"[0]R flush_numbers[0..12]: [304, 304, 306, 290, 302, 283, 295, 300, 297, 299, 302, 304]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:09.187421751Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1868,"":"downstairs","":"downstairs","session_id":"9b6dfdbd-8ca7-43ea-82b1-095a0b84c0f0"} {"msg":"[0]R generation[0..12]: [170, 170, 171, 163, 170, 159, 166, 169, 166, 168, 170, 170]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:09.18743946Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1868,"":"downstairs","":"downstairs","session_id":"9b6dfdbd-8ca7-43ea-82b1-095a0b84c0f0"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:09.18745391Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1868,"":"downstairs","":"downstairs","session_id":"9b6dfdbd-8ca7-43ea-82b1-095a0b84c0f0"} {"msg":"[1]R flush_numbers[0..12]: [304, 304, 306, 290, 302, 283, 295, 300, 297, 299, 302, 304]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:09.187467969Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1868,"":"downstairs","":"downstairs","session_id":"9b6dfdbd-8ca7-43ea-82b1-095a0b84c0f0"} {"msg":"[1]R generation[0..12]: [170, 170, 171, 163, 170, 159, 166, 169, 166, 168, 170, 170]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:09.187481848Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1868,"":"downstairs","":"downstairs","session_id":"9b6dfdbd-8ca7-43ea-82b1-095a0b84c0f0"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:09.187495978Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1868,"":"downstairs","":"downstairs","session_id":"9b6dfdbd-8ca7-43ea-82b1-095a0b84c0f0"} {"msg":"[2]R flush_numbers[0..12]: [304, 304, 306, 290, 302, 283, 295, 300, 297, 299, 302, 304]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:09.187509877Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1868,"":"downstairs","":"downstairs","session_id":"9b6dfdbd-8ca7-43ea-82b1-095a0b84c0f0"} {"msg":"[2]R generation[0..12]: [170, 170, 171, 163, 170, 159, 166, 169, 166, 168, 170, 170]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:09.187523746Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1868,"":"downstairs","":"downstairs","session_id":"9b6dfdbd-8ca7-43ea-82b1-095a0b84c0f0"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:09.187537816Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1868,"":"downstairs"Wait for a query_work_queue command to finish before sending IO ,"":"downstairs","session_id":"9b6dfdbd-8ca7-43ea-82b1-095a0b84c0f0"} {"msg":"Max found gen is 172","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:09.187559255Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1868,"":"downstairs","":"downstairs","session_id":"9b6dfdbd-8ca7-43ea-82b1-095a0b84c0f0"} {"msg":"Generation requested: 172 >= found:172","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:09.187573374Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1868,"":"downstairs","":"downstairs","session_id":"9b6dfdbd-8ca7-43ea-82b1-095a0b84c0f0"} {"msg":"Next flush: 307","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:09.187597983Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1868,"":"downstairs","":"downstairs","session_id":"9b6dfdbd-8ca7-43ea-82b1-095a0b84c0f0"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:09.187619861Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1868,"":"downstairs","":"downstairs","session_id":"9b6dfdbd-8ca7-43ea-82b1-095a0b84c0f0"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:09.187634081Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1868,"session_id":"9b6dfdbd-8ca7-43ea-82b1-095a0b84c0f0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:09.18765339Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1868,"client":"0","":"downstairs","session_id":"9b6dfdbd-8ca7-43ea-82b1-095a0b84c0f0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:09.187679929Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1868,"client":"1","":"downstairs","session_id":"9b6dfdbd-8ca7-43ea-82b1-095a0b84c0f0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:09.187695478Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1868,"client":"2","":"downstairs","session_id":"9b6dfdbd-8ca7-43ea-82b1-095a0b84c0f0"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2024-11-08T20:29:09.187710187Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1868,"session_id":"9b6dfdbd-8ca7-43ea-82b1-095a0b84c0f0"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:09.187724946Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1868,"session_id":"9b6dfdbd-8ca7-43ea-82b1-095a0b84c0f0"} {"msg":"01db189c-d978-4cbe-bd41-8512cc3cc174 is now active with session: 9b6dfdbd-8ca7-43ea-82b1-095a0b84c0f0","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:09.187738146Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1868,"session_id":"9b6dfdbd-8ca7-43ea-82b1-095a0b84c0f0"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:09.187750505Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1868,"session_id":"9b6dfdbd-8ca7-43ea-82b1-095a0b84c0f0"} {"msg":"The guest has finished waiting for activation with:172","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:09.187763105Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1868} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 433 len 40960 data: 17 17 21 20 18 16 15 13 16 17 02/30 Write block 435 len 36864 data: 22 21 19 17 16 14 17 18 12 03/30 Read block 564 len 4096 04/30 Read block 552 len 8192 05/30 Read block 58 len 24576 06/30 Write block 50 len 20480 data: 14 11 11 12 14 07/30 Write block 163 len 28672 data: 9 10 10 10 10 15 14 08/30 Read block 521 len 4096 09/30 Write block 373 len 40960 data: 12 11 11 11 15 15 15 13 12 8 10/30 Read block 172 len 40960 11/30 Write block 363 len 16384 data: 18 18 16 15 12/30 Read block 335 len 24576 13/30 Read block 305 len 20480 14/30 Write block 248 len 20480 data: 16 17 19 16 20 15/30 Write block 87 len 40960 data: 12 13 12 11 9 10 10 9 11 9 16/30 Write block 308 len 4096 data: 8 17/30 Write block 2 len 28672 data: 7 7 8 12 12 13 14 18/30 Write block 0 len 12288 data: 3 5 8 19/30 Read block 30 len 40960 20/30 Read block 367 len 32768 21/30 Read block 462 len 4096 22/30 Write block 328 len 28672 data: 10 10 11 12 13 13 13 23/30 Write block 303 len 32768 data: 14 13 11 11 7 9 10 12 24/30 Flush 25/30 Write block 139 len 12288 data: 11 9 12 26/30 Write block 317 len 16384 data: 12 13 16 17 27/30 Write block 273 len 32768 data: 18 18 17 18 15 13 13 13 28/30 Read block 37 len 40960 29/30 Read block 540 len 32768 30/30 Read block 555 len 20480 ---------------------------------------------------------------- Crucible gen:172 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 3 Done Done Done false 1025 Acked Write 4 Done Done Done false 1026 Acked Write 8 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Read 5 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 6 5 5 16 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:29:09.211866507Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1868} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 172 172 172 307 307 307 F F F 1 020-039 170 170 170 304 304 304 F F F 2 040-059 172 172 172 307 307 307 F F F 3 060-079 163 163 163 290 290 290 F F F 4 080-099 172 172 172 307 307 307 F F F 5 100-119 159 159 159 283 283 283 F F F 6 120-139 166 166 166 295 295 295 T T T 7 140-159 169 169 169 300 300 300 T T T 8 160-179 172 172 172 307 307 307 F F F 9 180-199 168 168 168 299 299 299 F F F 10 200-219 170 170 170 302 302 302 F F F 11 220-239 170 170 170 304 304 304 F F F 12 240-259 172 172 172 307 307 307 F F F 13 260-279 170 170 170 305 305 305 T T T 14 280-299 164 164 164 293 293 293 T T T 15 300-319 172 172 172 307 307 307 T T T 16 320-339 172 172 172 307 307 307 T T T 17 340-359 164 164 164 293 293 293 F F F 18 360-379 172 172 172 307 307 307 F F F 19 380-399 172 172 172 307 307 307 F F F 20 400-419 170 170 170 302 302 302 F F F 21 420-439 172 172 172 307 307 307 F F F 22 440-459 172 172 172 307 307 307 F F F 23 460-479 171 171 171 306 306 306 F F F 24 480-499 168 168 168 299 299 299 F F F 25 500-519 168 168 168 299 299 299 F F F 26 520-539 170 170 170 304 304 304 F F F 27 540-559 169 169 169 300 300 300 F F F 28 560-579 164 164 164 293 293 293 F F F 29 580-599 168 168 168 299 299 299 F F F Max gen: 172, Max flush: 307 A Difference in extent metadata is expected here On loop 86 Verifying data now No change after verify Loop: 86 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 172 172 172 307 307 307 F F F 1 020-039 170 170 170 304 304 304 F F F 2 040-059 172 172 172 307 307 307 F F F 3 060-079 163 163 163 290 290 290 F F F 4 080-099 172 172 172 307 307 307 F F F 5 100-119 159 159 159 283 283 283 F F F 6 120-139 173 173 173 308 308 308 F F F 7 140-159 173 173 173 308 308 308 F F F 8 160-179 172 172 172 307 307 307 F F F 9 180-199 168 168 168 299 299 299 F F F 10 200-219 170 170 170 302 302 302 F F F 11 220-239 170 170 170 304 304 304 F F F 12 240-259 172 172 172 307 307 307 F F F 13 260-279 173 173 173 308 308 308 F F F 14 280-299 173 173 173 308 308 308 F F F 15 300-319 173 173 173 308 308 308 F F F 16 320-339 173 173 173 308 308 308 F F F 17 340-359 164 164 164 293 293 293 F F F 18 360-379 172 172 172 307 307 307 F F F 19 380-399 172 172 172 307 307 307 F F F 20 400-419 170 170 170 302 302 302 F F F 21 420-439 172 172 172 307 307 307 F F F 22 440-459 172 172 172 307 307 307 F F F 23 460-479 171 171 171 306 306 306 F F F 24 480-499 168 168 168 299 299 299 F F F 25 500-519 168 168 168 299 299 299 F F F 26 520-539 170 170 170 304 304 304 F F F 27 540-559 169 169 169 300 300 300 F F F 28 560-579 164 164 164 293 293 293 F F F 29 580-599 168 168 168 299 299 299 F F F Max gen: 173, Max flush: 308 Begin loop 87 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:13.57423775Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1876,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:13.619594463Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1876,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:13.620229152Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1876} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"0fd98d0d5d909611f8b408ccda2559d35fe117bb\",\n git_commit_timestamp: \"2024-11-08T20:05:44.000000000Z\",\n git_branch: \"mkeeter/drop-skipped-backpressure-guard\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:13.62026491Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1876} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:13.62027907Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1876} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:13.620293299Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1876} {"msg":"Crucible df2abc56-f268-498a-bceb-2fc08eb21033 has session id: 59e1532c-2133-441c-a364-b59e387d7a30","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:13.620726308Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1876,"session_id":"59e1532c-2133-441c-a364-b59e387d7a30"} {"msg":"Upstairs opts: Upstairs UUID: df2abc56-f268-498a-bceb-2fc08eb21033, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:13.620759446Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1876,"session_id":"59e1532c-2133-441c-a364-b59e387d7a30"} {"msg":"Crucible stats registered with UUID: df2abc56-f268-498a-bceb-2fc08eb21033","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:13.620774465Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1876,"session_id":"59e1532c-2133-441c-a364-b59e387d7a30"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:13.651889514Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1876,"task":"crutest"} {"msg":"The guest has requested activation with gen:174","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:13.651922083Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1876} {"msg":"df2abc56-f268-498a-bceb-2fc08eb21033 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:13.651936762Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1876,"session_id":"59e1532c-2133-441c-a364-b59e387d7a30"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:13.651950291Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1876,"client":"0","":"downstairs","session_id":"59e1532c-2133-441c-a364-b59e387d7a30"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:13.651970031Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1876,"client":"0","":"downstairs","session_id":"59e1532c-2133-441c-a364-b59e387d7a30"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:13.651988Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1876,"client":"1","":"downstairs","session_id":"59e1532c-2133-441c-a364-b59e387d7a30"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:13.652002199Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1876,"client":"1","":"downstairs","session_id":"59e1532c-2133-441c-a364-b59e387d7a30"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:13.652015678Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1876,"client":"2","":"downstairs","session_id":"59e1532c-2133-441c-a364-b59e387d7a30"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:13.652029138Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1876,"client":"2","":"downstairs","session_id":"59e1532c-2133-441c-a364-b59e387d7a30"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:13.652042627Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1876,"":"io task","client":"2","":"downstairs","session_id":"59e1532c-2133-441c-a364-b59e387d7a30"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:13.652056496Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1876,"":"io task","client":"0","":"downstairs","session_id":"59e1532c-2133-441c-a364-b59e387d7a30"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:13.652078765Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1876,"":"io task","client":"1","":"downstairs","session_id":"59e1532c-2133-441c-a364-b59e387d7a30"} {"msg":"ds_connection connected from Ok(127.0.0.1:43511)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:13.652209359Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1876,"":"io task","client":"0","":"downstairs","session_id":"59e1532c-2133-441c-a364-b59e387d7a30"} {"msg":"ds_connection connected from Ok(127.0.0.1:62737)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:13.652430268Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1876,"":"io task","client":"2","":"downstairs","session_id":"59e1532c-2133-441c-a364-b59e387d7a30"} {"msg":"ds_connection connected from Ok(127.0.0.1:52131)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:13.652451907Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1876,"":"io task","client":"1","":"downstairs","session_id":"59e1532c-2133-441c-a364-b59e387d7a30"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:13.652967172Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1876,"client":"2","":"downstairs","session_id":"59e1532c-2133-441c-a364-b59e387d7a30"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:13.65300809Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1876,"client":"2","":"downstairs","session_id":"59e1532c-2133-441c-a364-b59e387d7a30"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:13.653142613Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1876,"client":"0","":"downstairs","session_id":"59e1532c-2133-441c-a364-b59e387d7a30"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:13.653168792Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1876,"client":"0","":"downstairs","session_id":"59e1532c-2133-441c-a364-b59e387d7a30"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:13.653290036Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1876,"client":"1","":"downstairs","session_id":"59e1532c-2133-441c-a364-b59e387d7a30"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:13.653358913Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1876,"client":"1","":"downstairs","session_id":"59e1532c-2133-441c-a364-b59e387d7a30"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:13.65361247Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1876,"client":"2","":"downstairs","session_id":"59e1532c-2133-441c-a364-b59e387d7a30"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:13.653790582Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1876,"client":"0","":"downstairs","session_id":"59e1532c-2133-441c-a364-b59e387d7a30"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:13.654015581Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1876,"client":"1","":"downstairs","session_id":"59e1532c-2133-441c-a364-b59e387d7a30"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:13.654188902Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1876,"client":"2","":"downstairs","session_id":"59e1532c-2133-441c-a364-b59e387d7a30"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:13.654210971Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1876,"client":"2","":"downstairs","session_id":"59e1532c-2133-441c-a364-b59e387d7a30"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:13.65422514Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1876,"session_id":"59e1532c-2133-441c-a364-b59e387d7a30"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:13.654292527Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1876,"client":"0","":"downstairs","session_id":"59e1532c-2133-441c-a364-b59e387d7a30"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:13.654311636Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1876,"client":"0","":"downstairs","session_id":"59e1532c-2133-441c-a364-b59e387d7a30"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:13.654325435Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1876,"session_id":"59e1532c-2133-441c-a364-b59e387d7a30"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:13.654472288Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1876,"client":"1","":"downstairs","session_id":"59e1532c-2133-441c-a364-b59e387d7a30"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:13.654490878Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1876,"client":"1","":"downstairs","session_id":"59e1532c-2133-441c-a364-b59e387d7a30"} {"msg":"[0]R flush_numbers[0..12]: [307, 304, 307, 290, 307, 283, 308, 308, 307, 299, 302, 304]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:13.654507687Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1876,"":"downstairs","":"downstairs","session_id":"59e1532c-2133-441c-a364-b59e387d7a30"} {"msg":"[0]R generation[0..12]: [172, 170, 172, 163, 172, 159, 173, 173, 172, 168, 170, 170]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:13.654521586Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1876,"":"downstairs","":"downstairs","session_id":"59e1532c-2133-441c-a364-b59e387d7a30"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:13.654535565Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1876,"":"downstairs","":"downstairs","session_id":"59e1532c-2133-441c-a364-b59e387d7a30"} {"msg":"[1]R flush_numbers[0..12]: [307, 304, 307, 290, 307, 283, 308, 308, 307, 299, 302, 304]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:13.654552364Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1876,"":"downstairs","":"downstairs","session_id":"59e1532c-2133-441c-a364-b59e387d7a30"} {"msg":"[1]R generation[0..12]: [172, 170, 172, 163, 172, 159, 173, 173, 172, 168, 170, 170]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:13.654566434Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1876,"":"downstairs","":"downstairs","session_id":"59e1532c-2133-441c-a364-b59e387d7a30"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:13.654580293Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1876,"":"downstairs","":"downstairs","session_id":"59e1532c-2133-441c-a364-b59e387d7a30"} {"msg":"[2]R flush_numbers[0..12]: [307, 304, 307, 290, 307, 283, 308, 308, 307, 299, 302, 304]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:13.654593782Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1876,"":"downstairs","":"downstairs","session_id":"59e1532c-2133-441c-a364-b59e387d7a30"} {"msg":"[2]R generation[0..12]: [172, 170, 172, 163, 172, 159, 173, 173, 172, 168, 170, 170]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:13.654607252Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1876,"":"downstairs","":"downstairs","session_id":"59e1532c-2133-441c-a364-b59e387d7a30"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:13.654621081Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1876,"":"downstairs","":"downstairs","session_id":"59e1532c-2133-441c-a364-b59e387d7a30"} {"msg":"Max found gen is 174","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:13.65463406Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1876,"":"downstairs","":"downstairs","session_id":"59e1532c-2133-441c-a364-b59e387d7a30"} {"msg":"Generation requested: 174 >= found:174","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:13.65464726Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1876,"":"downstairs","":"downstairs","session_id":"59e1532c-2133-441c-a364-b59e387d7a30"} {"msg":"Next flush: 309","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:13.654660409Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1876,"":"downstairs","":"downstairs","session_id":"59e1532c-2133-441c-a364-b59e387d7a30"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:13.654673389Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1876,"":"downstairs","":"downstairs","session_id":"59e1532c-2133-441c-a364-b59e387d7a30"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:13.654686578Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1876,"session_id":"59e1532c-2133-441c-a364-b59e387d7a30"} {Wait for a query_work_queue command to finish before sending IO "msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:13.654730036Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1876,"client":"0","":"downstairs","session_id":"59e1532c-2133-441c-a364-b59e387d7a30"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:13.654760324Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1876,"client":"1","":"downstairs","session_id":"59e1532c-2133-441c-a364-b59e387d7a30"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:13.654780173Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1876,"client":"2","":"downstairs","session_id":"59e1532c-2133-441c-a364-b59e387d7a30"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:13.654809002Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1876,"session_id":"59e1532c-2133-441c-a364-b59e387d7a30"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:13.654826051Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1876,"session_id":"59e1532c-2133-441c-a364-b59e387d7a30"} {"msg":"df2abc56-f268-498a-bceb-2fc08eb21033 is now active with session: 59e1532c-2133-441c-a364-b59e387d7a30","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:13.65483995Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1876,"session_id":"59e1532c-2133-441c-a364-b59e387d7a30"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:13.65485247Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1876,"session_id":"59e1532c-2133-441c-a364-b59e387d7a30"} {"msg":"The guest has finished waiting for activation with:174","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:13.654865279Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1876} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 137 len 20480 data: 14 12 12 10 13 02/30 Write block 436 len 12288 data: 22 20 18 03/30 Write block 554 len 28672 data: 10 12 12 12 12 12 11 04/30 Write block 92 len 12288 data: 11 11 10 05/30 Flush 06/30 Write block 298 len 40960 data: 13 17 16 16 15 15 14 12 12 8 07/30 Read block 286 len 28672 08/30 Read block 76 len 12288 09/30 Write block 117 len 24576 data: 13 12 11 11 13 13 10/30 Read block 9 len 32768 11/30 Read block 311 len 36864 12/30 Read block 420 len 4096 13/30 Flush 14/30 Write block 438 len 28672 data: 19 17 15 18 19 13 11 15/30 Flush 16/30 Write block 142 len 24576 data: 11 13 13 15 14 15 17/30 Read block 358 len 8192 18/30 Read block 227 len 24576 19/30 Read block 38 len 36864 20/30 Flush 21/30 Write block 237 len 4096 data: 12 22/30 Write block 333 len 32768 data: 14 14 13 12 14 16 16 15 23/30 Read block 511 len 24576 24/30 Write block 395 len 16384 data: 9 9 11 11 25/30 Read block 283 len 36864 26/30 Read block 60 len 40960 27/30 Read block 505 len 24576 28/30 Read block 398 len 20480 29/30 Write block 456 len 20480 data: 9 10 8 9 9 30/30 Read block 102 len 20480 ---------------------------------------------------------------- Crucible gen:174 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1020 Acked Write 1 Done Done Done false 1021 Acked Write 8 Done Done Done false 1022 Acked Read 6 Done Done Done false 1023 Acked Write 4 Done Done Done false 1024 Acked Read 9 Done Done Done false 1025 Acked Read 10 Done Done Done false 1026 Acked Read 6 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Write 5 Done Done Done false 1029 Acked Read 5 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 1 2 Done 9 10 9 28 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1019 1019 1019 Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:29:13.683049432Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1876} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 172 172 172 307 307 307 F F F 1 020-039 170 170 170 304 304 304 F F F 2 040-059 172 172 172 307 307 307 F F F 3 060-079 163 163 163 290 290 290 F F F 4 080-099 174 174 174 309 309 309 F F F 5 100-119 174 174 174 310 310 310 F F F 6 120-139 174 174 174 310 310 310 F F F 7 140-159 174 174 174 312 312 312 F F F 8 160-179 172 172 172 307 307 307 F F F 9 180-199 168 168 168 299 299 299 F F F 10 200-219 170 170 170 302 302 302 F F F 11 220-239 170 170 170 304 304 304 T T T 12 240-259 172 172 172 307 307 307 F F F 13 260-279 173 173 173 308 308 308 F F F 14 280-299 174 174 174 310 310 310 F F F 15 300-319 174 174 174 310 310 310 F F F 16 320-339 173 173 173 308 308 308 T T T 17 340-359 164 164 164 293 293 293 T T T 18 360-379 172 172 172 307 307 307 F F F 19 380-399 172 172 172 307 307 307 T T T 20 400-419 170 170 170 302 302 302 F F F 21 420-439 174 174 174 311 311 311 F F F 22 440-459 174 174 174 311 311 311 T T T 23 460-479 171 171 171 306 306 306 T T T 24 480-499 168 168 168 299 299 299 F F F 25 500-519 168 168 168 299 299 299 F F F 26 520-539 170 170 170 304 304 304 F F F 27 540-559 174 174 174 309 309 309 F F F 28 560-579 174 174 174 309 309 309 F F F 29 580-599 168 168 168 299 299 299 F F F Max gen: 174, Max flush: 312 A Difference in extent metadata is expected here On loop 87 Verifying data now No change after verify Loop: 87 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 172 172 172 307 307 307 F F F 1 020-039 170 170 170 304 304 304 F F F 2 040-059 172 172 172 307 307 307 F F F 3 060-079 163 163 163 290 290 290 F F F 4 080-099 174 174 174 309 309 309 F F F 5 100-119 174 174 174 310 310 310 F F F 6 120-139 174 174 174 310 310 310 F F F 7 140-159 174 174 174 312 312 312 F F F 8 160-179 172 172 172 307 307 307 F F F 9 180-199 168 168 168 299 299 299 F F F 10 200-219 170 170 170 302 302 302 F F F 11 220-239 175 175 175 313 313 313 F F F 12 240-259 172 172 172 307 307 307 F F F 13 260-279 173 173 173 308 308 308 F F F 14 280-299 174 174 174 310 310 310 F F F 15 300-319 174 174 174 310 310 310 F F F 16 320-339 175 175 175 313 313 313 F F F 17 340-359 175 175 175 313 313 313 F F F 18 360-379 172 172 172 307 307 307 F F F 19 380-399 175 175 175 313 313 313 F F F 20 400-419 170 170 170 302 302 302 F F F 21 420-439 174 174 174 311 311 311 F F F 22 440-459 175 175 175 313 313 313 F F F 23 460-479 175 175 175 313 313 313 F F F 24 480-499 168 168 168 299 299 299 F F F 25 500-519 168 168 168 299 299 299 F F F 26 520-539 170 170 170 304 304 304 F F F 27 540-559 174 174 174 309 309 309 F F F 28 560-579 174 174 174 309 309 309 F F F 29 580-599 168 168 168 299 299 299 F F F Max gen: 175, Max flush: 313 Begin loop 88 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:18.019660934Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1884,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:18.064378349Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1884,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:18.068367914Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1884} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"0fd98d0d5d909611f8b408ccda2559d35fe117bb\",\n git_commit_timestamp: \"2024-11-08T20:05:44.000000000Z\",\n git_branch: \"mkeeter/drop-skipped-backpressure-guard\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:18.068415052Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1884} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:18.06843893Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1884} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:18.06845801Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1884} {"msg":"Crucible 557bdd0f-e8be-4933-b094-737521a0b0f4 has session id: b7542641-82bb-49e0-96b3-3202716f8e60","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:18.068885889Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1884,"session_id":"b7542641-82bb-49e0-96b3-3202716f8e60"} {"msg":"Upstairs opts: Upstairs UUID: 557bdd0f-e8be-4933-b094-737521a0b0f4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:18.068934856Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1884,"session_id":"b7542641-82bb-49e0-96b3-3202716f8e60"} {"msg":"Crucible stats registered with UUID: 557bdd0f-e8be-4933-b094-737521a0b0f4","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:18.068960265Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1884,"session_id":"b7542641-82bb-49e0-96b3-3202716f8e60"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:18.100069014Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1884,"task":"crutest"} {"msg":"The guest has requested activation with gen:176","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:18.100102273Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1884} {"msg":"557bdd0f-e8be-4933-b094-737521a0b0f4 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:18.100136371Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1884,"session_id":"b7542641-82bb-49e0-96b3-3202716f8e60"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:18.10015333Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1884,"client":"0","":"downstairs","session_id":"b7542641-82bb-49e0-96b3-3202716f8e60"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:18.100182599Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1884,"client":"0","":"downstairs","session_id":"b7542641-82bb-49e0-96b3-3202716f8e60"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:18.100200378Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1884,"client":"1","":"downstairs","session_id":"b7542641-82bb-49e0-96b3-3202716f8e60"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:18.100217367Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1884,"client":"1","":"downstairs","session_id":"b7542641-82bb-49e0-96b3-3202716f8e60"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:18.100233926Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1884,"client":"2","":"downstairs","session_id":"b7542641-82bb-49e0-96b3-3202716f8e60"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:18.100267235Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1884,"client":"2","":"downstairs","session_id":"b7542641-82bb-49e0-96b3-3202716f8e60"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:18.100301343Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1884,"":"io task","client":"2","":"downstairs","session_id":"b7542641-82bb-49e0-96b3-3202716f8e60"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:18.100335492Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1884,"":"io task","client":"0","":"downstairs","session_id":"b7542641-82bb-49e0-96b3-3202716f8e60"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:18.10036999Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1884,"":"io task","client":"1","":"downstairs","session_id":"b7542641-82bb-49e0-96b3-3202716f8e60"} {"msg":"ds_connection connected from Ok(127.0.0.1:65143)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:18.100407218Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1884,"":"io task","client":"0","":"downstairs","session_id":"b7542641-82bb-49e0-96b3-3202716f8e60"} {"msg":"ds_connection connected from Ok(127.0.0.1:32780)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:18.100583459Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1884,"":"io task","client":"2","":"downstairs","session_id":"b7542641-82bb-49e0-96b3-3202716f8e60"} {"msg":"ds_connection connected from Ok(127.0.0.1:49751)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:18.100625507Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1884,"":"io task","client":"1","":"downstairs","session_id":"b7542641-82bb-49e0-96b3-3202716f8e60"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:18.10118397Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1884,"client":"2","":"downstairs","session_id":"b7542641-82bb-49e0-96b3-3202716f8e60"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:18.101226468Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1884,"client":"2","":"downstairs","session_id":"b7542641-82bb-49e0-96b3-3202716f8e60"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:18.101261046Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1884,"client":"0","":"downstairs","session_id":"b7542641-82bb-49e0-96b3-3202716f8e60"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:18.101292905Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1884,"client":"0","":"downstairs","session_id":"b7542641-82bb-49e0-96b3-3202716f8e60"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:18.101322873Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1884,"client":"1","":"downstairs","session_id":"b7542641-82bb-49e0-96b3-3202716f8e60"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:18.101351482Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1884,"client":"1","":"downstairs","session_id":"b7542641-82bb-49e0-96b3-3202716f8e60"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:18.10178983Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1884,"client":"2","":"downstairs","session_id":"b7542641-82bb-49e0-96b3-3202716f8e60"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:18.101893695Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1884,"client":"1","":"downstairs","session_id":"b7542641-82bb-49e0-96b3-3202716f8e60"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:18.101928584Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1884,"client":"0","":"downstairs","session_id":"b7542641-82bb-49e0-96b3-3202716f8e60"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:18.102350623Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1884,"client":"2","":"downstairs","session_id":"b7542641-82bb-49e0-96b3-3202716f8e60"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:18.102395501Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1884,"client":"2","":"downstairs","session_id":"b7542641-82bb-49e0-96b3-3202716f8e60"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:18.102424929Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1884,"session_id":"b7542641-82bb-49e0-96b3-3202716f8e60"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:18.102464128Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1884,"client":"1","":"downstairs","session_id":"b7542641-82bb-49e0-96b3-3202716f8e60"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:18.102483516Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1884,"client":"1","":"downstairs","session_id":"b7542641-82bb-49e0-96b3-3202716f8e60"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:18.102517745Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1884,"session_id":"b7542641-82bb-49e0-96b3-3202716f8e60"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:18.102538434Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1884,"client":"0","":"downstairs","session_id":"b7542641-82bb-49e0-96b3-3202716f8e60"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:18.102555423Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1884,"client":"0","":"downstairs","session_id":"b7542641-82bb-49e0-96b3-3202716f8e60"} {"msg":"[0]R flush_numbers[0..12]: [307, 304, 307, 290, 309, 310, 310, 312, 307, 299, 302, 313]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:18.102586962Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1884,"":"downstairs","":"downstairs","session_id":"b7542641-82bb-49e0-96b3-3202716f8e60"} {"msg":"[0]R generation[0..12]: [172, 170, 172, 163, 174, 174, 174, 174, 172, 168, 170, 175]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:18.10260465Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1884,"":"downstairs","":"downstairs","session_id":"b7542641-82bb-49e0-96b3-3202716f8e60"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:18.10262254Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1884,"":"downstairs","":"downstairs","session_id":"b7542641-82bb-49e0-96b3-3202716f8e60"} {"msg":"[1]R flush_numbers[0..12]: [307, 304, 307, 290, 309, 310, 310, 312, 307, 299, 302, 313]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:18.102639949Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1884,"":"downstairs","":"downstairs","session_id":"b7542641-82bb-49e0-96b3-3202716f8e60"} {"msg":"[1]R generation[0..12]: [172, 170, 172, 163, 174, 174, 174, 174, 172, 168, 170, 175]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:18.102657118Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1884,"":"downstairs","":"downstairs","session_id":"b7542641-82bb-49e0-96b3-3202716f8e60"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:18.102673447Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1884,"":"downstairs","":"downstairs","session_id":"b7542641-82bb-49e0-96b3-3202716f8e60"} {"msg":"[2]R flush_numbers[0..12]: [307, 304, 307, 290, 309, 310, 310, 312, 307, 299, 302, 313]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:18.102690157Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1884,"":"downstairs","":"downstairs","session_id":"b7542641-82bb-49e0-96b3-3202716f8e60"} {"msg":"[2]R generation[0..12]: [172, 170, 172, 163, 174, 174, 174, 174, 172, 168, 170, 175]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:18.102703976Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1884,"":"downstairs","":"downstairs","session_id":"b7542641-82bb-49e0-96b3-3202716f8e60"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:18.102717925Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1884,"":"downstairs","":"downstairs","session_id":"b7542641-82bb-49e0-96b3-3202716f8e60"} {"msg":"Max found gen is 176","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:18.102730904Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1884,"":"downstairs","":"downstairs","session_id":"b7542641-82bb-49e0-96b3-3202716f8e60"} {"msg":"Generation requested: 176 >= found:176","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:18.102744224Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1884,"":"downstairs","":"downstairs","session_id":"b7542641-82bb-49e0-96b3-3202716f8e60"} {"msg":"Next flush: 314","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:18.102757183Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1884,"":"downstairs","":"downstairs","session_id":"b7542641-82bb-49e0-96b3-3202716f8e60"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:18.102770073Z","hostname":"Wait for a query_work_queue command to finish before sending IO w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1884,"":"downstairs","":"downstairs","session_id":"b7542641-82bb-49e0-96b3-3202716f8e60"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:18.102792381Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1884,"session_id":"b7542641-82bb-49e0-96b3-3202716f8e60"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:18.102804521Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1884,"client":"0","":"downstairs","session_id":"b7542641-82bb-49e0-96b3-3202716f8e60"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:18.10281796Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1884,"client":"1","":"downstairs","session_id":"b7542641-82bb-49e0-96b3-3202716f8e60"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:18.10283115Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1884,"client":"2","":"downstairs","session_id":"b7542641-82bb-49e0-96b3-3202716f8e60"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:18.102844229Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1884,"session_id":"b7542641-82bb-49e0-96b3-3202716f8e60"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:18.102856498Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1884,"session_id":"b7542641-82bb-49e0-96b3-3202716f8e60"} {"msg":"557bdd0f-e8be-4933-b094-737521a0b0f4 is now active with session: b7542641-82bb-49e0-96b3-3202716f8e60","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:18.102869188Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1884,"session_id":"b7542641-82bb-49e0-96b3-3202716f8e60"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:18.102881177Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1884,"session_id":"b7542641-82bb-49e0-96b3-3202716f8e60"} {"msg":"The guest has finished waiting for activation with:176","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:18.102895326Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1884} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 421 len 36864 data: 15 15 13 13 14 14 15 15 20 02/30 Write block 74 len 40960 data: 13 12 13 14 12 10 9 9 9 11 03/30 Read block 381 len 40960 04/30 Read block 466 len 28672 05/30 Read block 462 len 4096 06/30 Write block 379 len 28672 data: 16 14 13 9 8 10 8 07/30 Write block 90 len 32768 data: 12 10 12 12 11 12 10 9 08/30 Flush 09/30 Write block 513 len 40960 data: 13 12 10 9 8 12 14 11 12 11 10/30 Read block 306 len 24576 11/30 Read block 9 len 8192 12/30 Write block 163 len 40960 data: 10 11 11 11 11 16 15 13 12 9 13/30 Read block 279 len 32768 14/30 Read block 373 len 4096 15/30 Write block 343 len 8192 data: 16 14 16/30 Write block 433 len 28672 data: 18 18 23 23 21 20 18 17/30 Read block 360 len 40960 18/30 Write block 163 len 8192 data: 11 12 19/30 Write block 345 len 36864 data: 15 16 14 12 16 15 16 16 22 20/30 Read block 572 len 28672 21/30 Read block 455 len 28672 22/30 Write block 565 len 24576 data: 11 9 7 11 11 9 23/30 Write block 252 len 8192 data: 21 18 24/30 Read block 497 len 20480 25/30 Read block 362 len 40960 26/30 Write block 293 len 28672 data: 13 14 14 11 13 14 18 27/30 Read block 284 len 40960 28/30 Read block 335 len 8192 29/30 Write block 460 len 4096 data: 10 30/30 Write block 81 len 28672 data: 10 10 12 10 13 13 13 ---------------------------------------------------------------- Crucible gen:176 GIO:true work queues: Upstairs:0 downstairs:22 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1008 Acked Write 10 Done Done Done false 1009 Acked Read 6 Done Done Done false 1010 Acked Read 2 Done Done Done false 1011 Acked Write 10 Done Done Done false 1012 Acked Read 8 Done Done Done false 1013 Acked Read 1 Done Done Done false 1014 Acked Write 2 Done Done Done false 1015 Acked Write 7 Done Done Done false 1016 Acked Read 10 Done Done Done false 1017 Acked Write 2 Done Done Done false 1018 Acked Write 9 Done Done Done false 1019 Acked Read 7 Done Done Done false 1020 Acked Read 7 Done Done Done false 1021 Acked Write 6 Done Done Done false 1022 Acked Write 2 Done Done Done false 1023 Acked Read 5 Done Done Done false 1024 Acked Read 10 Done Done Done false 1025 Acked Write 7 Done Done Done false 1026 Acked Read 10 Done Done Done false 1027 Acked Read 2 Done Done Done false 1028 Acked Write 1 Done Done Done false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 21 21 21 63 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1007 1007 1007 Downstairs last five completed: 1007 1006 1005 1004 1003 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:29:18.126448835Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1884} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 172 172 172 307 307 307 F F F 1 020-039 170 170 170 304 304 304 F F F 2 040-059 172 172 172 307 307 307 F F F 3 060-079 176 176 176 314 314 314 F F F 4 080-099 176 176 176 314 314 314 T T T 5 100-119 174 174 174 310 310 310 F F F 6 120-139 174 174 174 310 310 310 F F F 7 140-159 174 174 174 312 312 312 F F F 8 160-179 172 172 172 307 307 307 T T T 9 180-199 168 168 168 299 299 299 F F F 10 200-219 170 170 170 302 302 302 F F F 11 220-239 175 175 175 313 313 313 F F F 12 240-259 172 172 172 307 307 307 T T T 13 260-279 173 173 173 308 308 308 F F F 14 280-299 174 174 174 310 310 310 T T T 15 300-319 174 174 174 310 310 310 F F F 16 320-339 175 175 175 313 313 313 F F F 17 340-359 175 175 175 313 313 313 T T T 18 360-379 176 176 176 314 314 314 F F F 19 380-399 176 176 176 314 314 314 F F F 20 400-419 170 170 170 302 302 302 F F F 21 420-439 176 176 176 314 314 314 T T T 22 440-459 175 175 175 313 313 313 F F F 23 460-479 175 175 175 313 313 313 T T T 24 480-499 168 168 168 299 299 299 F F F 25 500-519 168 168 168 299 299 299 T T T 26 520-539 170 170 170 304 304 304 T T T 27 540-559 174 174 174 309 309 309 F F F 28 560-579 174 174 174 309 309 309 T T T 29 580-599 168 168 168 299 299 299 F F F Max gen: 176, Max flush: 314 A Difference in extent metadata is expected here On loop 88 Verifying data now No change after verify Loop: 88 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 172 172 172 307 307 307 F F F 1 020-039 170 170 170 304 304 304 F F F 2 040-059 172 172 172 307 307 307 F F F 3 060-079 176 176 176 314 314 314 F F F 4 080-099 177 177 177 315 315 315 F F F 5 100-119 174 174 174 310 310 310 F F F 6 120-139 174 174 174 310 310 310 F F F 7 140-159 174 174 174 312 312 312 F F F 8 160-179 177 177 177 315 315 315 F F F 9 180-199 168 168 168 299 299 299 F F F 10 200-219 170 170 170 302 302 302 F F F 11 220-239 175 175 175 313 313 313 F F F 12 240-259 177 177 177 315 315 315 F F F 13 260-279 173 173 173 308 308 308 F F F 14 280-299 177 177 177 315 315 315 F F F 15 300-319 174 174 174 310 310 310 F F F 16 320-339 175 175 175 313 313 313 F F F 17 340-359 177 177 177 315 315 315 F F F 18 360-379 176 176 176 314 314 314 F F F 19 380-399 176 176 176 314 314 314 F F F 20 400-419 170 170 170 302 302 302 F F F 21 420-439 177 177 177 315 315 315 F F F 22 440-459 175 175 175 313 313 313 F F F 23 460-479 177 177 177 315 315 315 F F F 24 480-499 168 168 168 299 299 299 F F F 25 500-519 177 177 177 315 315 315 F F F 26 520-539 177 177 177 315 315 315 F F F 27 540-559 174 174 174 309 309 309 F F F 28 560-579 177 177 177 315 315 315 F F F 29 580-599 168 168 168 299 299 299 F F F Max gen: 177, Max flush: 315 Begin loop 89 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:22.518500378Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1892,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:22.563012062Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1892,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:22.563595954Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1892} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"0fd98d0d5d909611f8b408ccda2559d35fe117bb\",\n git_commit_timestamp: \"2024-11-08T20:05:44.000000000Z\",\n git_branch: \"mkeeter/drop-skipped-backpressure-guard\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:22.563637942Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1892} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:22.563653741Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1892} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:22.56366823Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1892} {"msg":"Crucible 0e3543af-77cf-4fcd-89d6-f9b1a9cd9580 has session id: 9d223071-c04f-4c8c-bd13-ec2e9a147990","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:22.564074591Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1892,"session_id":"9d223071-c04f-4c8c-bd13-ec2e9a147990"} {"msg":"Upstairs opts: Upstairs UUID: 0e3543af-77cf-4fcd-89d6-f9b1a9cd9580, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:22.564119988Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1892,"session_id":"9d223071-c04f-4c8c-bd13-ec2e9a147990"} {"msg":"Crucible stats registered with UUID: 0e3543af-77cf-4fcd-89d6-f9b1a9cd9580","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:22.564144137Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1892,"session_id":"9d223071-c04f-4c8c-bd13-ec2e9a147990"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:22.595237617Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1892,"task":"crutest"} {"msg":"The guest has requested activation with gen:178","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:22.595270136Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1892} {"msg":"0e3543af-77cf-4fcd-89d6-f9b1a9cd9580 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:22.595350262Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1892,"session_id":"9d223071-c04f-4c8c-bd13-ec2e9a147990"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:22.595373921Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1892,"client":"0","":"downstairs","session_id":"9d223071-c04f-4c8c-bd13-ec2e9a147990"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:22.595401379Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1892,"client":"0","":"downstairs","session_id":"9d223071-c04f-4c8c-bd13-ec2e9a147990"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:22.595418939Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1892,"client":"1","":"downstairs","session_id":"9d223071-c04f-4c8c-bd13-ec2e9a147990"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:22.595436028Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1892,"client":"1","":"downstairs","session_id":"9d223071-c04f-4c8c-bd13-ec2e9a147990"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:22.595452507Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1892,"client":"2","":"downstairs","session_id":"9d223071-c04f-4c8c-bd13-ec2e9a147990"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:22.595474056Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1892,"client":"2","":"downstairs","session_id":"9d223071-c04f-4c8c-bd13-ec2e9a147990"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:22.595491335Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1892,"":"io task","client":"2","":"downstairs","session_id":"9d223071-c04f-4c8c-bd13-ec2e9a147990"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:22.595522543Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1892,"":"io task","client":"0","":"downstairs","session_id":"9d223071-c04f-4c8c-bd13-ec2e9a147990"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:22.595550342Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1892,"":"io task","client":"1","":"downstairs","session_id":"9d223071-c04f-4c8c-bd13-ec2e9a147990"} {"msg":"ds_connection connected from Ok(127.0.0.1:57443)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:22.595578841Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1892,"":"io task","client":"2","":"downstairs","session_id":"9d223071-c04f-4c8c-bd13-ec2e9a147990"} {"msg":"ds_connection connected from Ok(127.0.0.1:57036)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:22.595608059Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1892,"":"io task","client":"0","":"downstairs","session_id":"9d223071-c04f-4c8c-bd13-ec2e9a147990"} {"msg":"ds_connection connected from Ok(127.0.0.1:59788)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:22.595640898Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1892,"":"io task","client":"1","":"downstairs","session_id":"9d223071-c04f-4c8c-bd13-ec2e9a147990"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:22.596428859Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1892,"client":"1","":"downstairs","session_id":"9d223071-c04f-4c8c-bd13-ec2e9a147990"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:22.596457928Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1892,"client":"1","":"downstairs","session_id":"9d223071-c04f-4c8c-bd13-ec2e9a147990"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:22.596473547Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1892,"client":"0","":"downstairs","session_id":"9d223071-c04f-4c8c-bd13-ec2e9a147990"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:22.596487726Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1892,"client":"0","":"downstairs","session_id":"9d223071-c04f-4c8c-bd13-ec2e9a147990"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:22.596564292Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1892,"client":"2","":"downstairs","session_id":"9d223071-c04f-4c8c-bd13-ec2e9a147990"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:22.596599731Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1892,"client":"2","":"downstairs","session_id":"9d223071-c04f-4c8c-bd13-ec2e9a147990"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:22.597130285Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1892,"client":"0","":"downstairs","session_id":"9d223071-c04f-4c8c-bd13-ec2e9a147990"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:22.597293327Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1892,"client":"1","":"downstairs","session_id":"9d223071-c04f-4c8c-bd13-ec2e9a147990"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:22.597325185Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1892,"client":"2","":"downstairs","session_id":"9d223071-c04f-4c8c-bd13-ec2e9a147990"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:22.59764105Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1892,"client":"0","":"downstairs","session_id":"9d223071-c04f-4c8c-bd13-ec2e9a147990"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:22.597667509Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1892,"client":"0","":"downstairs","session_id":"9d223071-c04f-4c8c-bd13-ec2e9a147990"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:22.597682358Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1892,"session_id":"9d223071-c04f-4c8c-bd13-ec2e9a147990"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:22.597857629Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1892,"client":"2","":"downstairs","session_id":"9d223071-c04f-4c8c-bd13-ec2e9a147990"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:22.597876668Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1892,"client":"2","":"downstairs","session_id":"9d223071-c04f-4c8c-bd13-ec2e9a147990"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:22.597890248Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1892,"session_id":"9d223071-c04f-4c8c-bd13-ec2e9a147990"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:22.597951935Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1892,"client":"1","":"downstairs","session_id":"9d223071-c04f-4c8c-bd13-ec2e9a147990"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:22.597971804Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1892,"client":"1","":"downstairs","session_id":"9d223071-c04f-4c8c-bd13-ec2e9a147990"} {"msg":"[0]R flush_numbers[0..12]: [307, 304, 307, 314, 315, 310, 310, 312, 315, 299, 302, 313]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:22.597986343Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1892,"":"downstairs","":"downstairs","session_id":"9d223071-c04f-4c8c-bd13-ec2e9a147990"} {"msg":"[0]R generation[0..12]: [172, 170, 172, 176, 177, 174, 174, 174, 177, 168, 170, 175]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:22.598000452Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1892,"":"downstairs","":"downstairs","session_id":"9d223071-c04f-4c8c-bd13-ec2e9a147990"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:22.598014722Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1892,"":"downstairs","":"downstairs","session_id":"9d223071-c04f-4c8c-bd13-ec2e9a147990"} {"msg":"[1]R flush_numbers[0..12]: [307, 304, 307, 314, 315, 310, 310, 312, 315, 299, 302, 313]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:22.598028601Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1892,"":"downstairs","":"downstairs","session_id":"9d223071-c04f-4c8c-bd13-ec2e9a147990"} {"msg":"[1]R generation[0..12]: [172, 170, 172, 176, 177, 174, 174, 174, 177, 168, 170, 175]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:22.59804236Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1892,"":"downstairs","":"downstairs","session_id":"9d223071-c04f-4c8c-bd13-ec2e9a147990"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:22.59805649Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1892,"":"downstairs","":"downstairs","session_id":"9d223071-c04f-4c8c-bd13-ec2e9a147990"} {"msg":"[2]R flush_numbers[0..12]: [307, 304, 307, 314, 315, 310, 310, 312, 315, 299, 302, 313]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:22.598074709Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1892,"":"downstairs","":"downstairs","session_id":"9d223071-c04f-4c8c-bd13-ec2e9a147990"} {"msg":"[2]R generation[0..12]: [172, 170, 172, 176, 177, 174, 174, 174, 177, 168, 170, 175]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:22.598089148Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1892,"":"downstairs","":"downstairs","session_id":"9d223071-c04f-4c8c-bd13-ec2e9a147990"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:22.598102927Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1892,"":"downstairs","":"downstairs","session_id":"9d223071-c04f-4c8c-bd13-ec2e9a147990"} {"msg":"Max found gen is 178","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:22.598116087Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1892,"":"downstairs","":"downstairs","session_id":"9d223071-c04f-4c8c-bd13-ec2e9a147990"} {"msg":"Generation requested: 178 >= found:178","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:22.598129236Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1892,"":"downstairs","":"downstairs","session_id":"9d223071-c04f-4c8c-bd13-ec2e9a147990"} {"msg":"Next flush: 316","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:22.598142135Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1892,"":"downstairs","":"downstairs","session_id":"9d223071-c04f-4c8c-bd13-ec2e9a147990"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:22.598154985Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1892,"":"downstairs","":"downstairs","session_id":"9d223071-c04f-4c8c-bd13-ec2e9a147990"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:22.598168094Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1892,"session_id":"9d223071-c04f-4c8c-bd13-ec2e9a147990"} {"msg":"Transition from WaitQuorum to Active","v":0Wait for a query_work_queue command to finish before sending IO ,"name":"crucible","level":30,"time":"2024-11-08T20:29:22.598199923Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1892,"client":"0","":"downstairs","session_id":"9d223071-c04f-4c8c-bd13-ec2e9a147990"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:22.598215532Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1892,"client":"1","":"downstairs","session_id":"9d223071-c04f-4c8c-bd13-ec2e9a147990"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:22.598228751Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1892,"client":"2","":"downstairs","session_id":"9d223071-c04f-4c8c-bd13-ec2e9a147990"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:22.59824207Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1892,"session_id":"9d223071-c04f-4c8c-bd13-ec2e9a147990"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:22.59825451Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1892,"session_id":"9d223071-c04f-4c8c-bd13-ec2e9a147990"} {"msg":"0e3543af-77cf-4fcd-89d6-f9b1a9cd9580 is now active with session: 9d223071-c04f-4c8c-bd13-ec2e9a147990","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:22.598267249Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1892,"session_id":"9d223071-c04f-4c8c-bd13-ec2e9a147990"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:22.598280349Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1892,"session_id":"9d223071-c04f-4c8c-bd13-ec2e9a147990"} {"msg":"The guest has finished waiting for activation with:178","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:22.598303078Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1892} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 251 len 20480 data: 17 22 19 17 16 02/30 Read block 548 len 32768 03/30 Read block 20 len 4096 04/30 Read block 394 len 40960 05/30 Write block 510 len 28672 data: 14 16 13 14 13 11 10 06/30 Read block 590 len 24576 07/30 Read block 473 len 36864 08/30 Flush 09/30 Write block 356 len 4096 data: 19 10/30 Read block 254 len 16384 11/30 Write block 18 len 28672 data: 12 13 13 14 14 11 12 12/30 Read block 36 len 12288 13/30 Write block 558 len 28672 data: 13 13 12 11 10 9 10 14/30 Write block 74 len 24576 data: 14 13 14 15 13 11 15/30 Write block 179 len 12288 data: 11 12 11 16/30 Write block 395 len 32768 data: 10 10 12 12 11 11 10 12 17/30 Flush 18/30 Write block 46 len 28672 data: 10 6 9 10 15 12 12 19/30 Read block 182 len 28672 20/30 Read block 230 len 16384 21/30 Write block 143 len 8192 data: 14 14 22/30 Write block 57 len 28672 data: 15 14 11 11 12 12 12 23/30 Write block 375 len 28672 data: 12 12 16 16 17 15 14 24/30 Read block 145 len 40960 25/30 Read block 120 len 8192 26/30 Write block 172 len 24576 data: 10 9 11 12 10 11 27/30 Read block 159 len 20480 28/30 Read block 566 len 28672 29/30 Read block 84 len 4096 30/30 Write block 569 len 20480 data: 12 10 14 14 15 ---------------------------------------------------------------- Crucible gen:178 GIO:true work queues: Upstairs:0 downstairs:13 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1017 Acked Write 7 Done Done Done false 1018 Acked Read 7 Done Done Done false 1019 Acked Read 4 Done Done Done false 1020 Acked Write 2 Done Done Done false 1021 Acked Write 7 Done Done Done false 1022 Acked Write 7 Done Done Done false 1023 Acked Read 10 Done Done Done false 1024 Acked Read 2 Done Done Done false 1025 Acked Write 6 Done Done Done false 1026 Acked Read 5 Done Done Done false 1027 Acked Read 7 Done Done Done false 1028 Acked Read 1 Done Done Done false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 12 12 12 36 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1016 1016 1016 Downstairs last five completed: 1016 1015 1014 1013 1012 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:29:22.622762202Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1892} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 178 178 178 317 317 317 F F F 1 020-039 178 178 178 317 317 317 F F F 2 040-059 172 172 172 307 307 307 T T T 3 060-079 178 178 178 317 317 317 T T T 4 080-099 177 177 177 315 315 315 F F F 5 100-119 174 174 174 310 310 310 F F F 6 120-139 174 174 174 310 310 310 F F F 7 140-159 174 174 174 312 312 312 T T T 8 160-179 178 178 178 317 317 317 T T T 9 180-199 178 178 178 317 317 317 F F F 10 200-219 170 170 170 302 302 302 F F F 11 220-239 175 175 175 313 313 313 F F F 12 240-259 178 178 178 316 316 316 F F F 13 260-279 173 173 173 308 308 308 F F F 14 280-299 177 177 177 315 315 315 F F F 15 300-319 174 174 174 310 310 310 F F F 16 320-339 175 175 175 313 313 313 F F F 17 340-359 178 178 178 317 317 317 F F F 18 360-379 176 176 176 314 314 314 T T T 19 380-399 178 178 178 317 317 317 T T T 20 400-419 178 178 178 317 317 317 F F F 21 420-439 177 177 177 315 315 315 F F F 22 440-459 175 175 175 313 313 313 F F F 23 460-479 177 177 177 315 315 315 F F F 24 480-499 168 168 168 299 299 299 F F F 25 500-519 178 178 178 316 316 316 F F F 26 520-539 177 177 177 315 315 315 F F F 27 540-559 178 178 178 317 317 317 F F F 28 560-579 178 178 178 317 317 317 T T T 29 580-599 168 168 168 299 299 299 F F F Max gen: 178, Max flush: 317 A Difference in extent metadata is expected here On loop 89 Verifying data now No change after verify Loop: 89 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 178 178 178 317 317 317 F F F 1 020-039 178 178 178 317 317 317 F F F 2 040-059 179 179 179 318 318 318 F F F 3 060-079 179 179 179 318 318 318 F F F 4 080-099 177 177 177 315 315 315 F F F 5 100-119 174 174 174 310 310 310 F F F 6 120-139 174 174 174 310 310 310 F F F 7 140-159 179 179 179 318 318 318 F F F 8 160-179 179 179 179 318 318 318 F F F 9 180-199 178 178 178 317 317 317 F F F 10 200-219 170 170 170 302 302 302 F F F 11 220-239 175 175 175 313 313 313 F F F 12 240-259 178 178 178 316 316 316 F F F 13 260-279 173 173 173 308 308 308 F F F 14 280-299 177 177 177 315 315 315 F F F 15 300-319 174 174 174 310 310 310 F F F 16 320-339 175 175 175 313 313 313 F F F 17 340-359 178 178 178 317 317 317 F F F 18 360-379 179 179 179 318 318 318 F F F 19 380-399 179 179 179 318 318 318 F F F 20 400-419 178 178 178 317 317 317 F F F 21 420-439 177 177 177 315 315 315 F F F 22 440-459 175 175 175 313 313 313 F F F 23 460-479 177 177 177 315 315 315 F F F 24 480-499 168 168 168 299 299 299 F F F 25 500-519 178 178 178 316 316 316 F F F 26 520-539 177 177 177 315 315 315 F F F 27 540-559 178 178 178 317 317 317 F F F 28 560-579 179 179 179 318 318 318 F F F 29 580-599 168 168 168 299 299 299 F F F Max gen: 179, Max flush: 318 Begin loop 90 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:26.977953545Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1900,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:27.022991014Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1900,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:27.023878561Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1900} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"0fd98d0d5d909611f8b408ccda2559d35fe117bb\",\n git_commit_timestamp: \"2024-11-08T20:05:44.000000000Z\",\n git_branch: \"mkeeter/drop-skipped-backpressure-guard\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:27.023923179Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1900} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:27.023946087Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1900} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:27.023964757Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1900} {"msg":"Crucible 9e4b22b7-cbf2-40d4-8fc2-0368feebceac has session id: d653d7f8-4fe5-4ede-9b04-c09bf0b38722","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:27.026523452Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1900,"session_id":"d653d7f8-4fe5-4ede-9b04-c09bf0b38722"} {"msg":"Upstairs opts: Upstairs UUID: 9e4b22b7-cbf2-40d4-8fc2-0368feebceac, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:27.026566129Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1900,"session_id":"d653d7f8-4fe5-4ede-9b04-c09bf0b38722"} {"msg":"Crucible stats registered with UUID: 9e4b22b7-cbf2-40d4-8fc2-0368feebceac","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:27.026585198Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1900,"session_id":"d653d7f8-4fe5-4ede-9b04-c09bf0b38722"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:27.057461599Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1900,"task":"crutest"} {"msg":"The guest has requested activation with gen:180","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:27.057491758Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1900} {"msg":"9e4b22b7-cbf2-40d4-8fc2-0368feebceac active request set","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:27.057564714Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1900,"session_id":"d653d7f8-4fe5-4ede-9b04-c09bf0b38722"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:27.057586763Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1900,"client":"0","":"downstairs","session_id":"d653d7f8-4fe5-4ede-9b04-c09bf0b38722"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:27.057614272Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1900,"client":"0","":"downstairs","session_id":"d653d7f8-4fe5-4ede-9b04-c09bf0b38722"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:27.057631841Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1900,"client":"1","":"downstairs","session_id":"d653d7f8-4fe5-4ede-9b04-c09bf0b38722"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:27.05764905Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1900,"client":"1","":"downstairs","session_id":"d653d7f8-4fe5-4ede-9b04-c09bf0b38722"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:27.057666419Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1900,"client":"2","":"downstairs","session_id":"d653d7f8-4fe5-4ede-9b04-c09bf0b38722"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:27.057689098Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1900,"client":"2","":"downstairs","session_id":"d653d7f8-4fe5-4ede-9b04-c09bf0b38722"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:27.057721487Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1900,"":"io task","client":"2","":"downstairs","session_id":"d653d7f8-4fe5-4ede-9b04-c09bf0b38722"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:27.057759515Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1900,"":"io task","client":"0","":"downstairs","session_id":"d653d7f8-4fe5-4ede-9b04-c09bf0b38722"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:27.057795043Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1900,"":"io task","client":"1","":"downstairs","session_id":"d653d7f8-4fe5-4ede-9b04-c09bf0b38722"} {"msg":"ds_connection connected from Ok(127.0.0.1:35880)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:27.057828921Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1900,"":"io task","client":"1","":"downstairs","session_id":"d653d7f8-4fe5-4ede-9b04-c09bf0b38722"} {"msg":"ds_connection connected from Ok(127.0.0.1:64317)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:27.0578647Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1900,"":"io task","client":"2","":"downstairs","session_id":"d653d7f8-4fe5-4ede-9b04-c09bf0b38722"} {"msg":"ds_connection connected from Ok(127.0.0.1:41007)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:27.058017082Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1900,"":"io task","client":"0","":"downstairs","session_id":"d653d7f8-4fe5-4ede-9b04-c09bf0b38722"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:27.058587404Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1900,"client":"0","":"downstairs","session_id":"d653d7f8-4fe5-4ede-9b04-c09bf0b38722"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:27.058628972Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1900,"client":"0","":"downstairs","session_id":"d653d7f8-4fe5-4ede-9b04-c09bf0b38722"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:27.058662541Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1900,"client":"2","":"downstairs","session_id":"d653d7f8-4fe5-4ede-9b04-c09bf0b38722"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:27.058717758Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1900,"client":"2","":"downstairs","session_id":"d653d7f8-4fe5-4ede-9b04-c09bf0b38722"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:27.058748397Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1900,"client":"1","":"downstairs","session_id":"d653d7f8-4fe5-4ede-9b04-c09bf0b38722"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:27.058776225Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1900,"client":"1","":"downstairs","session_id":"d653d7f8-4fe5-4ede-9b04-c09bf0b38722"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:27.059256702Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1900,"client":"0","":"downstairs","session_id":"d653d7f8-4fe5-4ede-9b04-c09bf0b38722"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:27.059403874Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1900,"client":"1","":"downstairs","session_id":"d653d7f8-4fe5-4ede-9b04-c09bf0b38722"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:27.059442752Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1900,"client":"2","":"downstairs","session_id":"d653d7f8-4fe5-4ede-9b04-c09bf0b38722"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:27.059783706Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1900,"client":"0","":"downstairs","session_id":"d653d7f8-4fe5-4ede-9b04-c09bf0b38722"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:27.059814994Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1900,"client":"0","":"downstairs","session_id":"d653d7f8-4fe5-4ede-9b04-c09bf0b38722"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:27.059830444Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1900,"session_id":"d653d7f8-4fe5-4ede-9b04-c09bf0b38722"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:27.059918459Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1900,"client":"2","":"downstairs","session_id":"d653d7f8-4fe5-4ede-9b04-c09bf0b38722"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:27.059938948Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1900,"client":"2","":"downstairs","session_id":"d653d7f8-4fe5-4ede-9b04-c09bf0b38722"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:27.059953198Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1900,"session_id":"d653d7f8-4fe5-4ede-9b04-c09bf0b38722"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:27.060040593Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1900,"client":"1","":"downstairs","session_id":"d653d7f8-4fe5-4ede-9b04-c09bf0b38722"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:27.060061512Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1900,"client":"1","":"downstairs","session_id":"d653d7f8-4fe5-4ede-9b04-c09bf0b38722"} {"msg":"[0]R flush_numbers[0..12]: [317, 317, 318, 318, 315, 310, 310, 318, 318, 317, 302, 313]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:27.060076061Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1900,"":"downstairs","":"downstairs","session_id":"d653d7f8-4fe5-4ede-9b04-c09bf0b38722"} {"msg":"[0]R generation[0..12]: [178, 178, 179, 179, 177, 174, 174, 179, 179, 178, 170, 175]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:27.060090301Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1900,"":"downstairs","":"downstairs","session_id":"d653d7f8-4fe5-4ede-9b04-c09bf0b38722"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:27.06010436Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1900,"":"downstairs","":"downstairs","session_id":"d653d7f8-4fe5-4ede-9b04-c09bf0b38722"} {"msg":"[1]R flush_numbers[0..12]: [317, 317, 318, 318, 315, 310, 310, 318, 318, 317, 302, 313]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:27.06011845Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1900,"":"downstairs","":"downstairs","session_id":"d653d7f8-4fe5-4ede-9b04-c09bf0b38722"} {"msg":"[1]R generation[0..12]: [178, 178, 179, 179, 177, 174, 174, 179, 179, 178, 170, 175]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:27.060132159Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1900,"":"downstairs","":"downstairs","session_id":"d653d7f8-4fe5-4ede-9b04-c09bf0b38722"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:27.060146198Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1900,"":"downstairs","":"downstairs","session_id":"d653d7f8-4fe5-4ede-9b04-c09bf0b38722"} {"msg":"[2]R flush_numbers[0..12]: [317, 317, 318, 318, 315, 310, 310, 318, 318, 317, 302, 313]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:27.060160057Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1900,"":"downstairs","":"downstairs","session_id":"d653d7f8-4fe5-4ede-9b04-c09bf0b38722"} {"msg":"[2]R generation[0..12]: [178, 178, 179, 179, 177, 174, 174, 179, 179, 178, 170, 175]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:27.060176647Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1900,"":"downstairs","":"downstairs","session_id":"d653d7f8-4fe5-4ede-9b04-c09bf0b38722"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:27.060190846Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1900,"":"downstairs","":"downstairs","session_id":"d653d7f8-4fe5-4ede-9b04-c09bf0b38722"} {"msg":"Max found gen is 180","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:27.060204005Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1900,"":"downstairs","":"downstairs","session_id":"d653d7f8-4fe5-4ede-9b04-c09bf0b38722"} {"msg":"Generation requested: 180 >= found:180","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:27.060217105Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1900,"":"downstairs","":"downstairs","session_id":"d653d7f8-4fe5-4ede-9b04-c09bf0b38722"} {"msg":"Next flush: 319","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-08T20:29:27.060230184Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1900,"":"downstairs","":"downstairs","session_id":"d653d7f8-4fe5-4ede-9b04-c09bf0b38722"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:27.060255563Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1900,"":"downstairs","":"downstairs","session_id":"d653d7f8-4fe5-4ede-9b04-c09bf0b38722"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:27.060280362Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1900,"session_id":"d653d7f8-4fe5-4ede-9b04-c09bf0b38722"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:27.060304461Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1900,"client":"0","":"downstairs","session_id":"d653d7f8-4fe5-4ede-9b04-c09bf0b38722"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:27.060329989Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1900,"client":"1","":"downstairs","session_id":"d653d7f8-4fe5-4ede-9b04-c09bf0b38722"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:27.060355438Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1900,"client":"2","":"downstairs","session_id":"d653d7f8-4fe5-4ede-9b04-c09bf0b38722"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:27.060381997Z","hostname":Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 "w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1900,"session_id":"d653d7f8-4fe5-4ede-9b04-c09bf0b38722"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:27.060410205Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1900,"session_id":"d653d7f8-4fe5-4ede-9b04-c09bf0b38722"} {"msg":"9e4b22b7-cbf2-40d4-8fc2-0368feebceac is now active with session: d653d7f8-4fe5-4ede-9b04-c09bf0b38722","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:27.060433844Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1900,"session_id":"d653d7f8-4fe5-4ede-9b04-c09bf0b38722"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:27.060447523Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1900,"session_id":"d653d7f8-4fe5-4ede-9b04-c09bf0b38722"} {"msg":"The guest has finished waiting for activation with:180","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:27.060459883Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1900} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 578 len 4096 data: 10 02/30 Read block 520 len 20480 03/30 Write block 159 len 36864 data: 7 7 8 8 12 13 12 12 12 04/30 Write block 159 len 20480 data: 8 8 9 9 13 05/30 Write block 269 len 40960 data: 16 17 17 17 19 19 18 19 16 14 06/30 Flush 07/30 Write block 29 len 8192 data: 7 5 08/30 Write block 393 len 8192 data: 11 11 09/30 Write block 496 len 12288 data: 11 11 12 10/30 Read block 153 len 36864 11/30 Write block 355 len 4096 data: 20 12/30 Read block 329 len 8192 13/30 Write block 31 len 4096 data: 7 14/30 Read block 4 len 20480 15/30 Write block 315 len 24576 data: 17 15 13 14 17 18 16/30 Read block 481 len 8192 17/30 Read block 555 len 40960 18/30 Write block 142 len 32768 data: 12 15 15 16 15 16 14 15 19/30 Read block 212 len 28672 20/30 Write block 240 len 16384 data: 15 14 17 14 21/30 Write block 370 len 24576 data: 13 13 15 13 12 13 22/30 Write block 221 len 20480 data: 17 16 14 15 13 23/30 Read block 51 len 16384 24/30 Read block 59 len 40960 25/30 Read block 250 len 24576 26/30 Write block 529 len 24576 data: 10 10 17 15 16 15 27/30 Read block 80 len 32768 28/30 Read block 0 len 16384 29/30 Read block 278 len 16384 30/30 Read block 253 len 20480 ---------------------------------------------------------------- Crucible gen:180 GIO:true work queues: Upstairs:0 downstairs:24 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1006 Acked Write 2 Done Done Done false 1007 Acked Write 2 Done Done Done false 1008 Acked Write 3 Done Done Done false 1009 Acked Read 9 Done Done Done false 1010 Acked Write 1 Done Done Done false 1011 Acked Read 2 Done Done Done false 1012 Acked Write 1 Done Done Done false 1013 Acked Read 5 Done Done Done false 1014 Acked Write 6 Done Done Done false 1015 Acked Read 2 Done Done Done false 1016 Acked Read 10 Done Done Done false 1017 Acked Write 8 Done Done Done false 1018 Acked Read 7 Done Done Done false 1019 Acked Write 4 Done Done Done false 1020 Acked Write 6 Done Done Done false 1021 Acked Write 5 Done Done Done false 1022 Acked Read 4 Done Done Done false 1023 Acked Read 10 Done Done Done false 1024 Acked Read 6 Done Done Done false 1025 Acked Write 6 Done Done Done false 1026 Acked Read 8 Done Done Done false 1027 Acked Read 4 Done Done Done false 1028 Acked Read 4 Done Done Done false 1029 Acked Read 5 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 0 1 Done 24 23 24 71 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1005 1005 1005 Downstairs last five completed: 1005 1004 1003 1002 1001 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:29:27.080677615Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1900} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 178 178 178 317 317 317 F F F 1 020-039 178 178 178 317 317 317 T T T 2 040-059 179 179 179 318 318 318 F F F 3 060-079 179 179 179 318 318 318 F F F 4 080-099 177 177 177 315 315 315 F F F 5 100-119 174 174 174 310 310 310 F F F 6 120-139 174 174 174 310 310 310 F F F 7 140-159 180 180 180 319 319 319 T T T 8 160-179 180 180 180 319 319 319 F F F 9 180-199 178 178 178 317 317 317 F F F 10 200-219 170 170 170 302 302 302 F F F 11 220-239 175 175 175 313 313 313 T T T 12 240-259 178 178 178 316 316 316 T T T 13 260-279 180 180 180 319 319 319 F F F 14 280-299 177 177 177 315 315 315 F F F 15 300-319 174 174 174 310 310 310 T T T 16 320-339 175 175 175 313 313 313 T T T 17 340-359 178 178 178 317 317 317 T T T 18 360-379 179 179 179 318 318 318 T T T 19 380-399 179 179 179 318 318 318 T T T 20 400-419 178 178 178 317 317 317 F F F 21 420-439 177 177 177 315 315 315 F F F 22 440-459 175 175 175 313 313 313 F F F 23 460-479 177 177 177 315 315 315 F F F 24 480-499 168 168 168 299 299 299 T T T 25 500-519 178 178 178 316 316 316 F F F 26 520-539 177 177 177 315 315 315 T T T 27 540-559 178 178 178 317 317 317 F F F 28 560-579 180 180 180 319 319 319 F F F 29 580-599 168 168 168 299 299 299 F F F Max gen: 180, Max flush: 319 A Difference in extent metadata is expected here On loop 90 Verifying data now No change after verify Loop: 90 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 178 178 178 317 317 317 F F F 1 020-039 181 181 181 320 320 320 F F F 2 040-059 179 179 179 318 318 318 F F F 3 060-079 179 179 179 318 318 318 F F F 4 080-099 177 177 177 315 315 315 F F F 5 100-119 174 174 174 310 310 310 F F F 6 120-139 174 174 174 310 310 310 F F F 7 140-159 181 181 181 320 320 320 F F F 8 160-179 180 180 180 319 319 319 F F F 9 180-199 178 178 178 317 317 317 F F F 10 200-219 170 170 170 302 302 302 F F F 11 220-239 181 181 181 320 320 320 F F F 12 240-259 181 181 181 320 320 320 F F F 13 260-279 180 180 180 319 319 319 F F F 14 280-299 177 177 177 315 315 315 F F F 15 300-319 181 181 181 320 320 320 F F F 16 320-339 181 181 181 320 320 320 F F F 17 340-359 181 181 181 320 320 320 F F F 18 360-379 181 181 181 320 320 320 F F F 19 380-399 181 181 181 320 320 320 F F F 20 400-419 178 178 178 317 317 317 F F F 21 420-439 177 177 177 315 315 315 F F F 22 440-459 175 175 175 313 313 313 F F F 23 460-479 177 177 177 315 315 315 F F F 24 480-499 181 181 181 320 320 320 F F F 25 500-519 178 178 178 316 316 316 F F F 26 520-539 181 181 181 320 320 320 F F F 27 540-559 178 178 178 317 317 317 F F F 28 560-579 180 180 180 319 319 319 F F F 29 580-599 168 168 168 299 299 299 F F F Max gen: 181, Max flush: 320 Begin loop 91 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:31.493241765Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1908,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:31.537645584Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1908,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:31.538450195Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1908} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"0fd98d0d5d909611f8b408ccda2559d35fe117bb\",\n git_commit_timestamp: \"2024-11-08T20:05:44.000000000Z\",\n git_branch: \"mkeeter/drop-skipped-backpressure-guard\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:31.538487183Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1908} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:31.538505452Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1908} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:31.538520352Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1908} {"msg":"Crucible 77eb014b-0a77-4f2e-a686-ee77b2fc25db has session id: ecf97b6d-cae2-40e1-9a70-b578f5945ffd","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:31.538923112Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1908,"session_id":"ecf97b6d-cae2-40e1-9a70-b578f5945ffd"} {"msg":"Upstairs opts: Upstairs UUID: 77eb014b-0a77-4f2e-a686-ee77b2fc25db, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:31.53896633Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1908,"session_id":"ecf97b6d-cae2-40e1-9a70-b578f5945ffd"} {"msg":"Crucible stats registered with UUID: 77eb014b-0a77-4f2e-a686-ee77b2fc25db","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:31.538983919Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1908,"session_id":"ecf97b6d-cae2-40e1-9a70-b578f5945ffd"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:31.570034831Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1908,"task":"crutest"} {"msg":"The guest has requested activation with gen:182","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:31.57006838Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1908} {"msg":"77eb014b-0a77-4f2e-a686-ee77b2fc25db active request set","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:31.570137966Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1908,"session_id":"ecf97b6d-cae2-40e1-9a70-b578f5945ffd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:31.570162905Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1908,"client":"0","":"downstairs","session_id":"ecf97b6d-cae2-40e1-9a70-b578f5945ffd"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:31.570184544Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1908,"client":"0","":"downstairs","session_id":"ecf97b6d-cae2-40e1-9a70-b578f5945ffd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:31.570198693Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1908,"client":"1","":"downstairs","session_id":"ecf97b6d-cae2-40e1-9a70-b578f5945ffd"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:31.570212313Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1908,"client":"1","":"downstairs","session_id":"ecf97b6d-cae2-40e1-9a70-b578f5945ffd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:31.570225902Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1908,"client":"2","":"downstairs","session_id":"ecf97b6d-cae2-40e1-9a70-b578f5945ffd"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:31.570239141Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1908,"client":"2","":"downstairs","session_id":"ecf97b6d-cae2-40e1-9a70-b578f5945ffd"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:31.57025563Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1908,"":"io task","client":"2","":"downstairs","session_id":"ecf97b6d-cae2-40e1-9a70-b578f5945ffd"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:31.57026949Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1908,"":"io task","client":"0","":"downstairs","session_id":"ecf97b6d-cae2-40e1-9a70-b578f5945ffd"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:31.570283049Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1908,"":"io task","client":"1","":"downstairs","session_id":"ecf97b6d-cae2-40e1-9a70-b578f5945ffd"} {"msg":"ds_connection connected from Ok(127.0.0.1:54023)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:31.570461951Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1908,"":"io task","client":"2","":"downstairs","session_id":"ecf97b6d-cae2-40e1-9a70-b578f5945ffd"} {"msg":"ds_connection connected from Ok(127.0.0.1:38029)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:31.570511738Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1908,"":"io task","client":"1","":"downstairs","session_id":"ecf97b6d-cae2-40e1-9a70-b578f5945ffd"} {"msg":"ds_connection connected from Ok(127.0.0.1:52119)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:31.570546986Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1908,"":"io task","client":"0","":"downstairs","session_id":"ecf97b6d-cae2-40e1-9a70-b578f5945ffd"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:31.571273751Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1908,"client":"2","":"downstairs","session_id":"ecf97b6d-cae2-40e1-9a70-b578f5945ffd"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:31.571302319Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1908,"client":"2","":"downstairs","session_id":"ecf97b6d-cae2-40e1-9a70-b578f5945ffd"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:31.571317109Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1908,"client":"0","":"downstairs","session_id":"ecf97b6d-cae2-40e1-9a70-b578f5945ffd"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:31.571331008Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1908,"client":"0","":"downstairs","session_id":"ecf97b6d-cae2-40e1-9a70-b578f5945ffd"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:31.571347307Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1908,"client":"1","":"downstairs","session_id":"ecf97b6d-cae2-40e1-9a70-b578f5945ffd"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:31.571386925Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1908,"client":"1","":"downstairs","session_id":"ecf97b6d-cae2-40e1-9a70-b578f5945ffd"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:31.572011785Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1908,"client":"0","":"downstairs","session_id":"ecf97b6d-cae2-40e1-9a70-b578f5945ffd"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:31.572139469Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1908,"client":"1","":"downstairs","session_id":"ecf97b6d-cae2-40e1-9a70-b578f5945ffd"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:31.572177696Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1908,"client":"2","":"downstairs","session_id":"ecf97b6d-cae2-40e1-9a70-b578f5945ffd"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:31.572579217Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1908,"client":"0","":"downstairs","session_id":"ecf97b6d-cae2-40e1-9a70-b578f5945ffd"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:31.572610246Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1908,"client":"0","":"downstairs","session_id":"ecf97b6d-cae2-40e1-9a70-b578f5945ffd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:31.572625085Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1908,"session_id":"ecf97b6d-cae2-40e1-9a70-b578f5945ffd"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:31.572637834Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1908,"client":"1","":"downstairs","session_id":"ecf97b6d-cae2-40e1-9a70-b578f5945ffd"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:31.572651413Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1908,"client":"1","":"downstairs","session_id":"ecf97b6d-cae2-40e1-9a70-b578f5945ffd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:31.572664753Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1908,"session_id":"ecf97b6d-cae2-40e1-9a70-b578f5945ffd"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:31.57272579Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1908,"client":"2","":"downstairs","session_id":"ecf97b6d-cae2-40e1-9a70-b578f5945ffd"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:31.572744689Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1908,"client":"2","":"downstairs","session_id":"ecf97b6d-cae2-40e1-9a70-b578f5945ffd"} {"msg":"[0]R flush_numbers[0..12]: [317, 320, 318, 318, 315, 310, 310, 320, 319, 317, 302, 320]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:31.572758788Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1908,"":"downstairs","":"downstairs","session_id":"ecf97b6d-cae2-40e1-9a70-b578f5945ffd"} {"msg":"[0]R generation[0..12]: [178, 181, 179, 179, 177, 174, 174, 181, 180, 178, 170, 181]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:31.572772547Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1908,"":"downstairs","":"downstairs","session_id":"ecf97b6d-cae2-40e1-9a70-b578f5945ffd"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:31.572786397Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1908,"":"downstairs","":"downstairs","session_id":"ecf97b6d-cae2-40e1-9a70-b578f5945ffd"} {"msg":"[1]R flush_numbers[0..12]: [317, 320, 318, 318, 315, 310, 310, 320, 319, 317, 302, 320]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:31.572800286Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1908,"":"downstairs","":"downstairs","session_id":"ecf97b6d-cae2-40e1-9a70-b578f5945ffd"} {"msg":"[1]R generation[0..12]: [178, 181, 179, 179, 177, 174, 174, 181, 180, 178, 170, 181]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:31.572814225Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1908,"":"downstairs","":"downstairs","session_id":"ecf97b6d-cae2-40e1-9a70-b578f5945ffd"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:31.572827885Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1908,"":"downstairs","":"downstairs","session_id":"ecf97b6d-cae2-40e1-9a70-b578f5945ffd"} {"msg":"[2]R flush_numbers[0..12]: [317, 320, 318, 318, 315, 310, 310, 320, 319, 317, 302, 320]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:31.572841394Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1908,"":"downstairs","":"downstairs","session_id":"ecf97b6d-cae2-40e1-9a70-b578f5945ffd"} {"msg":"[2]R generation[0..12]: [178, 181, 179, 179, 177, 174, 174, 181, 180, 178, 170, 181]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:31.572857813Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1908,"":"downstairs","":"downstairs","session_id":"ecf97b6d-cae2-40e1-9a70-b578f5945ffd"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:31.572871753Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1908,"":"downstairs","":"downstairs","session_id":"ecf97b6d-cae2-40e1-9a70-b578f5945ffd"} {"msg":"Max found gen is 182","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:31.572884652Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1908,"":"downstairs","":"downstairs","session_id":"ecf97b6d-cae2-40e1-9a70-b578f5945ffd"} {"msg":"Generation requested: 182 >= found:182","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:31.572897791Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1908,"":"downstairs","":"downstairs","session_id":"ecf97b6d-cae2-40e1-9a70-b578f5945ffd"} {"msg":"Next flush: 321","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:31.572910711Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1908,"":"downstairs","":"downstairs","session_id":"ecf97b6d-cae2-40e1-9a70-b578f5945ffd"} {"msg":"All extents match","v":0,"name":"crucible","level"Wait for a query_work_queue command to finish before sending IO :30,"time":"2024-11-08T20:29:31.57293124Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1908,"":"downstairs","":"downstairs","session_id":"ecf97b6d-cae2-40e1-9a70-b578f5945ffd"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:31.572950839Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1908,"session_id":"ecf97b6d-cae2-40e1-9a70-b578f5945ffd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:31.572974248Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1908,"client":"0","":"downstairs","session_id":"ecf97b6d-cae2-40e1-9a70-b578f5945ffd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:31.573016195Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1908,"client":"1","":"downstairs","session_id":"ecf97b6d-cae2-40e1-9a70-b578f5945ffd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:31.573045304Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1908,"client":"2","":"downstairs","session_id":"ecf97b6d-cae2-40e1-9a70-b578f5945ffd"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2024-11-08T20:29:31.573071833Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1908,"session_id":"ecf97b6d-cae2-40e1-9a70-b578f5945ffd"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:31.573096882Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1908,"session_id":"ecf97b6d-cae2-40e1-9a70-b578f5945ffd"} {"msg":"77eb014b-0a77-4f2e-a686-ee77b2fc25db is now active with session: ecf97b6d-cae2-40e1-9a70-b578f5945ffd","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:31.573117971Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1908,"session_id":"ecf97b6d-cae2-40e1-9a70-b578f5945ffd"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:31.57313101Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1908,"session_id":"ecf97b6d-cae2-40e1-9a70-b578f5945ffd"} {"msg":"The guest has finished waiting for activation with:182","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:31.573143599Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1908} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 223 len 24576 data: 15 16 14 16 18 14 02/30 Read block 566 len 28672 03/30 Write block 90 len 36864 data: 13 11 13 13 12 13 11 10 8 04/30 Read block 350 len 8192 05/30 Write block 277 len 4096 data: 17 06/30 Write block 538 len 20480 data: 14 14 16 16 16 07/30 Read block 88 len 28672 08/30 Write block 283 len 12288 data: 14 15 18 09/30 Read block 457 len 32768 10/30 Read block 305 len 4096 11/30 Write block 444 len 16384 data: 12 12 15 15 12/30 Read block 264 len 40960 13/30 Read block 61 len 28672 14/30 Read block 367 len 20480 15/30 Write block 269 len 8192 data: 17 18 16/30 Write block 576 len 12288 data: 11 11 11 17/30 Read block 285 len 4096 18/30 Read block 48 len 28672 19/30 Read block 496 len 36864 20/30 Read block 454 len 36864 21/30 Read block 321 len 4096 22/30 Read block 373 len 28672 23/30 Write block 370 len 4096 data: 14 24/30 Write block 593 len 28672 data: 11 9 10 8 6 4 4 25/30 Write block 339 len 36864 data: 17 16 16 18 17 15 16 17 15 26/30 Read block 489 len 8192 27/30 Read block 552 len 36864 28/30 Write block 384 len 24576 data: 11 9 8 10 10 10 29/30 Write block 338 len 28672 data: 17 18 17 17 19 18 16 30/30 Write block 41 len 32768 data: 12 16 15 13 12 11 7 10 ---------------------------------------------------------------- Crucible gen:182 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 6 Done Done Done false 1001 Acked Read 7 Done Done Done false 1002 Acked Write 9 Done Done Done false 1003 Acked Read 2 Done Done Done false 1004 Acked Write 1 Done Done Done false 1005 Acked Write 5 Done Done Done false 1006 Acked Read 7 Done Done Done false 1007 Acked Write 3 Done Done Done false 1008 Acked Read 8 Done Done Done false 1009 Acked Read 1 Done Done Done false 1010 Acked Write 4 Done Done Done false 1011 Acked Read 10 Done Done Done false 1012 Acked Read 7 Done Done Done false 1013 Acked Read 5 Done Done Done false 1014 Acked Write 2 Done Done Done false 1015 Acked Write 3 Done Done Done false 1016 Acked Read 1 Done Done Done false 1017 Acked Read 7 Done Done Done false 1018 Acked Read 9 Done Done Done false 1019 Acked Read 9 Done Done Done false 1020 Acked Read 1 Done Done Done false 1021 Acked Read 7 Done Done Done false 1022 Acked Write 1 Done Done Done false 1023 Acked Write 7 Done Done Done false 1024 Acked Write 9 Done Done Done false 1025 Acked Read 2 Done Done Done false 1026 Acked Read 9 Done Done Done false 1027 Acked Write 6 Done Done Done false 1028 Acked Write 7 Done Done Sent false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 2 4 Done 29 29 28 86 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 0 0 0 Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:29:31.592284474Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1908} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 178 178 178 317 317 317 F F F 1 020-039 181 181 181 320 320 320 F F F 2 040-059 179 179 179 318 318 318 T T T 3 060-079 179 179 179 318 318 318 F F F 4 080-099 177 177 177 315 315 315 T T T 5 100-119 174 174 174 310 310 310 F F F 6 120-139 174 174 174 310 310 310 F F F 7 140-159 181 181 181 320 320 320 F F F 8 160-179 180 180 180 319 319 319 F F F 9 180-199 178 178 178 317 317 317 F F F 10 200-219 170 170 170 302 302 302 F F F 11 220-239 181 181 181 320 320 320 T T T 12 240-259 181 181 181 320 320 320 F F F 13 260-279 180 180 180 319 319 319 T T T 14 280-299 177 177 177 315 315 315 T T T 15 300-319 181 181 181 320 320 320 F F F 16 320-339 181 181 181 320 320 320 T T T 17 340-359 181 181 181 320 320 320 T T T 18 360-379 181 181 181 320 320 320 T T T 19 380-399 181 181 181 320 320 320 T T T 20 400-419 178 178 178 317 317 317 F F F 21 420-439 177 177 177 315 315 315 F F F 22 440-459 175 175 175 313 313 313 T T T 23 460-479 177 177 177 315 315 315 F F F 24 480-499 181 181 181 320 320 320 F F F 25 500-519 178 178 178 316 316 316 F F F 26 520-539 181 181 181 320 320 320 T T T 27 540-559 178 178 178 317 317 317 T T T 28 560-579 180 180 180 319 319 319 T T T 29 580-599 168 168 168 299 299 299 T T T Max gen: 181, Max flush: 320 A Difference in extent metadata is expected here On loop 91 Verifying data now No change after verify Loop: 91 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 178 178 178 317 317 317 F F F 1 020-039 181 181 181 320 320 320 F F F 2 040-059 182 182 182 321 321 321 F F F 3 060-079 179 179 179 318 318 318 F F F 4 080-099 182 182 182 321 321 321 F F F 5 100-119 174 174 174 310 310 310 F F F 6 120-139 174 174 174 310 310 310 F F F 7 140-159 181 181 181 320 320 320 F F F 8 160-179 180 180 180 319 319 319 F F F 9 180-199 178 178 178 317 317 317 F F F 10 200-219 170 170 170 302 302 302 F F F 11 220-239 182 182 182 321 321 321 F F F 12 240-259 181 181 181 320 320 320 F F F 13 260-279 182 182 182 321 321 321 F F F 14 280-299 182 182 182 321 321 321 F F F 15 300-319 181 181 181 320 320 320 F F F 16 320-339 182 182 182 321 321 321 F F F 17 340-359 182 182 182 321 321 321 F F F 18 360-379 182 182 182 321 321 321 F F F 19 380-399 182 182 182 321 321 321 F F F 20 400-419 178 178 178 317 317 317 F F F 21 420-439 177 177 177 315 315 315 F F F 22 440-459 182 182 182 321 321 321 F F F 23 460-479 177 177 177 315 315 315 F F F 24 480-499 181 181 181 320 320 320 F F F 25 500-519 178 178 178 316 316 316 F F F 26 520-539 182 182 182 321 321 321 F F F 27 540-559 182 182 182 321 321 321 F F F 28 560-579 182 182 182 321 321 321 F F F 29 580-599 182 182 182 321 321 321 F F F Max gen: 182, Max flush: 321 Begin loop 92 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:36.051071774Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1916,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:36.095481034Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1916,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:36.096187839Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1916} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"0fd98d0d5d909611f8b408ccda2559d35fe117bb\",\n git_commit_timestamp: \"2024-11-08T20:05:44.000000000Z\",\n git_branch: \"mkeeter/drop-skipped-backpressure-guard\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:36.096230857Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1916} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:36.096246366Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1916} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:36.096262425Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1916} {"msg":"Crucible f261aeb6-abc6-436e-b015-8d38b55ebc31 has session id: b3a64b0b-17b5-4a5b-8d37-e85b0ada2d9c","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:36.096687715Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1916,"session_id":"b3a64b0b-17b5-4a5b-8d37-e85b0ada2d9c"} {"msg":"Upstairs opts: Upstairs UUID: f261aeb6-abc6-436e-b015-8d38b55ebc31, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:36.096735222Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1916,"session_id":"b3a64b0b-17b5-4a5b-8d37-e85b0ada2d9c"} {"msg":"Crucible stats registered with UUID: f261aeb6-abc6-436e-b015-8d38b55ebc31","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:36.096760981Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1916,"session_id":"b3a64b0b-17b5-4a5b-8d37-e85b0ada2d9c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:36.127768816Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1916,"task":"crutest"} {"msg":"The guest has requested activation with gen:184","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:36.127804284Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1916} {"msg":"f261aeb6-abc6-436e-b015-8d38b55ebc31 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:36.127838682Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1916,"session_id":"b3a64b0b-17b5-4a5b-8d37-e85b0ada2d9c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:36.127856631Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1916,"client":"0","":"downstairs","session_id":"b3a64b0b-17b5-4a5b-8d37-e85b0ada2d9c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:36.12788478Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1916,"client":"0","":"downstairs","session_id":"b3a64b0b-17b5-4a5b-8d37-e85b0ada2d9c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:36.127902809Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1916,"client":"1","":"downstairs","session_id":"b3a64b0b-17b5-4a5b-8d37-e85b0ada2d9c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:36.127919988Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1916,"client":"1","":"downstairs","session_id":"b3a64b0b-17b5-4a5b-8d37-e85b0ada2d9c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:36.127949997Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1916,"client":"2","":"downstairs","session_id":"b3a64b0b-17b5-4a5b-8d37-e85b0ada2d9c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:36.128075931Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1916,"client":"2","":"downstairs","session_id":"b3a64b0b-17b5-4a5b-8d37-e85b0ada2d9c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:36.128118128Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1916,"":"io task","client":"2","":"downstairs","session_id":"b3a64b0b-17b5-4a5b-8d37-e85b0ada2d9c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:36.128146777Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1916,"":"io task","client":"0","":"downstairs","session_id":"b3a64b0b-17b5-4a5b-8d37-e85b0ada2d9c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:36.128172716Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1916,"":"io task","client":"1","":"downstairs","session_id":"b3a64b0b-17b5-4a5b-8d37-e85b0ada2d9c"} {"msg":"ds_connection connected from Ok(127.0.0.1:60866)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:36.128309489Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1916,"":"io task","client":"1","":"downstairs","session_id":"b3a64b0b-17b5-4a5b-8d37-e85b0ada2d9c"} {"msg":"ds_connection connected from Ok(127.0.0.1:40840)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:36.128341828Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1916,"":"io task","client":"0","":"downstairs","session_id":"b3a64b0b-17b5-4a5b-8d37-e85b0ada2d9c"} {"msg":"ds_connection connected from Ok(127.0.0.1:39620)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:36.128370316Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1916,"":"io task","client":"2","":"downstairs","session_id":"b3a64b0b-17b5-4a5b-8d37-e85b0ada2d9c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:36.128888321Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1916,"client":"0","":"downstairs","session_id":"b3a64b0b-17b5-4a5b-8d37-e85b0ada2d9c"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:36.128922169Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1916,"client":"0","":"downstairs","session_id":"b3a64b0b-17b5-4a5b-8d37-e85b0ada2d9c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:36.128938388Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1916,"client":"2","":"downstairs","session_id":"b3a64b0b-17b5-4a5b-8d37-e85b0ada2d9c"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:36.128952618Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1916,"client":"2","":"downstairs","session_id":"b3a64b0b-17b5-4a5b-8d37-e85b0ada2d9c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:36.128966757Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1916,"client":"1","":"downstairs","session_id":"b3a64b0b-17b5-4a5b-8d37-e85b0ada2d9c"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:36.129022744Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1916,"client":"1","":"downstairs","session_id":"b3a64b0b-17b5-4a5b-8d37-e85b0ada2d9c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:36.129561428Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1916,"client":"0","":"downstairs","session_id":"b3a64b0b-17b5-4a5b-8d37-e85b0ada2d9c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:36.129597276Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1916,"client":"1","":"downstairs","session_id":"b3a64b0b-17b5-4a5b-8d37-e85b0ada2d9c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:36.129631455Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1916,"client":"2","":"downstairs","session_id":"b3a64b0b-17b5-4a5b-8d37-e85b0ada2d9c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:36.130153469Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1916,"client":"0","":"downstairs","session_id":"b3a64b0b-17b5-4a5b-8d37-e85b0ada2d9c"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:36.130182258Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1916,"client":"0","":"downstairs","session_id":"b3a64b0b-17b5-4a5b-8d37-e85b0ada2d9c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:36.130197997Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1916,"session_id":"b3a64b0b-17b5-4a5b-8d37-e85b0ada2d9c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:36.130272623Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1916,"client":"1","":"downstairs","session_id":"b3a64b0b-17b5-4a5b-8d37-e85b0ada2d9c"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:36.130298952Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1916,"client":"1","":"downstairs","session_id":"b3a64b0b-17b5-4a5b-8d37-e85b0ada2d9c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:36.130313071Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1916,"session_id":"b3a64b0b-17b5-4a5b-8d37-e85b0ada2d9c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:36.130325491Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1916,"client":"2","":"downstairs","session_id":"b3a64b0b-17b5-4a5b-8d37-e85b0ada2d9c"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:36.13033923Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1916,"client":"2","":"downstairs","session_id":"b3a64b0b-17b5-4a5b-8d37-e85b0ada2d9c"} {"msg":"[0]R flush_numbers[0..12]: [317, 320, 321, 318, 321, 310, 310, 320, 319, 317, 302, 321]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:36.130353379Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1916,"":"downstairs","":"downstairs","session_id":"b3a64b0b-17b5-4a5b-8d37-e85b0ada2d9c"} {"msg":"[0]R generation[0..12]: [178, 181, 182, 179, 182, 174, 174, 181, 180, 178, 170, 182]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:36.130367279Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1916,"":"downstairs","":"downstairs","session_id":"b3a64b0b-17b5-4a5b-8d37-e85b0ada2d9c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:36.130381618Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1916,"":"downstairs","":"downstairs","session_id":"b3a64b0b-17b5-4a5b-8d37-e85b0ada2d9c"} {"msg":"[1]R flush_numbers[0..12]: [317, 320, 321, 318, 321, 310, 310, 320, 319, 317, 302, 321]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:36.130395787Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1916,"":"downstairs","":"downstairs","session_id":"b3a64b0b-17b5-4a5b-8d37-e85b0ada2d9c"} {"msg":"[1]R generation[0..12]: [178, 181, 182, 179, 182, 174, 174, 181, 180, 178, 170, 182]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:36.130409747Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1916,"":"downstairs","":"downstairs","session_id":"b3a64b0b-17b5-4a5b-8d37-e85b0ada2d9c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:36.130423946Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1916,"":"downstairs","":"downstairs","session_id":"b3a64b0b-17b5-4a5b-8d37-e85b0ada2d9c"} {"msg":"[2]R flush_numbers[0..12]: [317, 320, 321, 318, 321, 310, 310, 320, 319, 317, 302, 321]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:36.130441725Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1916,"":"downstairs","":"downstairs","session_id":"b3a64b0b-17b5-4a5b-8d37-e85b0ada2d9c"} {"msg":"[2]R generation[0..12]: [178, 181, 182, 179, 182, 174, 174, 181, 180, 178, 170, 182]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:36.130455734Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1916,"":"downstairs","":"downstairs","session_id":"b3a64b0b-17b5-4a5b-8d37-e85b0ada2d9c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:36.130469784Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1916,"":"downstairs","":"downstairs","session_id":"b3a64b0b-17b5-4a5b-8d37-e85b0ada2d9c"} {"msg":"Max found gen is 183","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:36.130482913Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1916,"":"downstairs","":"downstairs","session_id":"b3a64b0b-17b5-4a5b-8d37-e85b0ada2d9c"} {"msg":"Generation requested: 184 >= found:183","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:36.130496282Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1916,"":"downstairs","":"downstairs","session_id":"b3a64b0b-17b5-4a5b-8d37-e85b0ada2d9c"} {"msg":"Next flush: 322","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:36.130509262Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1916,"":"downstairs","":"downstairs","session_id":"b3a64b0b-17b5-4a5b-8d37-e85b0ada2d9c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:36.130522191Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1916,"":"downstairs","":"downstairs","Wait for a query_work_queue command to finish before sending IO session_id":"b3a64b0b-17b5-4a5b-8d37-e85b0ada2d9c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:36.13054412Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1916,"session_id":"b3a64b0b-17b5-4a5b-8d37-e85b0ada2d9c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:36.130561829Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1916,"client":"0","":"downstairs","session_id":"b3a64b0b-17b5-4a5b-8d37-e85b0ada2d9c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:36.130578698Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1916,"client":"1","":"downstairs","session_id":"b3a64b0b-17b5-4a5b-8d37-e85b0ada2d9c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:36.130592188Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1916,"client":"2","":"downstairs","session_id":"b3a64b0b-17b5-4a5b-8d37-e85b0ada2d9c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:36.130605607Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1916,"session_id":"b3a64b0b-17b5-4a5b-8d37-e85b0ada2d9c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:36.130618016Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1916,"session_id":"b3a64b0b-17b5-4a5b-8d37-e85b0ada2d9c"} {"msg":"f261aeb6-abc6-436e-b015-8d38b55ebc31 is now active with session: b3a64b0b-17b5-4a5b-8d37-e85b0ada2d9c","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:36.130630796Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1916,"session_id":"b3a64b0b-17b5-4a5b-8d37-e85b0ada2d9c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:36.130642755Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1916,"session_id":"b3a64b0b-17b5-4a5b-8d37-e85b0ada2d9c"} {"msg":"The guest has finished waiting for activation with:184","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:36.130657184Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1916} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 16 len 8192 data: 16 15 02/30 Read block 258 len 4096 03/30 Write block 315 len 4096 data: 18 04/30 Read block 421 len 28672 05/30 Read block 446 len 4096 06/30 Write block 325 len 20480 data: 11 14 9 11 11 07/30 Read block 283 len 12288 08/30 Read block 519 len 20480 09/30 Read block 337 len 12288 10/30 Read block 24 len 24576 11/30 Write block 476 len 8192 data: 18 21 12/30 Write block 299 len 24576 data: 19 17 17 16 16 15 13/30 Write block 551 len 36864 data: 7 10 9 11 13 13 13 14 14 14/30 Write block 261 len 28672 data: 9 10 11 12 15 14 12 15/30 Write block 514 len 36864 data: 14 12 11 9 13 15 12 13 12 16/30 Read block 97 len 16384 17/30 Read block 366 len 16384 18/30 Write block 391 len 28672 data: 10 10 12 12 11 11 13 19/30 Read block 526 len 4096 20/30 Read block 530 len 4096 21/30 Read block 33 len 16384 22/30 Read block 324 len 8192 23/30 Write block 304 len 28672 data: 16 13 13 9 10 11 13 24/30 Write block 193 len 28672 data: 14 14 13 13 13 13 11 25/30 Read block 584 len 20480 26/30 Read block 303 len 28672 27/30 Write block 127 len 24576 data: 10 10 8 11 11 11 28/30 Read block 585 len 40960 29/30 Read block 580 len 28672 30/30 Read block 51 len 12288 ---------------------------------------------------------------- Crucible gen:184 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 2 Done Done Done false 1001 Acked Read 1 Done Done Done false 1002 Acked Write 1 Done Done Done false 1003 Acked Read 7 Done Done Done false 1004 Acked Read 1 Done Done Done false 1005 Acked Write 5 Done Done Done false 1006 Acked Read 3 Done Done Done false 1007 Acked Read 5 Done Done Done false 1008 Acked Read 3 Done Done Done false 1009 Acked Read 6 Done Done Done false 1010 Acked Write 2 Done Done Done false 1011 Acked Write 6 Done Done Done false 1012 Acked Write 9 Done Done Done false 1013 Acked Write 7 Done Done Done false 1014 Acked Write 9 Done Done Done false 1015 Acked Read 4 Done Done Done false 1016 Acked Read 4 Done Done Done false 1017 Acked Write 7 Done Done Done false 1018 Acked Read 1 Done Done Done false 1019 Acked Read 1 Done Done Done false 1020 Acked Read 4 Done Done Done false 1021 Acked Read 2 Done Done Done false 1022 Acked Write 7 Done Done Done false 1023 Acked Write 7 Done Done Done false 1024 Acked Read 5 Done Done Done false 1025 Acked Read 7 Done Done Done false 1026 Acked Write 6 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Read 7 Done Done Done false 1029 Acked Read 3 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 30 30 29 89 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 0 0 0 Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:29:36.149899514Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1916} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 178 178 178 317 317 317 T T T 1 020-039 181 181 181 320 320 320 F F F 2 040-059 182 182 182 321 321 321 F F F 3 060-079 179 179 179 318 318 318 F F F 4 080-099 182 182 182 321 321 321 F F F 5 100-119 174 174 174 310 310 310 F F F 6 120-139 174 174 174 310 310 310 T T T 7 140-159 181 181 181 320 320 320 F F F 8 160-179 180 180 180 319 319 319 F F F 9 180-199 178 178 178 317 317 317 T T T 10 200-219 170 170 170 302 302 302 F F F 11 220-239 182 182 182 321 321 321 F F F 12 240-259 181 181 181 320 320 320 F F F 13 260-279 182 182 182 321 321 321 T T T 14 280-299 182 182 182 321 321 321 T T T 15 300-319 181 181 181 320 320 320 T T T 16 320-339 182 182 182 321 321 321 T T T 17 340-359 182 182 182 321 321 321 F F F 18 360-379 182 182 182 321 321 321 F F F 19 380-399 182 182 182 321 321 321 T T T 20 400-419 178 178 178 317 317 317 F F F 21 420-439 177 177 177 315 315 315 F F F 22 440-459 182 182 182 321 321 321 F F F 23 460-479 177 177 177 315 315 315 T T T 24 480-499 181 181 181 320 320 320 F F F 25 500-519 178 178 178 316 316 316 T T T 26 520-539 182 182 182 321 321 321 T T T 27 540-559 182 182 182 321 321 321 T T T 28 560-579 182 182 182 321 321 321 F F F 29 580-599 182 182 182 321 321 321 F F F Max gen: 182, Max flush: 321 A Difference in extent metadata is expected here On loop 92 Verifying data now No change after verify Loop: 92 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 183 183 183 322 322 322 F F F 1 020-039 181 181 181 320 320 320 F F F 2 040-059 182 182 182 321 321 321 F F F 3 060-079 179 179 179 318 318 318 F F F 4 080-099 182 182 182 321 321 321 F F F 5 100-119 174 174 174 310 310 310 F F F 6 120-139 183 183 183 322 322 322 F F F 7 140-159 181 181 181 320 320 320 F F F 8 160-179 180 180 180 319 319 319 F F F 9 180-199 183 183 183 322 322 322 F F F 10 200-219 170 170 170 302 302 302 F F F 11 220-239 182 182 182 321 321 321 F F F 12 240-259 181 181 181 320 320 320 F F F 13 260-279 183 183 183 322 322 322 F F F 14 280-299 183 183 183 322 322 322 F F F 15 300-319 183 183 183 322 322 322 F F F 16 320-339 183 183 183 322 322 322 F F F 17 340-359 182 182 182 321 321 321 F F F 18 360-379 182 182 182 321 321 321 F F F 19 380-399 183 183 183 322 322 322 F F F 20 400-419 178 178 178 317 317 317 F F F 21 420-439 177 177 177 315 315 315 F F F 22 440-459 182 182 182 321 321 321 F F F 23 460-479 183 183 183 322 322 322 F F F 24 480-499 181 181 181 320 320 320 F F F 25 500-519 183 183 183 322 322 322 F F F 26 520-539 183 183 183 322 322 322 F F F 27 540-559 183 183 183 322 322 322 F F F 28 560-579 182 182 182 321 321 321 F F F 29 580-599 182 182 182 321 321 321 F F F Max gen: 183, Max flush: 322 Begin loop 93 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:40.564039016Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1924,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:29:40.606965498Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1924,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:40.607374098Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1924,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:40.639595314Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1924,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:40.640197084Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1924} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"0fd98d0d5d909611f8b408ccda2559d35fe117bb\",\n git_commit_timestamp: \"2024-11-08T20:05:44.000000000Z\",\n git_branch: \"mkeeter/drop-skipped-backpressure-guard\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:40.640233382Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1924} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:40.640247462Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1924} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:40.640261821Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1924} {"msg":"Crucible fb7cb9de-4d10-4a11-97c8-54b51b66c1c0 has session id: e1bac113-3c75-402f-8614-71f11e16c64f","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:40.640644642Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1924,"session_id":"e1bac113-3c75-402f-8614-71f11e16c64f"} {"msg":"Upstairs opts: Upstairs UUID: fb7cb9de-4d10-4a11-97c8-54b51b66c1c0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:40.64069201Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1924,"session_id":"e1bac113-3c75-402f-8614-71f11e16c64f"} {"msg":"Crucible stats registered with UUID: fb7cb9de-4d10-4a11-97c8-54b51b66c1c0","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:40.640717769Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1924,"session_id":"e1bac113-3c75-402f-8614-71f11e16c64f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:40.671911854Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1924,"task":"crutest"} {"msg":"The guest has requested activation with gen:186","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:40.671944732Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1924} {"msg":"fb7cb9de-4d10-4a11-97c8-54b51b66c1c0 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:40.671979241Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1924,"session_id":"e1bac113-3c75-402f-8614-71f11e16c64f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:40.67199609Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1924,"client":"0","":"downstairs","session_id":"e1bac113-3c75-402f-8614-71f11e16c64f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:40.672028379Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1924,"client":"0","":"downstairs","session_id":"e1bac113-3c75-402f-8614-71f11e16c64f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:40.672048118Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1924,"client":"1","":"downstairs","session_id":"e1bac113-3c75-402f-8614-71f11e16c64f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:40.672065087Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1924,"client":"1","":"downstairs","session_id":"e1bac113-3c75-402f-8614-71f11e16c64f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:40.672081686Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1924,"client":"2","":"downstairs","session_id":"e1bac113-3c75-402f-8614-71f11e16c64f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:40.672098515Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1924,"client":"2","":"downstairs","session_id":"e1bac113-3c75-402f-8614-71f11e16c64f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:40.672125194Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1924,"":"io task","client":"0","":"downstairs","session_id":"e1bac113-3c75-402f-8614-71f11e16c64f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:40.672145723Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1924,"":"io task","client":"2","":"downstairs","session_id":"e1bac113-3c75-402f-8614-71f11e16c64f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:40.672163032Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1924,"":"io task","client":"1","":"downstairs","session_id":"e1bac113-3c75-402f-8614-71f11e16c64f"} {"msg":"ds_connection connected from Ok(127.0.0.1:62570)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:40.672270567Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1924,"":"io task","client":"0","":"downstairs","session_id":"e1bac113-3c75-402f-8614-71f11e16c64f"} {"msg":"ds_connection connected from Ok(127.0.0.1:64569)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:40.672494166Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1924,"":"io task","client":"1","":"downstairs","session_id":"e1bac113-3c75-402f-8614-71f11e16c64f"} {"msg":"ds_connection connected from Ok(127.0.0.1:50856)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:40.672594051Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1924,"":"io task","client":"2","":"downstairs","session_id":"e1bac113-3c75-402f-8614-71f11e16c64f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:40.673097836Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1924,"client":"1","":"downstairs","session_id":"e1bac113-3c75-402f-8614-71f11e16c64f"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:40.673127035Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1924,"client":"1","":"downstairs","session_id":"e1bac113-3c75-402f-8614-71f11e16c64f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:40.673141944Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1924,"client":"2","":"downstairs","session_id":"e1bac113-3c75-402f-8614-71f11e16c64f"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:40.673155523Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1924,"client":"2","":"downstairs","session_id":"e1bac113-3c75-402f-8614-71f11e16c64f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:40.673173983Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1924,"client":"0","":"downstairs","session_id":"e1bac113-3c75-402f-8614-71f11e16c64f"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:40.673205021Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1924,"client":"0","":"downstairs","session_id":"e1bac113-3c75-402f-8614-71f11e16c64f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:40.673750634Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1924,"client":"1","":"downstairs","session_id":"e1bac113-3c75-402f-8614-71f11e16c64f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:40.673782403Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1924,"client":"2","":"downstairs","session_id":"e1bac113-3c75-402f-8614-71f11e16c64f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:40.673805322Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1924,"client":"0","":"downstairs","session_id":"e1bac113-3c75-402f-8614-71f11e16c64f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:40.674397653Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1924,"client":"1","":"downstairs","session_id":"e1bac113-3c75-402f-8614-71f11e16c64f"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:40.674426691Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1924,"client":"1","":"downstairs","session_id":"e1bac113-3c75-402f-8614-71f11e16c64f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:40.67444155Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1924,"session_id":"e1bac113-3c75-402f-8614-71f11e16c64f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:40.6744543Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1924,"client":"0","":"downstairs","session_id":"e1bac113-3c75-402f-8614-71f11e16c64f"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:40.674468029Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1924,"client":"0","":"downstairs","session_id":"e1bac113-3c75-402f-8614-71f11e16c64f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:40.674481209Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1924,"session_id":"e1bac113-3c75-402f-8614-71f11e16c64f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:40.674493508Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1924,"client":"2","":"downstairs","session_id":"e1bac113-3c75-402f-8614-71f11e16c64f"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:40.674507047Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1924,"client":"2","":"downstairs","session_id":"e1bac113-3c75-402f-8614-71f11e16c64f"} {"msg":"[0]R flush_numbers[0..12]: [322, 320, 321, 318, 321, 310, 322, 320, 319, 322, 302, 321]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:40.674520987Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1924,"":"downstairs","":"downstairs","session_id":"e1bac113-3c75-402f-8614-71f11e16c64f"} {"msg":"[0]R generation[0..12]: [183, 181, 182, 179, 182, 174, 183, 181, 180, 183, 170, 182]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:40.674535786Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1924,"":"downstairs","":"downstairs","session_id":"e1bac113-3c75-402f-8614-71f11e16c64f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:40.674550105Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1924,"":"downstairs","":"downstairs","session_id":"e1bac113-3c75-402f-8614-71f11e16c64f"} {"msg":"[1]R flush_numbers[0..12]: [322, 320, 321, 318, 321, 310, 322, 320, 319, 322, 302, 321]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:40.674568124Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1924,"":"downstairs","":"downstairs","session_id":"e1bac113-3c75-402f-8614-71f11e16c64f"} {"msg":"[1]R generation[0..12]: [183, 181, 182, 179, 182, 174, 183, 181, 180, 183, 170, 182]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:40.674582654Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1924,"":"downstairs","":"downstairs","session_id":"e1bac113-3c75-402f-8614-71f11e16c64f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:40.674596793Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1924,"":"downstairs","":"downstairs","session_id":"e1bac113-3c75-402f-8614-71f11e16c64f"} {"msg":"[2]R flush_numbers[0..12]: [322, 320, 321, 318, 321, 310, 322, 320, 319, 322, 302, 321]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:40.674610702Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1924,"":"downstairs","":"downstairs","session_id":"e1bac113-3c75-402f-8614-71f11e16c64f"} {"msg":"[2]R generation[0..12]: [183, 181, 182, 179, 182, 174, 183, 181, 180, 183, 170, 182]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:40.674624312Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1924,"":"downstairs","":"downstairs","session_id":"e1bac113-3c75-402f-8614-71f11e16c64f"} {Wait for a query_work_queue command to finish before sending IO "msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:40.67464618Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1924,"":"downstairs","":"downstairs","session_id":"e1bac113-3c75-402f-8614-71f11e16c64f"} {"msg":"Max found gen is 184","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:40.6746595Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1924,"":"downstairs","":"downstairs","session_id":"e1bac113-3c75-402f-8614-71f11e16c64f"} {"msg":"Generation requested: 186 >= found:184","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:40.674672859Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1924,"":"downstairs","":"downstairs","session_id":"e1bac113-3c75-402f-8614-71f11e16c64f"} {"msg":"Next flush: 323","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:40.674685779Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1924,"":"downstairs","":"downstairs","session_id":"e1bac113-3c75-402f-8614-71f11e16c64f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:40.674698638Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1924,"":"downstairs","":"downstairs","session_id":"e1bac113-3c75-402f-8614-71f11e16c64f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:40.674711747Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1924,"session_id":"e1bac113-3c75-402f-8614-71f11e16c64f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:40.674723637Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1924,"client":"0","":"downstairs","session_id":"e1bac113-3c75-402f-8614-71f11e16c64f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:40.674737056Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1924,"client":"1","":"downstairs","session_id":"e1bac113-3c75-402f-8614-71f11e16c64f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:40.674750135Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1924,"client":"2","":"downstairs","session_id":"e1bac113-3c75-402f-8614-71f11e16c64f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:40.674765245Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1924,"session_id":"e1bac113-3c75-402f-8614-71f11e16c64f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:40.674777714Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1924,"session_id":"e1bac113-3c75-402f-8614-71f11e16c64f"} {"msg":"fb7cb9de-4d10-4a11-97c8-54b51b66c1c0 is now active with session: e1bac113-3c75-402f-8614-71f11e16c64f","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:40.674790464Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1924,"session_id":"e1bac113-3c75-402f-8614-71f11e16c64f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:40.674802513Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1924,"session_id":"e1bac113-3c75-402f-8614-71f11e16c64f"} {"msg":"The guest has finished waiting for activation with:186","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:40.674814832Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1924} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 18 len 4096 data: 13 02/30 Read block 493 len 28672 03/30 Write block 144 len 32768 data: 16 17 16 17 15 16 14 15 04/30 Read block 270 len 8192 05/30 Write block 369 len 4096 data: 15 06/30 Write block 432 len 4096 data: 18 07/30 Read block 420 len 12288 08/30 Read block 103 len 8192 09/30 Write block 271 len 28672 data: 18 18 20 20 19 20 18 10/30 Write block 78 len 4096 data: 14 11/30 Flush 12/30 Write block 98 len 36864 data: 9 9 13 13 13 13 15 17 19 13/30 Read block 71 len 4096 14/30 Write block 165 len 16384 data: 13 13 13 17 15/30 Read block 339 len 40960 16/30 Read block 16 len 8192 17/30 Read block 524 len 32768 18/30 Flush 19/30 Write block 431 len 8192 data: 20 19 20/30 Write block 103 len 8192 data: 14 16 21/30 Write block 282 len 28672 data: 12 15 16 19 20 20 20 22/30 Write block 456 len 4096 data: 10 23/30 Read block 563 len 20480 24/30 Write block 580 len 28672 data: 14 16 17 20 21 24 18 25/30 Write block 433 len 28672 data: 19 19 24 24 22 21 19 26/30 Read block 19 len 20480 27/30 Write block 396 len 28672 data: 12 14 13 12 12 11 13 28/30 Read block 253 len 40960 29/30 Read block 116 len 20480 30/30 Write block 425 len 20480 data: 15 15 16 16 21 ---------------------------------------------------------------- Crucible gen:186 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1018 Acked Write 2 Done Done Done false 1019 Acked Write 2 Done Done Done false 1020 Acked Write 7 Done Done Done false 1021 Acked Write 1 Done Done Done false 1022 Acked Read 5 Done Done Done false 1023 Acked Write 7 Done Done Done false 1024 Acked Write 7 Done Done Done false 1025 Acked Read 5 Done Done Done false 1026 Acked Write 7 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Read 5 Done Done Done false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 11 11 11 33 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1017 1017 1017 Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:29:40.69819004Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1924} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 186 186 186 323 323 323 F F F 1 020-039 181 181 181 320 320 320 F F F 2 040-059 182 182 182 321 321 321 F F F 3 060-079 186 186 186 323 323 323 F F F 4 080-099 186 186 186 324 324 324 F F F 5 100-119 186 186 186 324 324 324 T T T 6 120-139 183 183 183 322 322 322 F F F 7 140-159 186 186 186 323 323 323 F F F 8 160-179 186 186 186 324 324 324 F F F 9 180-199 183 183 183 322 322 322 F F F 10 200-219 170 170 170 302 302 302 F F F 11 220-239 182 182 182 321 321 321 F F F 12 240-259 181 181 181 320 320 320 F F F 13 260-279 186 186 186 323 323 323 F F F 14 280-299 183 183 183 322 322 322 T T T 15 300-319 183 183 183 322 322 322 F F F 16 320-339 183 183 183 322 322 322 F F F 17 340-359 182 182 182 321 321 321 F F F 18 360-379 186 186 186 323 323 323 F F F 19 380-399 183 183 183 322 322 322 T T T 20 400-419 178 178 178 317 317 317 T T T 21 420-439 186 186 186 323 323 323 T T T 22 440-459 182 182 182 321 321 321 T T T 23 460-479 183 183 183 322 322 322 F F F 24 480-499 181 181 181 320 320 320 F F F 25 500-519 183 183 183 322 322 322 F F F 26 520-539 183 183 183 322 322 322 F F F 27 540-559 183 183 183 322 322 322 F F F 28 560-579 182 182 182 321 321 321 F F F 29 580-599 182 182 182 321 321 321 T T T Max gen: 186, Max flush: 324 A Difference in extent metadata is expected here On loop 93 Verifying data now No change after verify Loop: 93 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 186 186 186 323 323 323 F F F 1 020-039 181 181 181 320 320 320 F F F 2 040-059 182 182 182 321 321 321 F F F 3 060-079 186 186 186 323 323 323 F F F 4 080-099 186 186 186 324 324 324 F F F 5 100-119 187 187 187 325 325 325 F F F 6 120-139 183 183 183 322 322 322 F F F 7 140-159 186 186 186 323 323 323 F F F 8 160-179 186 186 186 324 324 324 F F F 9 180-199 183 183 183 322 322 322 F F F 10 200-219 170 170 170 302 302 302 F F F 11 220-239 182 182 182 321 321 321 F F F 12 240-259 181 181 181 320 320 320 F F F 13 260-279 186 186 186 323 323 323 F F F 14 280-299 187 187 187 325 325 325 F F F 15 300-319 183 183 183 322 322 322 F F F 16 320-339 183 183 183 322 322 322 F F F 17 340-359 182 182 182 321 321 321 F F F 18 360-379 186 186 186 323 323 323 F F F 19 380-399 187 187 187 325 325 325 F F F 20 400-419 187 187 187 325 325 325 F F F 21 420-439 187 187 187 325 325 325 F F F 22 440-459 187 187 187 325 325 325 F F F 23 460-479 183 183 183 322 322 322 F F F 24 480-499 181 181 181 320 320 320 F F F 25 500-519 183 183 183 322 322 322 F F F 26 520-539 183 183 183 322 322 322 F F F 27 540-559 183 183 183 322 322 322 F F F 28 560-579 182 182 182 321 321 321 F F F 29 580-599 187 187 187 325 325 325 F F F Max gen: 187, Max flush: 325 Begin loop 94 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:45.077648927Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1932,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:29:45.121521603Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1932,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:45.121926003Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1932,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:45.154463243Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1932,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:45.155002167Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1932} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"0fd98d0d5d909611f8b408ccda2559d35fe117bb\",\n git_commit_timestamp: \"2024-11-08T20:05:44.000000000Z\",\n git_branch: \"mkeeter/drop-skipped-backpressure-guard\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:45.155039355Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1932} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:45.155053694Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1932} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:45.155067774Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1932} {"msg":"Crucible 8819afeb-1894-4bed-a472-47e1c07e77c2 has session id: 0ade6c95-90ed-4efe-b2a8-2e8a0877d873","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:45.155479433Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1932,"session_id":"0ade6c95-90ed-4efe-b2a8-2e8a0877d873"} {"msg":"Upstairs opts: Upstairs UUID: 8819afeb-1894-4bed-a472-47e1c07e77c2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:45.155527691Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1932,"session_id":"0ade6c95-90ed-4efe-b2a8-2e8a0877d873"} {"msg":"Crucible stats registered with UUID: 8819afeb-1894-4bed-a472-47e1c07e77c2","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:45.15554917Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1932,"session_id":"0ade6c95-90ed-4efe-b2a8-2e8a0877d873"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:45.186829391Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1932,"task":"crutest"} {"msg":"The guest has requested activation with gen:188","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:45.186875829Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1932} {"msg":"8819afeb-1894-4bed-a472-47e1c07e77c2 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:45.186908377Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1932,"session_id":"0ade6c95-90ed-4efe-b2a8-2e8a0877d873"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:45.186939996Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1932,"client":"0","":"downstairs","session_id":"0ade6c95-90ed-4efe-b2a8-2e8a0877d873"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:45.186987194Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1932,"client":"0","":"downstairs","session_id":"0ade6c95-90ed-4efe-b2a8-2e8a0877d873"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:45.187029412Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1932,"client":"1","":"downstairs","session_id":"0ade6c95-90ed-4efe-b2a8-2e8a0877d873"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:45.18706576Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1932,"client":"1","":"downstairs","session_id":"0ade6c95-90ed-4efe-b2a8-2e8a0877d873"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:45.187100578Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1932,"client":"2","":"downstairs","session_id":"0ade6c95-90ed-4efe-b2a8-2e8a0877d873"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:45.187180534Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1932,"client":"2","":"downstairs","session_id":"0ade6c95-90ed-4efe-b2a8-2e8a0877d873"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:45.187224712Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1932,"":"io task","client":"1","":"downstairs","session_id":"0ade6c95-90ed-4efe-b2a8-2e8a0877d873"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:45.187377404Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1932,"":"io task","client":"0","":"downstairs","session_id":"0ade6c95-90ed-4efe-b2a8-2e8a0877d873"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:45.187428722Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1932,"":"io task","client":"2","":"downstairs","session_id":"0ade6c95-90ed-4efe-b2a8-2e8a0877d873"} {"msg":"ds_connection connected from Ok(127.0.0.1:51443)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:45.18746603Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1932,"":"io task","client":"1","":"downstairs","session_id":"0ade6c95-90ed-4efe-b2a8-2e8a0877d873"} {"msg":"ds_connection connected from Ok(127.0.0.1:43724)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:45.187501578Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1932,"":"io task","client":"2","":"downstairs","session_id":"0ade6c95-90ed-4efe-b2a8-2e8a0877d873"} {"msg":"ds_connection connected from Ok(127.0.0.1:47185)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:45.187528757Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1932,"":"io task","client":"0","":"downstairs","session_id":"0ade6c95-90ed-4efe-b2a8-2e8a0877d873"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:45.187920548Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1932,"client":"1","":"downstairs","session_id":"0ade6c95-90ed-4efe-b2a8-2e8a0877d873"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:45.187962996Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1932,"client":"1","":"downstairs","session_id":"0ade6c95-90ed-4efe-b2a8-2e8a0877d873"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:45.187994994Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1932,"client":"2","":"downstairs","session_id":"0ade6c95-90ed-4efe-b2a8-2e8a0877d873"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:45.188024363Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1932,"client":"2","":"downstairs","session_id":"0ade6c95-90ed-4efe-b2a8-2e8a0877d873"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:45.188054821Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1932,"client":"0","":"downstairs","session_id":"0ade6c95-90ed-4efe-b2a8-2e8a0877d873"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:45.18807512Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1932,"client":"0","":"downstairs","session_id":"0ade6c95-90ed-4efe-b2a8-2e8a0877d873"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:45.188523818Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1932,"client":"2","":"downstairs","session_id":"0ade6c95-90ed-4efe-b2a8-2e8a0877d873"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:45.188625054Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1932,"client":"1","":"downstairs","session_id":"0ade6c95-90ed-4efe-b2a8-2e8a0877d873"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:45.188650662Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1932,"client":"0","":"downstairs","session_id":"0ade6c95-90ed-4efe-b2a8-2e8a0877d873"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:45.189117989Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1932,"client":"2","":"downstairs","session_id":"0ade6c95-90ed-4efe-b2a8-2e8a0877d873"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:45.189155237Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1932,"client":"2","":"downstairs","session_id":"0ade6c95-90ed-4efe-b2a8-2e8a0877d873"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:45.189171207Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1932,"session_id":"0ade6c95-90ed-4efe-b2a8-2e8a0877d873"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:45.189233194Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1932,"client":"1","":"downstairs","session_id":"0ade6c95-90ed-4efe-b2a8-2e8a0877d873"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:45.189253053Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1932,"client":"1","":"downstairs","session_id":"0ade6c95-90ed-4efe-b2a8-2e8a0877d873"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:45.189266542Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1932,"session_id":"0ade6c95-90ed-4efe-b2a8-2e8a0877d873"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:45.189278762Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1932,"client":"0","":"downstairs","session_id":"0ade6c95-90ed-4efe-b2a8-2e8a0877d873"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:45.189291931Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1932,"client":"0","":"downstairs","session_id":"0ade6c95-90ed-4efe-b2a8-2e8a0877d873"} {"msg":"[0]R flush_numbers[0..12]: [323, 320, 321, 323, 324, 325, 322, 323, 324, 322, 302, 321]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:45.18930559Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1932,"":"downstairs","":"downstairs","session_id":"0ade6c95-90ed-4efe-b2a8-2e8a0877d873"} {"msg":"[0]R generation[0..12]: [186, 181, 182, 186, 186, 187, 183, 186, 186, 183, 170, 182]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:45.189319609Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1932,"":"downstairs","":"downstairs","session_id":"0ade6c95-90ed-4efe-b2a8-2e8a0877d873"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:45.189333709Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1932,"":"downstairs","":"downstairs","session_id":"0ade6c95-90ed-4efe-b2a8-2e8a0877d873"} {"msg":"[1]R flush_numbers[0..12]: [323, 320, 321, 323, 324, 325, 322, 323, 324, 322, 302, 321]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:45.189351508Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1932,"":"downstairs","":"downstairs","session_id":"0ade6c95-90ed-4efe-b2a8-2e8a0877d873"} {"msg":"[1]R generation[0..12]: [186, 181, 182, 186, 186, 187, 183, 186, 186, 183, 170, 182]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:45.189365937Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1932,"":"downstairs","":"downstairs","session_id":"0ade6c95-90ed-4efe-b2a8-2e8a0877d873"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:45.189379817Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1932,"":"downstairs","":"downstairs","session_id":"0ade6c95-90ed-4efe-b2a8-2e8a0877d873"} {"msg":"[2]R flush_numbers[0..12]: [323, 320, 321, 323, 324, 325, 322, 323, 324, 322, 302, 321]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:45.189393476Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1932,"":"downstairs","":"downstairs","session_id":"0ade6c95-90ed-4efe-b2a8-2e8a0877d873"} {"msg":"[2]R generation[0..12]: [186, 181, 182, 186, 186, 187, 183, 186, 186, 183, 170, 182]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:45.189407375Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1932,"":"downstairs","":"downstairs","session_id":"0ade6c95-90ed-4efe-b2a8-2e8a0877d873"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:45.189421524Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1932,"":"downstairs","":"downstairs","session_id":"0ade6c95-90ed-4efe-b2a8-2e8a0877d873"} {"msg":"Max found gen is 188","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:45.189434554Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1932,"":"downstairs","":"downstairs","session_id":"0ade6c95-90ed-4efe-b2a8-2e8a0877d873"} {"msg":"Generation requested: 188 >= found:188","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-08T20:29:45.189447573Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1932,"":"downstairs","":"downstairs","session_id":"0ade6c95-90ed-4efe-b2a8-2e8a0877d873"} {"msg":"Next flush: 326","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:45.189468222Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1932,"":"downstairs","":"downstairs","session_id":"0ade6c95-90ed-4efe-b2a8-2e8a0877d873"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:45.189481301Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1932,"":"downstairs","":"downstairs","session_id":"0ade6c95-90ed-4efe-b2a8-2e8a0877d873"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:45.189494511Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1932,"session_id":"0ade6c95-90ed-4efe-b2a8-2e8a0877d873"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:45.18950657Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1932,"client":"0","":"downstairs","session_id":"0ade6c95-90ed-4efe-b2a8-2e8a0877d873"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:45.18952005Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1932,"client":"1","":"downstairs","session_id":"0ade6c95-90ed-4efe-b2a8-2e8a0877d873"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:45.189533309Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1932,"client":"2","":"downstairs","session_id":"0ade6c95-90ed-4efe-b2a8-2e8a0877d873"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:45.189546828Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1932,"session_id":"0ade6c95-90ed-4efe-b2a8-2e8a0877d873"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:45.189561208Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1932,"session_id":"0ade6c95-90ed-4efe-b2a8-2e8a0877d873"} {"msg":"8819afeb-1894-4bed-a472-47e1c07e77c2 is now active with session: 0ade6c95-90ed-4efe-b2a8-2e8a0877d873","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:45.189574167Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1932,"session_id":"0ade6c95-90ed-4efe-b2a8-2e8a0877d873"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:45.189586187Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1932,"session_id":"0ade6c95-90ed-4efe-b2a8-2e8a0877d873"} {"msg":"The guest has finished waiting for activation with:188","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:45.189598426Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1932} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 135 len 8192 data: 16 14 02/30 Read block 458 len 24576 03/30 Read block 79 len 40960 04/30 Read block 520 len 4096 05/30 Write block 403 len 28672 data: 11 10 11 12 13 14 13 06/30 Write block 108 len 36864 data: 17 16 15 11 11 11 9 9 11 07/30 Read block 383 len 20480 08/30 Write block 344 len 36864 data: 17 17 18 16 13 17 16 17 17 09/30 Write block 449 len 40960 data: 14 12 12 14 10 10 7 11 11 9 10/30 Write block 119 len 16384 data: 12 12 14 14 11/30 Read block 98 len 36864 12/30 Read block 319 len 8192 13/30 Read block 100 len 8192 14/30 Read block 433 len 24576 15/30 Read block 287 len 28672 16/30 Write block 395 len 12288 data: 12 13 15 17/30 Read block 67 len 12288 18/30 Read block 305 len 8192 19/30 Write block 38 len 8192 data: 15 16 20/30 Write block 224 len 28672 data: 17 15 17 19 15 14 11 21/30 Read block 316 len 32768 22/30 Write block 209 len 12288 data: 9 7 10 23/30 Read block 32 len 32768 24/30 Write block 415 len 28672 data: 13 13 13 13 15 15 16 25/30 Write block 75 len 4096 data: 14 26/30 Read block 27 len 36864 27/30 Write block 486 len 32768 data: 12 10 13 15 16 13 11 11 28/30 Write block 88 len 24576 data: 14 13 14 12 14 14 29/30 Write block 576 len 4096 data: 12 30/30 Write block 485 len 36864 data: 12 13 11 14 16 17 14 12 12 ---------------------------------------------------------------- Crucible gen:188 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 2 Done Done Done false 1001 Acked Read 6 Done Done Done false 1002 Acked Read 10 Done Done Done false 1003 Acked Read 1 Done Done Done false 1004 Acked Write 7 Done Done Done false 1005 Acked Write 9 Done Done Done false 1006 Acked Read 5 Done Done Done false 1007 Acked Write 9 Done Done Done false 1008 Acked Write 10 Done Done Done false 1009 Acked Write 4 Done Done Done false 1010 Acked Read 9 Done Done Done false 1011 Acked Read 2 Done Done Done false 1012 Acked Read 2 Done Done Done false 1013 Acked Read 6 Done Done Done false 1014 Acked Read 7 Done Done Done false 1015 Acked Write 3 Done Done Done false 1016 Acked Read 3 Done Done Done false 1017 Acked Read 2 Done Done Done false 1018 Acked Write 2 Done Done Done false 1019 Acked Write 7 Done Done Done false 1020 Acked Read 8 Done Done Done false 1021 Acked Write 3 Done Done Done false 1022 Acked Read 8 Done Done Done false 1023 Acked Write 7 Done Done Done false 1024 Acked Write 1 Done Done Done false 1025 Acked Read 9 Done Done Done false 1026 Acked Write 8 Done Done Done false 1027 Acked Write 6 Sent Sent Sent false 1028 Acked Write 1 Sent Sent Sent false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 3 3 9 Done 27 27 27 81 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 0 0 0 Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:29:45.208689343Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1932} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 186 186 186 323 323 323 F F F 1 020-039 181 181 181 320 320 320 T T T 2 040-059 182 182 182 321 321 321 F F F 3 060-079 186 186 186 323 323 323 T T T 4 080-099 186 186 186 324 324 324 T T T 5 100-119 187 187 187 325 325 325 T T T 6 120-139 183 183 183 322 322 322 T T T 7 140-159 186 186 186 323 323 323 F F F 8 160-179 186 186 186 324 324 324 F F F 9 180-199 183 183 183 322 322 322 F F F 10 200-219 170 170 170 302 302 302 T T T 11 220-239 182 182 182 321 321 321 T T T 12 240-259 181 181 181 320 320 320 F F F 13 260-279 186 186 186 323 323 323 F F F 14 280-299 187 187 187 325 325 325 F F F 15 300-319 183 183 183 322 322 322 F F F 16 320-339 183 183 183 322 322 322 F F F 17 340-359 182 182 182 321 321 321 T T T 18 360-379 186 186 186 323 323 323 F F F 19 380-399 187 187 187 325 325 325 T T T 20 400-419 187 187 187 325 325 325 T T T 21 420-439 187 187 187 325 325 325 T T T 22 440-459 187 187 187 325 325 325 T T T 23 460-479 183 183 183 322 322 322 F F F 24 480-499 181 181 181 320 320 320 T T T 25 500-519 183 183 183 322 322 322 F F F 26 520-539 183 183 183 322 322 322 F F F 27 540-559 183 183 183 322 322 322 F F F 28 560-579 182 182 182 321 321 321 T T T 29 580-599 187 187 187 325 325 325 F F F Max gen: 187, Max flush: 325 A Difference in extent metadata is expected here On loop 94 Verifying data now No change after verify Loop: 94 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 186 186 186 323 323 323 F F F 1 020-039 188 188 188 326 326 326 F F F 2 040-059 182 182 182 321 321 321 F F F 3 060-079 188 188 188 326 326 326 F F F 4 080-099 188 188 188 326 326 326 F F F 5 100-119 188 188 188 326 326 326 F F F 6 120-139 188 188 188 326 326 326 F F F 7 140-159 186 186 186 323 323 323 F F F 8 160-179 186 186 186 324 324 324 F F F 9 180-199 183 183 183 322 322 322 F F F 10 200-219 188 188 188 326 326 326 F F F 11 220-239 188 188 188 326 326 326 F F F 12 240-259 181 181 181 320 320 320 F F F 13 260-279 186 186 186 323 323 323 F F F 14 280-299 187 187 187 325 325 325 F F F 15 300-319 183 183 183 322 322 322 F F F 16 320-339 183 183 183 322 322 322 F F F 17 340-359 188 188 188 326 326 326 F F F 18 360-379 186 186 186 323 323 323 F F F 19 380-399 188 188 188 326 326 326 F F F 20 400-419 188 188 188 326 326 326 F F F 21 420-439 188 188 188 326 326 326 F F F 22 440-459 188 188 188 326 326 326 F F F 23 460-479 183 183 183 322 322 322 F F F 24 480-499 188 188 188 326 326 326 F F F 25 500-519 183 183 183 322 322 322 F F F 26 520-539 183 183 183 322 322 322 F F F 27 540-559 183 183 183 322 322 322 F F F 28 560-579 188 188 188 326 326 326 F F F 29 580-599 187 187 187 325 325 325 F F F Max gen: 188, Max flush: 326 Begin loop 95 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:49.697490457Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1940,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:29:49.741582732Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1940,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:49.741967573Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1940,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:49.774680124Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1940,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:49.775228877Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1940} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"0fd98d0d5d909611f8b408ccda2559d35fe117bb\",\n git_commit_timestamp: \"2024-11-08T20:05:44.000000000Z\",\n git_branch: \"mkeeter/drop-skipped-backpressure-guard\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:49.775264656Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1940} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:49.775278715Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1940} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:49.775293184Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1940} {"msg":"Crucible 806821aa-d7e5-4e71-b747-630d6d97fd67 has session id: 0c9fa852-d7c1-4e4a-8308-4a80e7e77e73","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:49.775739263Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1940,"session_id":"0c9fa852-d7c1-4e4a-8308-4a80e7e77e73"} {"msg":"Upstairs opts: Upstairs UUID: 806821aa-d7e5-4e71-b747-630d6d97fd67, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:49.77578558Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1940,"session_id":"0c9fa852-d7c1-4e4a-8308-4a80e7e77e73"} {"msg":"Crucible stats registered with UUID: 806821aa-d7e5-4e71-b747-630d6d97fd67","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:49.775813429Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1940,"session_id":"0c9fa852-d7c1-4e4a-8308-4a80e7e77e73"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:49.807545477Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1940,"task":"crutest"} {"msg":"The guest has requested activation with gen:190","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:49.807577716Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1940} {"msg":"806821aa-d7e5-4e71-b747-630d6d97fd67 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:49.807650072Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1940,"session_id":"0c9fa852-d7c1-4e4a-8308-4a80e7e77e73"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:49.807673191Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1940,"client":"0","":"downstairs","session_id":"0c9fa852-d7c1-4e4a-8308-4a80e7e77e73"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:49.807709889Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1940,"client":"0","":"downstairs","session_id":"0c9fa852-d7c1-4e4a-8308-4a80e7e77e73"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:49.807744138Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1940,"client":"1","":"downstairs","session_id":"0c9fa852-d7c1-4e4a-8308-4a80e7e77e73"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:49.807779716Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1940,"client":"1","":"downstairs","session_id":"0c9fa852-d7c1-4e4a-8308-4a80e7e77e73"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:49.807815054Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1940,"client":"2","":"downstairs","session_id":"0c9fa852-d7c1-4e4a-8308-4a80e7e77e73"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:49.807851732Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1940,"client":"2","":"downstairs","session_id":"0c9fa852-d7c1-4e4a-8308-4a80e7e77e73"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:49.80788914Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1940,"":"io task","client":"2","":"downstairs","session_id":"0c9fa852-d7c1-4e4a-8308-4a80e7e77e73"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:49.807926378Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1940,"":"io task","client":"0","":"downstairs","session_id":"0c9fa852-d7c1-4e4a-8308-4a80e7e77e73"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:49.807962687Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1940,"":"io task","client":"1","":"downstairs","session_id":"0c9fa852-d7c1-4e4a-8308-4a80e7e77e73"} {"msg":"ds_connection connected from Ok(127.0.0.1:37481)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:49.808000095Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1940,"":"io task","client":"0","":"downstairs","session_id":"0c9fa852-d7c1-4e4a-8308-4a80e7e77e73"} {"msg":"ds_connection connected from Ok(127.0.0.1:57445)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:49.808036133Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1940,"":"io task","client":"2","":"downstairs","session_id":"0c9fa852-d7c1-4e4a-8308-4a80e7e77e73"} {"msg":"ds_connection connected from Ok(127.0.0.1:60734)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:49.808120869Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1940,"":"io task","client":"1","":"downstairs","session_id":"0c9fa852-d7c1-4e4a-8308-4a80e7e77e73"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:49.808843064Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1940,"client":"1","":"downstairs","session_id":"0c9fa852-d7c1-4e4a-8308-4a80e7e77e73"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:49.808887742Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1940,"client":"1","":"downstairs","session_id":"0c9fa852-d7c1-4e4a-8308-4a80e7e77e73"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:49.80892094Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1940,"client":"2","":"downstairs","session_id":"0c9fa852-d7c1-4e4a-8308-4a80e7e77e73"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:49.808952028Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1940,"client":"2","":"downstairs","session_id":"0c9fa852-d7c1-4e4a-8308-4a80e7e77e73"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:49.808982017Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1940,"client":"0","":"downstairs","session_id":"0c9fa852-d7c1-4e4a-8308-4a80e7e77e73"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:49.809009366Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1940,"client":"0","":"downstairs","session_id":"0c9fa852-d7c1-4e4a-8308-4a80e7e77e73"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:49.80952776Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1940,"client":"1","":"downstairs","session_id":"0c9fa852-d7c1-4e4a-8308-4a80e7e77e73"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:49.809560009Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1940,"client":"0","":"downstairs","session_id":"0c9fa852-d7c1-4e4a-8308-4a80e7e77e73"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:49.809683003Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1940,"client":"2","":"downstairs","session_id":"0c9fa852-d7c1-4e4a-8308-4a80e7e77e73"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:49.81014276Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1940,"client":"1","":"downstairs","session_id":"0c9fa852-d7c1-4e4a-8308-4a80e7e77e73"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:49.810168679Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1940,"client":"1","":"downstairs","session_id":"0c9fa852-d7c1-4e4a-8308-4a80e7e77e73"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:49.810182978Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1940,"session_id":"0c9fa852-d7c1-4e4a-8308-4a80e7e77e73"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:49.810270934Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1940,"client":"0","":"downstairs","session_id":"0c9fa852-d7c1-4e4a-8308-4a80e7e77e73"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:49.810306592Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1940,"client":"0","":"downstairs","session_id":"0c9fa852-d7c1-4e4a-8308-4a80e7e77e73"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:49.810328991Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1940,"session_id":"0c9fa852-d7c1-4e4a-8308-4a80e7e77e73"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:49.810342391Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1940,"client":"2","":"downstairs","session_id":"0c9fa852-d7c1-4e4a-8308-4a80e7e77e73"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:49.81035592Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1940,"client":"2","":"downstairs","session_id":"0c9fa852-d7c1-4e4a-8308-4a80e7e77e73"} {"msg":"[0]R flush_numbers[0..12]: [323, 326, 321, 326, 326, 326, 326, 323, 324, 322, 326, 326]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:49.810373239Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1940,"":"downstairs","":"downstairs","session_id":"0c9fa852-d7c1-4e4a-8308-4a80e7e77e73"} {"msg":"[0]R generation[0..12]: [186, 188, 182, 188, 188, 188, 188, 186, 186, 183, 188, 188]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:49.810387458Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1940,"":"downstairs","":"downstairs","session_id":"0c9fa852-d7c1-4e4a-8308-4a80e7e77e73"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:49.810401718Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1940,"":"downstairs","":"downstairs","session_id":"0c9fa852-d7c1-4e4a-8308-4a80e7e77e73"} {"msg":"[1]R flush_numbers[0..12]: [323, 326, 321, 326, 326, 326, 326, 323, 324, 322, 326, 326]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:49.810415367Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1940,"":"downstairs","":"downstairs","session_id":"0c9fa852-d7c1-4e4a-8308-4a80e7e77e73"} {"msg":"[1]R generation[0..12]: [186, 188, 182, 188, 188, 188, 188, 186, 186, 183, 188, 188]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:49.810429106Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1940,"":"downstairs","":"downstairs","session_id":"0c9fa852-d7c1-4e4a-8308-4a80e7e77e73"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:49.810443675Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1940,"":"downstairs","":"downstairs","session_id":"0c9fa852-d7c1-4e4a-8308-4a80e7e77e73"} {"msg":"[2]R flush_numbers[0..12]: [323, 326, 321, 326, 326, 326, 326, 323, 324, 322, 326, 326]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:49.810457595Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1940,"":"downstairs","":"downstairs","session_id":"0c9fa852-d7c1-4e4a-8308-4a80e7e77e73"} {"msg":"[2]R generation[0..12]: [186, 188, 182, 188, 188, 188, 188, 186, 186, 183, 188, 188]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:49.810471344Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1940,"":"downstairs","":"downstairs","session_id":"0c9fa852-d7c1-4e4a-8308-4a80e7e77e73"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:49.810485813Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1940,"":"downstairs","":"downstairs","session_id":"0c9fa852-d7c1-4e4a-8308-4a80e7e77e73"} {"msg":"Max found gen is 189","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:49.810509382Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","Wait for a query_work_queue command to finish before sending IO pid":1940,"":"downstairs","":"downstairs","session_id":"0c9fa852-d7c1-4e4a-8308-4a80e7e77e73"} {"msg":"Generation requested: 190 >= found:189","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:49.810543921Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1940,"":"downstairs","":"downstairs","session_id":"0c9fa852-d7c1-4e4a-8308-4a80e7e77e73"} {"msg":"Next flush: 327","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:49.81056824Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1940,"":"downstairs","":"downstairs","session_id":"0c9fa852-d7c1-4e4a-8308-4a80e7e77e73"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:49.810582749Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1940,"":"downstairs","":"downstairs","session_id":"0c9fa852-d7c1-4e4a-8308-4a80e7e77e73"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:49.810596318Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1940,"session_id":"0c9fa852-d7c1-4e4a-8308-4a80e7e77e73"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:49.810611007Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1940,"client":"0","":"downstairs","session_id":"0c9fa852-d7c1-4e4a-8308-4a80e7e77e73"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:49.810624747Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1940,"client":"1","":"downstairs","session_id":"0c9fa852-d7c1-4e4a-8308-4a80e7e77e73"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:49.810638206Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1940,"client":"2","":"downstairs","session_id":"0c9fa852-d7c1-4e4a-8308-4a80e7e77e73"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:49.810651615Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1940,"session_id":"0c9fa852-d7c1-4e4a-8308-4a80e7e77e73"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:49.810663885Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1940,"session_id":"0c9fa852-d7c1-4e4a-8308-4a80e7e77e73"} {"msg":"806821aa-d7e5-4e71-b747-630d6d97fd67 is now active with session: 0c9fa852-d7c1-4e4a-8308-4a80e7e77e73","v":0,"name":"crucible","level":30Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2024-11-08T20:29:49.810676664Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1940,"session_id":"0c9fa852-d7c1-4e4a-8308-4a80e7e77e73"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:49.810690734Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1940,"session_id":"0c9fa852-d7c1-4e4a-8308-4a80e7e77e73"} {"msg":"The guest has finished waiting for activation with:190","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:49.810703383Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1940} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 566 len 32768 data: 10 8 12 13 11 15 15 16 02/30 Read block 404 len 40960 03/30 Write block 512 len 36864 data: 14 15 15 13 12 10 14 16 13 04/30 Flush 05/30 Write block 33 len 36864 data: 11 13 13 13 13 16 17 16 13 06/30 Write block 534 len 12288 data: 16 17 16 07/30 Write block 460 len 40960 data: 11 11 11 11 8 10 14 16 18 15 08/30 Read block 550 len 16384 09/30 Read block 186 len 8192 10/30 Read block 0 len 12288 11/30 Read block 164 len 40960 12/30 Read block 361 len 20480 13/30 Read block 363 len 12288 14/30 Write block 178 len 8192 data: 10 12 15/30 Write block 309 len 32768 data: 12 14 14 17 17 14 19 16 16/30 Write block 167 len 8192 data: 14 18 17/30 Read block 312 len 4096 18/30 Read block 350 len 32768 19/30 Write block 536 len 20480 data: 17 14 15 15 17 20/30 Read block 174 len 8192 21/30 Flush 22/30 Write block 49 len 32768 data: 11 16 13 13 13 15 15 13 23/30 Write block 584 len 20480 data: 22 25 19 21 16 24/30 Read block 227 len 12288 25/30 Flush 26/30 Write block 340 len 32768 data: 18 18 20 19 18 18 19 17 27/30 Write block 250 len 24576 data: 20 18 23 20 18 17 28/30 Read block 175 len 32768 29/30 Write block 378 len 8192 data: 17 18 30/30 Read block 471 len 20480 ---------------------------------------------------------------- Crucible gen:190 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 8 Done Done Done false 1026 Acked Write 6 Done Done Done false 1027 Acked Read 8 Done Done Done false 1028 Acked Write 2 Done Done Sent false 1029 Acked Read 5 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 2 3 Done 4 5 3 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:29:49.840419641Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1940} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 186 186 186 323 323 323 F F F 1 020-039 190 190 190 328 328 328 F F F 2 040-059 190 190 190 329 329 329 F F F 3 060-079 188 188 188 326 326 326 F F F 4 080-099 188 188 188 326 326 326 F F F 5 100-119 188 188 188 326 326 326 F F F 6 120-139 188 188 188 326 326 326 F F F 7 140-159 186 186 186 323 323 323 F F F 8 160-179 190 190 190 328 328 328 F F F 9 180-199 183 183 183 322 322 322 F F F 10 200-219 188 188 188 326 326 326 F F F 11 220-239 188 188 188 326 326 326 F F F 12 240-259 181 181 181 320 320 320 T T T 13 260-279 186 186 186 323 323 323 F F F 14 280-299 187 187 187 325 325 325 F F F 15 300-319 190 190 190 328 328 328 F F F 16 320-339 183 183 183 322 322 322 F F F 17 340-359 188 188 188 326 326 326 T T T 18 360-379 186 186 186 323 323 323 T T T 19 380-399 188 188 188 326 326 326 F F F 20 400-419 188 188 188 326 326 326 F F F 21 420-439 188 188 188 326 326 326 F F F 22 440-459 188 188 188 326 326 326 F F F 23 460-479 190 190 190 328 328 328 F F F 24 480-499 188 188 188 326 326 326 F F F 25 500-519 190 190 190 327 327 327 F F F 26 520-539 190 190 190 328 328 328 F F F 27 540-559 190 190 190 328 328 328 F F F 28 560-579 190 190 190 327 327 327 F F F 29 580-599 190 190 190 329 329 329 F F F Max gen: 190, Max flush: 329 A Difference in extent metadata is expected here On loop 95 Verifying data now No change after verify Loop: 95 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 186 186 186 323 323 323 F F F 1 020-039 190 190 190 328 328 328 F F F 2 040-059 190 190 190 329 329 329 F F F 3 060-079 188 188 188 326 326 326 F F F 4 080-099 188 188 188 326 326 326 F F F 5 100-119 188 188 188 326 326 326 F F F 6 120-139 188 188 188 326 326 326 F F F 7 140-159 186 186 186 323 323 323 F F F 8 160-179 190 190 190 328 328 328 F F F 9 180-199 183 183 183 322 322 322 F F F 10 200-219 188 188 188 326 326 326 F F F 11 220-239 188 188 188 326 326 326 F F F 12 240-259 191 191 191 330 330 330 F F F 13 260-279 186 186 186 323 323 323 F F F 14 280-299 187 187 187 325 325 325 F F F 15 300-319 190 190 190 328 328 328 F F F 16 320-339 183 183 183 322 322 322 F F F 17 340-359 191 191 191 330 330 330 F F F 18 360-379 191 191 191 330 330 330 F F F 19 380-399 188 188 188 326 326 326 F F F 20 400-419 188 188 188 326 326 326 F F F 21 420-439 188 188 188 326 326 326 F F F 22 440-459 188 188 188 326 326 326 F F F 23 460-479 190 190 190 328 328 328 F F F 24 480-499 188 188 188 326 326 326 F F F 25 500-519 190 190 190 327 327 327 F F F 26 520-539 190 190 190 328 328 328 F F F 27 540-559 190 190 190 328 328 328 F F F 28 560-579 190 190 190 327 327 327 F F F 29 580-599 190 190 190 329 329 329 F F F Max gen: 191, Max flush: 330 Begin loop 96 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:54.157833301Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1948,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:29:54.200562633Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1948,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:54.200970663Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1948,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:54.233064124Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1948,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:54.233653165Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1948} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"0fd98d0d5d909611f8b408ccda2559d35fe117bb\",\n git_commit_timestamp: \"2024-11-08T20:05:44.000000000Z\",\n git_branch: \"mkeeter/drop-skipped-backpressure-guard\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:54.233690553Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1948} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:54.233704993Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1948} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:54.233719142Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1948} {"msg":"Crucible 75667b69-7cf1-4abe-aafd-6fd6a18ecb5c has session id: cd21191b-e5a1-4005-8818-e0441ae5c01c","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:54.23416396Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1948,"session_id":"cd21191b-e5a1-4005-8818-e0441ae5c01c"} {"msg":"Upstairs opts: Upstairs UUID: 75667b69-7cf1-4abe-aafd-6fd6a18ecb5c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:54.234210018Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1948,"session_id":"cd21191b-e5a1-4005-8818-e0441ae5c01c"} {"msg":"Crucible stats registered with UUID: 75667b69-7cf1-4abe-aafd-6fd6a18ecb5c","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:54.234236127Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1948,"session_id":"cd21191b-e5a1-4005-8818-e0441ae5c01c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:54.265054921Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1948,"task":"crutest"} {"msg":"The guest has requested activation with gen:192","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:54.265087869Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1948} {"msg":"75667b69-7cf1-4abe-aafd-6fd6a18ecb5c active request set","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:54.265113338Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1948,"session_id":"cd21191b-e5a1-4005-8818-e0441ae5c01c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:54.265134047Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1948,"client":"0","":"downstairs","session_id":"cd21191b-e5a1-4005-8818-e0441ae5c01c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:54.265162005Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1948,"client":"0","":"downstairs","session_id":"cd21191b-e5a1-4005-8818-e0441ae5c01c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:54.265179845Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1948,"client":"1","":"downstairs","session_id":"cd21191b-e5a1-4005-8818-e0441ae5c01c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:54.265197204Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1948,"client":"1","":"downstairs","session_id":"cd21191b-e5a1-4005-8818-e0441ae5c01c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:54.265214133Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1948,"client":"2","":"downstairs","session_id":"cd21191b-e5a1-4005-8818-e0441ae5c01c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:54.265231282Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1948,"client":"2","":"downstairs","session_id":"cd21191b-e5a1-4005-8818-e0441ae5c01c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:54.265247801Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1948,"":"io task","client":"2","":"downstairs","session_id":"cd21191b-e5a1-4005-8818-e0441ae5c01c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:54.2652651Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1948,"":"io task","client":"0","":"downstairs","session_id":"cd21191b-e5a1-4005-8818-e0441ae5c01c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:54.265297169Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1948,"":"io task","client":"1","":"downstairs","session_id":"cd21191b-e5a1-4005-8818-e0441ae5c01c"} {"msg":"ds_connection connected from Ok(127.0.0.1:40640)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:54.265570806Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1948,"":"io task","client":"2","":"downstairs","session_id":"cd21191b-e5a1-4005-8818-e0441ae5c01c"} {"msg":"ds_connection connected from Ok(127.0.0.1:59797)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:54.265622873Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1948,"":"io task","client":"1","":"downstairs","session_id":"cd21191b-e5a1-4005-8818-e0441ae5c01c"} {"msg":"ds_connection connected from Ok(127.0.0.1:36364)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:54.265658411Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1948,"":"io task","client":"0","":"downstairs","session_id":"cd21191b-e5a1-4005-8818-e0441ae5c01c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:54.266373916Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1948,"client":"1","":"downstairs","session_id":"cd21191b-e5a1-4005-8818-e0441ae5c01c"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:54.266404085Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1948,"client":"1","":"downstairs","session_id":"cd21191b-e5a1-4005-8818-e0441ae5c01c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:54.266419804Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1948,"client":"2","":"downstairs","session_id":"cd21191b-e5a1-4005-8818-e0441ae5c01c"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:54.266438223Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1948,"client":"2","":"downstairs","session_id":"cd21191b-e5a1-4005-8818-e0441ae5c01c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:54.26650374Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1948,"client":"0","":"downstairs","session_id":"cd21191b-e5a1-4005-8818-e0441ae5c01c"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:54.266535208Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1948,"client":"0","":"downstairs","session_id":"cd21191b-e5a1-4005-8818-e0441ae5c01c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:54.267133769Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1948,"client":"2","":"downstairs","session_id":"cd21191b-e5a1-4005-8818-e0441ae5c01c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:54.267163377Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1948,"client":"0","":"downstairs","session_id":"cd21191b-e5a1-4005-8818-e0441ae5c01c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:54.267180217Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1948,"client":"1","":"downstairs","session_id":"cd21191b-e5a1-4005-8818-e0441ae5c01c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:54.267760228Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1948,"client":"1","":"downstairs","session_id":"cd21191b-e5a1-4005-8818-e0441ae5c01c"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:54.267789537Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1948,"client":"1","":"downstairs","session_id":"cd21191b-e5a1-4005-8818-e0441ae5c01c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:54.267804556Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1948,"session_id":"cd21191b-e5a1-4005-8818-e0441ae5c01c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:54.267817116Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1948,"client":"0","":"downstairs","session_id":"cd21191b-e5a1-4005-8818-e0441ae5c01c"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:54.267830815Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1948,"client":"0","":"downstairs","session_id":"cd21191b-e5a1-4005-8818-e0441ae5c01c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:54.267844184Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1948,"session_id":"cd21191b-e5a1-4005-8818-e0441ae5c01c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:54.267856344Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1948,"client":"2","":"downstairs","session_id":"cd21191b-e5a1-4005-8818-e0441ae5c01c"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:54.267869653Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1948,"client":"2","":"downstairs","session_id":"cd21191b-e5a1-4005-8818-e0441ae5c01c"} {"msg":"[0]R flush_numbers[0..12]: [323, 328, 329, 326, 326, 326, 326, 323, 328, 322, 326, 326]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:54.267883502Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1948,"":"downstairs","":"downstairs","session_id":"cd21191b-e5a1-4005-8818-e0441ae5c01c"} {"msg":"[0]R generation[0..12]: [186, 190, 190, 188, 188, 188, 188, 186, 190, 183, 188, 188]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:54.267902281Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1948,"":"downstairs","":"downstairs","session_id":"cd21191b-e5a1-4005-8818-e0441ae5c01c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:54.267916981Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1948,"":"downstairs","":"downstairs","session_id":"cd21191b-e5a1-4005-8818-e0441ae5c01c"} {"msg":"[1]R flush_numbers[0..12]: [323, 328, 329, 326, 326, 326, 326, 323, 328, 322, 326, 326]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:54.26793101Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1948,"":"downstairs","":"downstairs","session_id":"cd21191b-e5a1-4005-8818-e0441ae5c01c"} {"msg":"[1]R generation[0..12]: [186, 190, 190, 188, 188, 188, 188, 186, 190, 183, 188, 188]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:54.267945029Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1948,"":"downstairs","":"downstairs","session_id":"cd21191b-e5a1-4005-8818-e0441ae5c01c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:54.267958929Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1948,"":"downstairs","":"downstairs","session_id":"cd21191b-e5a1-4005-8818-e0441ae5c01c"} {"msg":"[2]R flush_numbers[0..12]: [323, 328, 329, 326, 326, 326, 326, 323, 328, 322, 326, 326]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:54.267972858Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1948,"":"downstairs","":"downstairs","session_id":"cd21191b-e5a1-4005-8818-e0441ae5c01c"} {"msg":"[2]R generation[0..12]: [186, 190, 190, 188, 188, 188, 188, 186, 190, 183, 188, 188]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:54.267986437Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1948,"":"downstairs","":"downstairs","session_id":"cd21191b-e5a1-4005-8818-e0441ae5c01c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:54.268000297Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1948,"":"downstairs","":"downstairs","session_id":"cd21191b-e5a1-4005-8818-e0441ae5c01c"} {"msg":"Max found gen is 192","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:54.268013286Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1948,"":"downstairs","":"downstairs","session_id":"cd21191b-e5a1-4005-8818-e0441ae5c01c"} {"msg":"Generation requested: 192 >= found:192","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:54.268038505Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1948,"":"downstairs","":"downstairs","session_id":"cd21191b-e5a1-4005-8818-e0441ae5c01c"} {"Wait for a query_work_queue command to finish before sending IO msg":"Next flush: 331","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:54.268060884Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1948,"":"downstairs","":"downstairs","session_id":"cd21191b-e5a1-4005-8818-e0441ae5c01c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:54.268074343Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1948,"":"downstairs","":"downstairs","session_id":"cd21191b-e5a1-4005-8818-e0441ae5c01c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:54.268087343Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1948,"session_id":"cd21191b-e5a1-4005-8818-e0441ae5c01c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:54.268099222Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1948,"client":"0","":"downstairs","session_id":"cd21191b-e5a1-4005-8818-e0441ae5c01c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:54.268112531Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1948,"client":"1","":"downstairs","session_id":"cd21191b-e5a1-4005-8818-e0441ae5c01c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:54.268128021Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1948,"client":"2","":"downstairs","session_id":"cd21191b-e5a1-4005-8818-e0441ae5c01c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:54.26814135Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1948,"session_id":"cd21191b-e5a1-4005-8818-e0441ae5c01c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:54.268153649Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1948,"session_id":"cd21191b-e5a1-4005-8818-e0441ae5c01c"} {"msg":"75667b69-7cf1-4abe-aafd-6fd6a18ecb5c is now active with session: cd21191b-e5a1-4005-8818-e0441ae5c01c","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:54.268166579ZRegion: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1948,"session_id":"cd21191b-e5a1-4005-8818-e0441ae5c01c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:54.268183398Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1948,"session_id":"cd21191b-e5a1-4005-8818-e0441ae5c01c"} {"msg":"The guest has finished waiting for activation with:192","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:54.268195557Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1948} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 547 len 28672 data: 13 15 11 11 8 11 10 02/30 Read block 99 len 4096 03/30 Read block 294 len 36864 04/30 Write block 132 len 32768 data: 12 13 13 17 15 15 13 13 05/30 Read block 270 len 36864 06/30 Write block 72 len 40960 data: 13 14 15 15 15 16 15 12 10 11 07/30 Write block 17 len 40960 data: 16 14 14 14 15 15 12 13 12 12 08/30 Write block 561 len 8192 data: 12 11 09/30 Read block 272 len 8192 10/30 Read block 346 len 28672 11/30 Write block 122 len 24576 data: 15 14 16 12 13 11 12/30 Read block 354 len 24576 13/30 Write block 562 len 36864 data: 12 10 11 12 11 9 13 14 12 14/30 Read block 164 len 24576 15/30 Read block 455 len 4096 16/30 Read block 541 len 8192 17/30 Read block 248 len 40960 18/30 Read block 76 len 16384 19/30 Read block 519 len 20480 20/30 Flush 21/30 Write block 117 len 12288 data: 14 13 13 22/30 Write block 224 len 40960 data: 18 16 18 20 16 15 12 8 11 11 23/30 Write block 463 len 24576 data: 12 9 11 15 17 19 24/30 Read block 118 len 12288 25/30 Write block 339 len 40960 data: 19 19 19 21 20 19 19 20 18 14 26/30 Read block 592 len 16384 27/30 Write block 259 len 4096 data: 12 28/30 Write block 22 len 40960 data: 16 13 14 13 13 10 8 8 6 8 29/30 Read block 468 len 36864 30/30 Write block 492 len 20480 data: 13 13 10 11 12 ---------------------------------------------------------------- Crucible gen:192 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1020 Acked Write 3 Done Done Done false 1021 Acked Write 10 Done Done Done false 1022 Acked Write 6 Done Done Done false 1023 Acked Read 3 Done Done Done false 1024 Acked Write 10 Done Done Done false 1025 Acked Read 4 Done Done Done false 1026 Acked Write 1 Done Done Done false 1027 Acked Write 10 Done Sent Sent false 1028 Acked Read 9 Done Sent Sent false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 3 3 7 Done 9 7 7 23 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1019 1019 1019 Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:29:54.294491622Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1948} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 192 192 192 331 331 331 F F F 1 020-039 192 192 192 331 331 331 T T T 2 040-059 190 190 190 329 329 329 F F F 3 060-079 192 192 192 331 331 331 F F F 4 080-099 192 192 192 331 331 331 F F F 5 100-119 188 188 188 326 326 326 T T T 6 120-139 192 192 192 331 331 331 F F F 7 140-159 186 186 186 323 323 323 F F F 8 160-179 190 190 190 328 328 328 F F F 9 180-199 183 183 183 322 322 322 F F F 10 200-219 188 188 188 326 326 326 F F F 11 220-239 188 188 188 326 326 326 T T T 12 240-259 191 191 191 330 330 330 T T T 13 260-279 186 186 186 323 323 323 F F F 14 280-299 187 187 187 325 325 325 F F F 15 300-319 190 190 190 328 328 328 F F F 16 320-339 183 183 183 322 322 322 T T T 17 340-359 191 191 191 330 330 330 T T T 18 360-379 191 191 191 330 330 330 F F F 19 380-399 188 188 188 326 326 326 F F F 20 400-419 188 188 188 326 326 326 F F F 21 420-439 188 188 188 326 326 326 F F F 22 440-459 188 188 188 326 326 326 F F F 23 460-479 190 190 190 328 328 328 T T T 24 480-499 188 188 188 326 326 326 T T T 25 500-519 190 190 190 327 327 327 F F F 26 520-539 190 190 190 328 328 328 F F F 27 540-559 192 192 192 331 331 331 F F F 28 560-579 192 192 192 331 331 331 F F F 29 580-599 190 190 190 329 329 329 F F F Max gen: 192, Max flush: 331 A Difference in extent metadata is expected here On loop 96 Verifying data now No change after verify Loop: 96 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 192 192 192 331 331 331 F F F 1 020-039 193 193 193 332 332 332 F F F 2 040-059 190 190 190 329 329 329 F F F 3 060-079 192 192 192 331 331 331 F F F 4 080-099 192 192 192 331 331 331 F F F 5 100-119 193 193 193 332 332 332 F F F 6 120-139 192 192 192 331 331 331 F F F 7 140-159 186 186 186 323 323 323 F F F 8 160-179 190 190 190 328 328 328 F F F 9 180-199 183 183 183 322 322 322 F F F 10 200-219 188 188 188 326 326 326 F F F 11 220-239 193 193 193 332 332 332 F F F 12 240-259 193 193 193 332 332 332 F F F 13 260-279 186 186 186 323 323 323 F F F 14 280-299 187 187 187 325 325 325 F F F 15 300-319 190 190 190 328 328 328 F F F 16 320-339 193 193 193 332 332 332 F F F 17 340-359 193 193 193 332 332 332 F F F 18 360-379 191 191 191 330 330 330 F F F 19 380-399 188 188 188 326 326 326 F F F 20 400-419 188 188 188 326 326 326 F F F 21 420-439 188 188 188 326 326 326 F F F 22 440-459 188 188 188 326 326 326 F F F 23 460-479 193 193 193 332 332 332 F F F 24 480-499 193 193 193 332 332 332 F F F 25 500-519 190 190 190 327 327 327 F F F 26 520-539 190 190 190 328 328 328 F F F 27 540-559 192 192 192 331 331 331 F F F 28 560-579 192 192 192 331 331 331 F F F 29 580-599 190 190 190 329 329 329 F F F Max gen: 193, Max flush: 332 Begin loop 97 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:58.718586729Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1956,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:58.763760521Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1956,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:58.764374821Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1956} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"0fd98d0d5d909611f8b408ccda2559d35fe117bb\",\n git_commit_timestamp: \"2024-11-08T20:05:44.000000000Z\",\n git_branch: \"mkeeter/drop-skipped-backpressure-guard\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:58.764411559Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1956} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:58.764426539Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1956} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:58.764441228Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1956} {"msg":"Crucible 3902046d-7630-4a61-9bda-be1f8714543f has session id: e651cf9f-56fe-458c-930b-cd6dc9a9ec34","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:58.764952683Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1956,"session_id":"e651cf9f-56fe-458c-930b-cd6dc9a9ec34"} {"msg":"Upstairs opts: Upstairs UUID: 3902046d-7630-4a61-9bda-be1f8714543f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:58.764999481Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1956,"session_id":"e651cf9f-56fe-458c-930b-cd6dc9a9ec34"} {"msg":"Crucible stats registered with UUID: 3902046d-7630-4a61-9bda-be1f8714543f","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:58.765022089Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1956,"session_id":"e651cf9f-56fe-458c-930b-cd6dc9a9ec34"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:58.797154638Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1956,"task":"crutest"} {"msg":"The guest has requested activation with gen:194","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:58.797186887Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1956} {"msg":"3902046d-7630-4a61-9bda-be1f8714543f active request set","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:58.797236605Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1956,"session_id":"e651cf9f-56fe-458c-930b-cd6dc9a9ec34"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:58.797256424Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1956,"client":"0","":"downstairs","session_id":"e651cf9f-56fe-458c-930b-cd6dc9a9ec34"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:58.797285292Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1956,"client":"0","":"downstairs","session_id":"e651cf9f-56fe-458c-930b-cd6dc9a9ec34"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:58.797306821Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1956,"client":"1","":"downstairs","session_id":"e651cf9f-56fe-458c-930b-cd6dc9a9ec34"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:58.7973246Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1956,"client":"1","":"downstairs","session_id":"e651cf9f-56fe-458c-930b-cd6dc9a9ec34"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:58.79734142Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1956,"client":"2","":"downstairs","session_id":"e651cf9f-56fe-458c-930b-cd6dc9a9ec34"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:58.797369308Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1956,"client":"2","":"downstairs","session_id":"e651cf9f-56fe-458c-930b-cd6dc9a9ec34"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:58.797399977Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1956,"":"io task","client":"2","":"downstairs","session_id":"e651cf9f-56fe-458c-930b-cd6dc9a9ec34"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:58.797434965Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1956,"":"io task","client":"0","":"downstairs","session_id":"e651cf9f-56fe-458c-930b-cd6dc9a9ec34"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:58.797469623Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1956,"":"io task","client":"1","":"downstairs","session_id":"e651cf9f-56fe-458c-930b-cd6dc9a9ec34"} {"msg":"ds_connection connected from Ok(127.0.0.1:33982)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:58.797503901Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1956,"":"io task","client":"0","":"downstairs","session_id":"e651cf9f-56fe-458c-930b-cd6dc9a9ec34"} {"msg":"ds_connection connected from Ok(127.0.0.1:39707)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:58.79753964Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1956,"":"io task","client":"2","":"downstairs","session_id":"e651cf9f-56fe-458c-930b-cd6dc9a9ec34"} {"msg":"ds_connection connected from Ok(127.0.0.1:39266)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:58.797667604Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1956,"":"io task","client":"1","":"downstairs","session_id":"e651cf9f-56fe-458c-930b-cd6dc9a9ec34"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:58.798391138Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1956,"client":"2","":"downstairs","session_id":"e651cf9f-56fe-458c-930b-cd6dc9a9ec34"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:58.798419427Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1956,"client":"2","":"downstairs","session_id":"e651cf9f-56fe-458c-930b-cd6dc9a9ec34"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:58.798434316Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1956,"client":"1","":"downstairs","session_id":"e651cf9f-56fe-458c-930b-cd6dc9a9ec34"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:58.798448085Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1956,"client":"1","":"downstairs","session_id":"e651cf9f-56fe-458c-930b-cd6dc9a9ec34"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:58.798461495Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1956,"client":"0","":"downstairs","session_id":"e651cf9f-56fe-458c-930b-cd6dc9a9ec34"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:58.798518782Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1956,"client":"0","":"downstairs","session_id":"e651cf9f-56fe-458c-930b-cd6dc9a9ec34"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:58.799084924Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1956,"client":"0","":"downstairs","session_id":"e651cf9f-56fe-458c-930b-cd6dc9a9ec34"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:58.799114283Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1956,"client":"1","":"downstairs","session_id":"e651cf9f-56fe-458c-930b-cd6dc9a9ec34"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:58.799130712Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1956,"client":"2","":"downstairs","session_id":"e651cf9f-56fe-458c-930b-cd6dc9a9ec34"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:58.799688625Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1956,"client":"1","":"downstairs","session_id":"e651cf9f-56fe-458c-930b-cd6dc9a9ec34"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:58.799741212Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1956,"client":"1","":"downstairs","session_id":"e651cf9f-56fe-458c-930b-cd6dc9a9ec34"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:58.799766831Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1956,"session_id":"e651cf9f-56fe-458c-930b-cd6dc9a9ec34"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:58.79978857Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1956,"client":"0","":"downstairs","session_id":"e651cf9f-56fe-458c-930b-cd6dc9a9ec34"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:58.799812379Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1956,"client":"0","":"downstairs","session_id":"e651cf9f-56fe-458c-930b-cd6dc9a9ec34"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:58.799836887Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1956,"session_id":"e651cf9f-56fe-458c-930b-cd6dc9a9ec34"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:58.799858477Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1956,"client":"2","":"downstairs","session_id":"e651cf9f-56fe-458c-930b-cd6dc9a9ec34"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:58.799881015Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1956,"client":"2","":"downstairs","session_id":"e651cf9f-56fe-458c-930b-cd6dc9a9ec34"} {"msg":"[0]R flush_numbers[0..12]: [331, 332, 329, 331, 331, 332, 331, 323, 328, 322, 326, 332]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:58.799904904Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1956,"":"downstairs","":"downstairs","session_id":"e651cf9f-56fe-458c-930b-cd6dc9a9ec34"} {"msg":"[0]R generation[0..12]: [192, 193, 190, 192, 192, 193, 192, 186, 190, 183, 188, 193]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:58.799931193Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1956,"":"downstairs","":"downstairs","session_id":"e651cf9f-56fe-458c-930b-cd6dc9a9ec34"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:58.799958652Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1956,"":"downstairs","":"downstairs","session_id":"e651cf9f-56fe-458c-930b-cd6dc9a9ec34"} {"msg":"[1]R flush_numbers[0..12]: [331, 332, 329, 331, 331, 332, 331, 323, 328, 322, 326, 332]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:58.79998872Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1956,"":"downstairs","":"downstairs","session_id":"e651cf9f-56fe-458c-930b-cd6dc9a9ec34"} {"msg":"[1]R generation[0..12]: [192, 193, 190, 192, 192, 193, 192, 186, 190, 183, 188, 193]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:58.800018649Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1956,"":"downstairs","":"downstairs","session_id":"e651cf9f-56fe-458c-930b-cd6dc9a9ec34"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-08T20:29:58.800049087Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1956,"":"downstairs","":"downstairs","session_id":"e651cf9f-56fe-458c-930b-cd6dc9a9ec34"} {"msg":"[2]R flush_numbers[0..12]: [331, 332, 329, 331, 331, 332, 331, 323, 328, 322, 326, 332]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:58.800083586Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1956,"":"downstairs","":"downstairs","session_id":"e651cf9f-56fe-458c-930b-cd6dc9a9ec34"} {"msg":"[2]R generation[0..12]: [192, 193, 190, 192, 192, 193, 192, 186, 190, 183, 188, 193]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:58.800114054Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1956,"":"downstairs","":"downstairs","session_id":"e651cf9f-56fe-458c-930b-cd6dc9a9ec34"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:58.800145532Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1956,"":"downstairs","":"downstairs","session_id":"e651cf9f-56fe-458c-930b-cd6dc9a9ec34"} {"msg":"Max found gen is 194","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:58.800173091Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1956,"":"downstairs","":"downstairs","session_id":"e651cf9f-56fe-458c-930b-cd6dc9a9ec34"} {"msg":"Generation requested: 194 >= found:194","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:58.8002005Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1956,"":"downstairs","":"downstairs","session_id":"e651cf9f-56fe-458c-930b-cd6dc9a9ec34"} {"msg":"Next flush: 333","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:58.800228228Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1956,"":"downstairs","":"downstairs","session_id":"e651cf9f-56fe-458c-930b-cd6dc9a9ec34"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:58.800255407Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1956,"":"downstairs","":"downstairs","session_id":"e651cf9f-56fe-458c-930b-cd6dc9a9ec34"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:58.800282346Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1956,"session_id":"e651cf9f-56fe-458c-930b-cd6dc9a9ec34"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:58.800307435Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1956,"client":"0","":"downstairs","session_idRegion: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ":"e651cf9f-56fe-458c-930b-cd6dc9a9ec34"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:58.800341173Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1956,"client":"1","":"downstairs","session_id":"e651cf9f-56fe-458c-930b-cd6dc9a9ec34"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:58.800358452Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1956,"client":"2","":"downstairs","session_id":"e651cf9f-56fe-458c-930b-cd6dc9a9ec34"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:58.800372191Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1956,"session_id":"e651cf9f-56fe-458c-930b-cd6dc9a9ec34"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:58.800387871Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1956,"session_id":"e651cf9f-56fe-458c-930b-cd6dc9a9ec34"} {"msg":"3902046d-7630-4a61-9bda-be1f8714543f is now active with session: e651cf9f-56fe-458c-930b-cd6dc9a9ec34","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:58.80040097Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1956,"session_id":"e651cf9f-56fe-458c-930b-cd6dc9a9ec34"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:58.800413199Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1956,"session_id":"e651cf9f-56fe-458c-930b-cd6dc9a9ec34"} {"msg":"The guest has finished waiting for activation with:194","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:29:58.800425519Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1956} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 383 len 36864 data: 9 12 10 9 11 11 11 11 11 02/30 Read block 94 len 12288 03/30 Write block 421 len 16384 data: 17 16 14 14 04/30 Flush 05/30 Write block 30 len 32768 data: 7 9 9 12 14 14 14 14 06/30 Write block 79 len 4096 data: 13 07/30 Read block 483 len 28672 08/30 Read block 363 len 40960 09/30 Read block 395 len 36864 10/30 Read block 128 len 20480 11/30 Read block 37 len 8192 12/30 Flush 13/30 Write block 103 len 12288 data: 15 17 18 14/30 Write block 524 len 16384 data: 8 10 10 12 15/30 Read block 558 len 28672 16/30 Write block 275 len 32768 data: 20 21 19 15 14 14 10 13 17/30 Read block 446 len 8192 18/30 Write block 333 len 32768 data: 15 15 14 13 15 18 20 20 19/30 Read block 418 len 8192 20/30 Read block 5 len 24576 21/30 Write block 338 len 4096 data: 19 22/30 Read block 134 len 16384 23/30 Write block 17 len 16384 data: 17 15 15 15 24/30 Read block 422 len 36864 25/30 Read block 0 len 20480 26/30 Read block 348 len 24576 27/30 Write block 464 len 4096 data: 10 28/30 Write block 38 len 4096 data: 17 29/30 Write block 291 len 32768 data: 13 14 14 15 15 12 14 15 30/30 Write block 1 len 8192 data: 6 9 ---------------------------------------------------------------- Crucible gen:194 GIO:true work queues: Upstairs:0 downstairs:18 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1012 Acked Write 3 Done Done Done false 1013 Acked Write 4 Done Done Done false 1014 Acked Read 7 Done Done Done false 1015 Acked Write 8 Done Done Done false 1016 Acked Read 2 Done Done Done false 1017 Acked Write 8 Done Done Done false 1018 Acked Read 2 Done Done Done false 1019 Acked Read 6 Done Done Done false 1020 Acked Write 1 Done Done Done false 1021 Acked Read 4 Done Done Done false 1022 Acked Write 4 Done Done Done false 1023 Acked Read 9 Done Done Done false 1024 Acked Read 5 Done Done Done false 1025 Acked Read 6 Done Done Done false 1026 Acked Write 1 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Write 8 Sent Sent Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 16 16 16 48 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1011 1011 1011 Downstairs last five completed: 1011 1010 1009 1008 1007 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:29:58.823343909Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1956} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 192 192 192 331 331 331 T T T 1 020-039 194 194 194 334 334 334 T T T 2 040-059 190 190 190 329 329 329 F F F 3 060-079 194 194 194 334 334 334 F F F 4 080-099 192 192 192 331 331 331 F F F 5 100-119 193 193 193 332 332 332 T T T 6 120-139 192 192 192 331 331 331 F F F 7 140-159 186 186 186 323 323 323 F F F 8 160-179 190 190 190 328 328 328 F F F 9 180-199 183 183 183 322 322 322 F F F 10 200-219 188 188 188 326 326 326 F F F 11 220-239 193 193 193 332 332 332 F F F 12 240-259 193 193 193 332 332 332 F F F 13 260-279 186 186 186 323 323 323 T T T 14 280-299 187 187 187 325 325 325 T T T 15 300-319 190 190 190 328 328 328 F F F 16 320-339 193 193 193 332 332 332 T T T 17 340-359 193 193 193 332 332 332 T T T 18 360-379 191 191 191 330 330 330 F F F 19 380-399 194 194 194 333 333 333 F F F 20 400-419 188 188 188 326 326 326 F F F 21 420-439 194 194 194 333 333 333 F F F 22 440-459 188 188 188 326 326 326 F F F 23 460-479 193 193 193 332 332 332 T T T 24 480-499 193 193 193 332 332 332 F F F 25 500-519 190 190 190 327 327 327 F F F 26 520-539 190 190 190 328 328 328 T T T 27 540-559 192 192 192 331 331 331 F F F 28 560-579 192 192 192 331 331 331 F F F 29 580-599 190 190 190 329 329 329 F F F Max gen: 194, Max flush: 334 A Difference in extent metadata is expected here On loop 97 Verifying data now No change after verify Loop: 97 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 195 195 195 335 335 335 F F F 1 020-039 195 195 195 335 335 335 F F F 2 040-059 190 190 190 329 329 329 F F F 3 060-079 194 194 194 334 334 334 F F F 4 080-099 192 192 192 331 331 331 F F F 5 100-119 195 195 195 335 335 335 F F F 6 120-139 192 192 192 331 331 331 F F F 7 140-159 186 186 186 323 323 323 F F F 8 160-179 190 190 190 328 328 328 F F F 9 180-199 183 183 183 322 322 322 F F F 10 200-219 188 188 188 326 326 326 F F F 11 220-239 193 193 193 332 332 332 F F F 12 240-259 193 193 193 332 332 332 F F F 13 260-279 195 195 195 335 335 335 F F F 14 280-299 195 195 195 335 335 335 F F F 15 300-319 190 190 190 328 328 328 F F F 16 320-339 195 195 195 335 335 335 F F F 17 340-359 195 195 195 335 335 335 F F F 18 360-379 191 191 191 330 330 330 F F F 19 380-399 194 194 194 333 333 333 F F F 20 400-419 188 188 188 326 326 326 F F F 21 420-439 194 194 194 333 333 333 F F F 22 440-459 188 188 188 326 326 326 F F F 23 460-479 195 195 195 335 335 335 F F F 24 480-499 193 193 193 332 332 332 F F F 25 500-519 190 190 190 327 327 327 F F F 26 520-539 195 195 195 335 335 335 F F F 27 540-559 192 192 192 331 331 331 F F F 28 560-579 192 192 192 331 331 331 F F F 29 580-599 190 190 190 329 329 329 F F F Max gen: 195, Max flush: 335 Begin loop 98 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:30:03.205019849Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1964,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:30:03.249927534Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1964,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:30:03.250658268Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1964} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"0fd98d0d5d909611f8b408ccda2559d35fe117bb\",\n git_commit_timestamp: \"2024-11-08T20:05:44.000000000Z\",\n git_branch: \"mkeeter/drop-skipped-backpressure-guard\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:30:03.250696916Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1964} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:30:03.250712735Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1964} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:30:03.250727105Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1964} {"msg":"Crucible 3258b9f0-9fbc-4ffb-bba5-80b1102e403d has session id: a2b41f80-a1e7-43b1-bdb9-47aa1bbbd8bc","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:30:03.251265499Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1964,"session_id":"a2b41f80-a1e7-43b1-bdb9-47aa1bbbd8bc"} {"msg":"Upstairs opts: Upstairs UUID: 3258b9f0-9fbc-4ffb-bba5-80b1102e403d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:30:03.251304577Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1964,"session_id":"a2b41f80-a1e7-43b1-bdb9-47aa1bbbd8bc"} {"msg":"Crucible stats registered with UUID: 3258b9f0-9fbc-4ffb-bba5-80b1102e403d","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:30:03.251320936Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1964,"session_id":"a2b41f80-a1e7-43b1-bdb9-47aa1bbbd8bc"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:30:03.282633746Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1964,"task":"crutest"} {"msg":"The guest has requested activation with gen:196","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:30:03.282668364Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1964} {"msg":"3258b9f0-9fbc-4ffb-bba5-80b1102e403d active request set","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:30:03.282735821Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1964,"session_id":"a2b41f80-a1e7-43b1-bdb9-47aa1bbbd8bc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:30:03.282770149Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1964,"client":"0","":"downstairs","session_id":"a2b41f80-a1e7-43b1-bdb9-47aa1bbbd8bc"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:30:03.282801327Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1964,"client":"0","":"downstairs","session_id":"a2b41f80-a1e7-43b1-bdb9-47aa1bbbd8bc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:30:03.282820526Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1964,"client":"1","":"downstairs","session_id":"a2b41f80-a1e7-43b1-bdb9-47aa1bbbd8bc"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:30:03.282841985Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1964,"client":"1","":"downstairs","session_id":"a2b41f80-a1e7-43b1-bdb9-47aa1bbbd8bc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:30:03.282859575Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1964,"client":"2","":"downstairs","session_id":"a2b41f80-a1e7-43b1-bdb9-47aa1bbbd8bc"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:30:03.282876703Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1964,"client":"2","":"downstairs","session_id":"a2b41f80-a1e7-43b1-bdb9-47aa1bbbd8bc"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:30:03.282893513Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1964,"":"io task","client":"2","":"downstairs","session_id":"a2b41f80-a1e7-43b1-bdb9-47aa1bbbd8bc"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:30:03.282911402Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1964,"":"io task","client":"0","":"downstairs","session_id":"a2b41f80-a1e7-43b1-bdb9-47aa1bbbd8bc"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:30:03.282929201Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1964,"":"io task","client":"1","":"downstairs","session_id":"a2b41f80-a1e7-43b1-bdb9-47aa1bbbd8bc"} {"msg":"ds_connection connected from Ok(127.0.0.1:51293)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:30:03.28295515Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1964,"":"io task","client":"0","":"downstairs","session_id":"a2b41f80-a1e7-43b1-bdb9-47aa1bbbd8bc"} {"msg":"ds_connection connected from Ok(127.0.0.1:37536)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:30:03.282989288Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1964,"":"io task","client":"2","":"downstairs","session_id":"a2b41f80-a1e7-43b1-bdb9-47aa1bbbd8bc"} {"msg":"ds_connection connected from Ok(127.0.0.1:46151)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:30:03.283224937Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1964,"":"io task","client":"1","":"downstairs","session_id":"a2b41f80-a1e7-43b1-bdb9-47aa1bbbd8bc"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:30:03.283809608Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1964,"client":"2","":"downstairs","session_id":"a2b41f80-a1e7-43b1-bdb9-47aa1bbbd8bc"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:30:03.283852446Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1964,"client":"2","":"downstairs","session_id":"a2b41f80-a1e7-43b1-bdb9-47aa1bbbd8bc"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:30:03.283883514Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1964,"client":"0","":"downstairs","session_id":"a2b41f80-a1e7-43b1-bdb9-47aa1bbbd8bc"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:30:03.283918463Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1964,"client":"0","":"downstairs","session_id":"a2b41f80-a1e7-43b1-bdb9-47aa1bbbd8bc"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:30:03.283985679Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1964,"client":"1","":"downstairs","session_id":"a2b41f80-a1e7-43b1-bdb9-47aa1bbbd8bc"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:30:03.284016788Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1964,"client":"1","":"downstairs","session_id":"a2b41f80-a1e7-43b1-bdb9-47aa1bbbd8bc"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:30:03.284479265Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1964,"client":"0","":"downstairs","session_id":"a2b41f80-a1e7-43b1-bdb9-47aa1bbbd8bc"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:30:03.284514523Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1964,"client":"2","":"downstairs","session_id":"a2b41f80-a1e7-43b1-bdb9-47aa1bbbd8bc"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:30:03.284625518Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1964,"client":"1","":"downstairs","session_id":"a2b41f80-a1e7-43b1-bdb9-47aa1bbbd8bc"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:30:03.285116354Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1964,"client":"0","":"downstairs","session_id":"a2b41f80-a1e7-43b1-bdb9-47aa1bbbd8bc"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:30:03.285145743Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1964,"client":"0","":"downstairs","session_id":"a2b41f80-a1e7-43b1-bdb9-47aa1bbbd8bc"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:30:03.285160312Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1964,"session_id":"a2b41f80-a1e7-43b1-bdb9-47aa1bbbd8bc"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:30:03.285172851Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1964,"client":"2","":"downstairs","session_id":"a2b41f80-a1e7-43b1-bdb9-47aa1bbbd8bc"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:30:03.285186911Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1964,"client":"2","":"downstairs","session_id":"a2b41f80-a1e7-43b1-bdb9-47aa1bbbd8bc"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:30:03.28520061Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1964,"session_id":"a2b41f80-a1e7-43b1-bdb9-47aa1bbbd8bc"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:30:03.285212989Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1964,"client":"1","":"downstairs","session_id":"a2b41f80-a1e7-43b1-bdb9-47aa1bbbd8bc"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:30:03.285226109Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1964,"client":"1","":"downstairs","session_id":"a2b41f80-a1e7-43b1-bdb9-47aa1bbbd8bc"} {"msg":"[0]R flush_numbers[0..12]: [335, 335, 329, 334, 331, 335, 331, 323, 328, 322, 326, 332]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:30:03.285239958Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1964,"":"downstairs","":"downstairs","session_id":"a2b41f80-a1e7-43b1-bdb9-47aa1bbbd8bc"} {"msg":"[0]R generation[0..12]: [195, 195, 190, 194, 192, 195, 192, 186, 190, 183, 188, 193]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:30:03.285253858Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1964,"":"downstairs","":"downstairs","session_id":"a2b41f80-a1e7-43b1-bdb9-47aa1bbbd8bc"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:30:03.285267857Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1964,"":"downstairs","":"downstairs","session_id":"a2b41f80-a1e7-43b1-bdb9-47aa1bbbd8bc"} {"msg":"[1]R flush_numbers[0..12]: [335, 335, 329, 334, 331, 335, 331, 323, 328, 322, 326, 332]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:30:03.285281626Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1964,"":"downstairs","":"downstairs","session_id":"a2b41f80-a1e7-43b1-bdb9-47aa1bbbd8bc"} {"msg":"[1]R generation[0..12]: [195, 195, 190, 194, 192, 195, 192, 186, 190, 183, 188, 193]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:30:03.285299035Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1964,"":"downstairs","":"downstairs","session_id":"a2b41f80-a1e7-43b1-bdb9-47aa1bbbd8bc"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:30:03.285313284Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1964,"":"downstairs","":"downstairs","session_id":"a2b41f80-a1e7-43b1-bdb9-47aa1bbbd8bc"} {"msg":"[2]R flush_numbers[0..12]: [335, 335, 329, 334, 331, 335, 331, 323, 328, 322, 326, 332]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:30:03.285327014Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1964,"":"downstairs","":"downstairs","session_id":"a2b41f80-a1e7-43b1-bdb9-47aa1bbbd8bc"} {"msg":"[2]R generation[0..12]: [195, 195, 190, 194, 192, 195, 192, 186, 190, 183, 188, 193]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:30:03.285340623Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1964,"":"downstairs","":"downstairs","session_id":"a2b41f80-a1e7-43b1-bdb9-47aa1bbbd8bc"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:30:03.285354502Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1964,"":"downstairs","":"downstairs","session_id":"a2b41f80-a1e7-43b1-bdb9-47aa1bbbd8bc"} {"msg":"Max found gen is 196","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:30:03.285367482Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1964,"":"downstairs","":"downstairs","session_id":"a2b41f80-a1e7-43b1-bdb9-47aa1bbbd8bc"} {"msg":"Generation requested: 196 >= found:196","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:30:03.285380541Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1964,"":"downstairs","":"downstairs","session_id":"a2b41f80-a1e7-43b1-bdb9-47aa1bbbd8bc"} {Wait for a query_work_queue command to finish before sending IO "msg":"Next flush: 336","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:30:03.2854019Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1964,"":"downstairs","":"downstairs","session_id":"a2b41f80-a1e7-43b1-bdb9-47aa1bbbd8bc"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:30:03.285415259Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1964,"":"downstairs","":"downstairs","session_id":"a2b41f80-a1e7-43b1-bdb9-47aa1bbbd8bc"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:30:03.285428369Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1964,"session_id":"a2b41f80-a1e7-43b1-bdb9-47aa1bbbd8bc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:30:03.285440548Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1964,"client":"0","":"downstairs","session_id":"a2b41f80-a1e7-43b1-bdb9-47aa1bbbd8bc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:30:03.285453828Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1964,"client":"1","":"downstairs","session_id":"a2b41f80-a1e7-43b1-bdb9-47aa1bbbd8bc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:30:03.285466897Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1964,"client":"2","":"downstairs","session_id":"a2b41f80-a1e7-43b1-bdb9-47aa1bbbd8bc"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:30:03.285480216Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1964,"session_id":"a2b41f80-a1e7-43b1-bdb9-47aa1bbbd8bc"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:30:03.285492426Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1964,"session_id":"a2b41f80-a1e7-43b1-bdb9-47aa1bbbd8bc"} {"msg":"3258b9f0-9fbc-4ffb-bba5-80b1102e403d is now active with session: a2b41f80-a1e7-43b1-bdb9-47aa1bbbd8bc","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:30:03.285507125Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1964,"session_id":"a2b41f80-a1e7-43b1-bdb9-47aa1bbbd8bc"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:30:03.285519434Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1964,"session_id":"a2b41f80-a1e7-43b1-bdb9-47aa1bbbd8bc"} {"msg":"The guest has finished waiting for activation with:196","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:30:03.285531694Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1964} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 191 len 20480 data: 13 13 15 15 14 02/30 Read block 346 len 28672 03/30 Write block 141 len 36864 data: 14 13 16 17 18 17 18 16 17 04/30 Write block 75 len 20480 data: 16 16 17 16 14 05/30 Write block 586 len 8192 data: 20 22 06/30 Read block 363 len 28672 07/30 Flush 08/30 Flush 09/30 Write block 196 len 16384 data: 14 14 14 12 10/30 Read block 76 len 16384 11/30 Read block 155 len 20480 12/30 Write block 443 len 28672 data: 14 13 13 16 16 13 15 13/30 Write block 468 len 4096 data: 20 14/30 Read block 512 len 32768 15/30 Read block 16 len 24576 16/30 Read block 188 len 40960 17/30 Write block 403 len 12288 data: 12 11 12 18/30 Write block 166 len 24576 data: 14 15 19 16 14 13 19/30 Flush 20/30 Write block 9 len 16384 data: 11 11 12 16 21/30 Write block 589 len 8192 data: 14 13 22/30 Read block 357 len 12288 23/30 Read block 589 len 28672 24/30 Write block 436 len 36864 data: 25 23 22 20 16 19 20 15 14 25/30 Write block 394 len 20480 data: 13 13 14 16 14 26/30 Write block 490 len 12288 data: 18 15 14 27/30 Read block 423 len 12288 28/30 Write block 555 len 28672 data: 14 14 14 15 15 13 13 29/30 Write block 151 len 20480 data: 16 12 10 11 9 30/30 Read block 442 len 16384 ---------------------------------------------------------------- Crucible gen:196 GIO:true work queues: Upstairs:0 downstairs:11 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1019 Acked Write 4 Done Done Done false 1020 Acked Write 2 Done Done Done false 1021 Acked Read 3 Done Done Done false 1022 Acked Read 7 Done Done Done false 1023 Acked Write 9 Done Done Done false 1024 Acked Write 5 Done Done Done false 1025 Acked Write 3 Done Done Done false 1026 Acked Read 3 Done Done Done false 1027 Acked Write 7 Done Done Done false 1028 Acked Write 5 Sent Done Done false 1029 Acked Read 4 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 0 1 3 Done 9 11 10 30 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1018 1018 1018 Downstairs last five completed: 1018 1017 1016 1015 1014 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:30:03.310333972Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1964} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 195 195 195 335 335 335 T T T 1 020-039 195 195 195 335 335 335 F F F 2 040-059 190 190 190 329 329 329 F F F 3 060-079 196 196 196 336 336 336 F F F 4 080-099 192 192 192 331 331 331 F F F 5 100-119 195 195 195 335 335 335 F F F 6 120-139 192 192 192 331 331 331 F F F 7 140-159 196 196 196 336 336 336 T T T 8 160-179 196 196 196 338 338 338 F F F 9 180-199 196 196 196 338 338 338 F F F 10 200-219 188 188 188 326 326 326 F F F 11 220-239 193 193 193 332 332 332 F F F 12 240-259 193 193 193 332 332 332 F F F 13 260-279 195 195 195 335 335 335 F F F 14 280-299 195 195 195 335 335 335 F F F 15 300-319 190 190 190 328 328 328 F F F 16 320-339 195 195 195 335 335 335 F F F 17 340-359 195 195 195 335 335 335 F F F 18 360-379 191 191 191 330 330 330 F F F 19 380-399 194 194 194 333 333 333 T T T 20 400-419 196 196 196 338 338 338 F F F 21 420-439 194 194 194 333 333 333 T T T 22 440-459 196 196 196 338 338 338 T T T 23 460-479 196 196 196 338 338 338 F F F 24 480-499 193 193 193 332 332 332 T T T 25 500-519 190 190 190 327 327 327 F F F 26 520-539 195 195 195 335 335 335 F F F 27 540-559 192 192 192 331 331 331 T T T 28 560-579 192 192 192 331 331 331 T T T 29 580-599 196 196 196 336 336 336 T T T Max gen: 196, Max flush: 338 A Difference in extent metadata is expected here On loop 98 Verifying data now No change after verify Loop: 98 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 197 197 197 339 339 339 F F F 1 020-039 195 195 195 335 335 335 F F F 2 040-059 190 190 190 329 329 329 F F F 3 060-079 196 196 196 336 336 336 F F F 4 080-099 192 192 192 331 331 331 F F F 5 100-119 195 195 195 335 335 335 F F F 6 120-139 192 192 192 331 331 331 F F F 7 140-159 197 197 197 339 339 339 F F F 8 160-179 196 196 196 338 338 338 F F F 9 180-199 196 196 196 338 338 338 F F F 10 200-219 188 188 188 326 326 326 F F F 11 220-239 193 193 193 332 332 332 F F F 12 240-259 193 193 193 332 332 332 F F F 13 260-279 195 195 195 335 335 335 F F F 14 280-299 195 195 195 335 335 335 F F F 15 300-319 190 190 190 328 328 328 F F F 16 320-339 195 195 195 335 335 335 F F F 17 340-359 195 195 195 335 335 335 F F F 18 360-379 191 191 191 330 330 330 F F F 19 380-399 197 197 197 339 339 339 F F F 20 400-419 196 196 196 338 338 338 F F F 21 420-439 197 197 197 339 339 339 F F F 22 440-459 197 197 197 339 339 339 F F F 23 460-479 196 196 196 338 338 338 F F F 24 480-499 197 197 197 339 339 339 F F F 25 500-519 190 190 190 327 327 327 F F F 26 520-539 195 195 195 335 335 335 F F F 27 540-559 197 197 197 339 339 339 F F F 28 560-579 197 197 197 339 339 339 F F F 29 580-599 197 197 197 339 339 339 F F F Max gen: 197, Max flush: 339 Begin loop 99 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:30:07.701646781Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1972,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:30:07.745526616Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1972,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:30:07.746127117Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1972} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"0fd98d0d5d909611f8b408ccda2559d35fe117bb\",\n git_commit_timestamp: \"2024-11-08T20:05:44.000000000Z\",\n git_branch: \"mkeeter/drop-skipped-backpressure-guard\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:30:07.746165285Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1972} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:30:07.746184304Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1972} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:30:07.746199733Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1972} {"msg":"Crucible 7cffe358-b65b-452a-9a7a-15ce7a78304f has session id: 0c33f15e-0075-4e3a-b05b-9ce3014cc337","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:30:07.746749416Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1972,"session_id":"0c33f15e-0075-4e3a-b05b-9ce3014cc337"} {"msg":"Upstairs opts: Upstairs UUID: 7cffe358-b65b-452a-9a7a-15ce7a78304f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:30:07.746796644Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1972,"session_id":"0c33f15e-0075-4e3a-b05b-9ce3014cc337"} {"msg":"Crucible stats registered with UUID: 7cffe358-b65b-452a-9a7a-15ce7a78304f","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:30:07.746821323Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1972,"session_id":"0c33f15e-0075-4e3a-b05b-9ce3014cc337"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:30:07.778202269Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1972,"task":"crutest"} {"msg":"The guest has requested activation with gen:198","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:30:07.778236027Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1972} {"msg":"7cffe358-b65b-452a-9a7a-15ce7a78304f active request set","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:30:07.778286675Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1972,"session_id":"0c33f15e-0075-4e3a-b05b-9ce3014cc337"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:30:07.778307404Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1972,"client":"0","":"downstairs","session_id":"0c33f15e-0075-4e3a-b05b-9ce3014cc337"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:30:07.778334343Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1972,"client":"0","":"downstairs","session_id":"0c33f15e-0075-4e3a-b05b-9ce3014cc337"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:30:07.778352102Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1972,"client":"1","":"downstairs","session_id":"0c33f15e-0075-4e3a-b05b-9ce3014cc337"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:30:07.778369401Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1972,"client":"1","":"downstairs","session_id":"0c33f15e-0075-4e3a-b05b-9ce3014cc337"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:30:07.77838608Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1972,"client":"2","":"downstairs","session_id":"0c33f15e-0075-4e3a-b05b-9ce3014cc337"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:30:07.778407169Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1972,"client":"2","":"downstairs","session_id":"0c33f15e-0075-4e3a-b05b-9ce3014cc337"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:30:07.778429758Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1972,"":"io task","client":"2","":"downstairs","session_id":"0c33f15e-0075-4e3a-b05b-9ce3014cc337"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:30:07.778460226Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1972,"":"io task","client":"0","":"downstairs","session_id":"0c33f15e-0075-4e3a-b05b-9ce3014cc337"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:30:07.778491325Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1972,"":"io task","client":"1","":"downstairs","session_id":"0c33f15e-0075-4e3a-b05b-9ce3014cc337"} {"msg":"ds_connection connected from Ok(127.0.0.1:44758)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:30:07.778523073Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1972,"":"io task","client":"0","":"downstairs","session_id":"0c33f15e-0075-4e3a-b05b-9ce3014cc337"} {"msg":"ds_connection connected from Ok(127.0.0.1:55941)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:30:07.778557721Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1972,"":"io task","client":"1","":"downstairs","session_id":"0c33f15e-0075-4e3a-b05b-9ce3014cc337"} {"msg":"ds_connection connected from Ok(127.0.0.1:33264)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:30:07.778705744Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1972,"":"io task","client":"2","":"downstairs","session_id":"0c33f15e-0075-4e3a-b05b-9ce3014cc337"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:30:07.779319794Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1972,"client":"2","":"downstairs","session_id":"0c33f15e-0075-4e3a-b05b-9ce3014cc337"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:30:07.779348263Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1972,"client":"2","":"downstairs","session_id":"0c33f15e-0075-4e3a-b05b-9ce3014cc337"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:30:07.779493756Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1972,"client":"1","":"downstairs","session_id":"0c33f15e-0075-4e3a-b05b-9ce3014cc337"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:30:07.779537654Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1972,"client":"1","":"downstairs","session_id":"0c33f15e-0075-4e3a-b05b-9ce3014cc337"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:30:07.779564612Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1972,"client":"0","":"downstairs","session_id":"0c33f15e-0075-4e3a-b05b-9ce3014cc337"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:30:07.779586001Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1972,"client":"0","":"downstairs","session_id":"0c33f15e-0075-4e3a-b05b-9ce3014cc337"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:30:07.779985492Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1972,"client":"2","":"downstairs","session_id":"0c33f15e-0075-4e3a-b05b-9ce3014cc337"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:30:07.78001711Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1972,"client":"1","":"downstairs","session_id":"0c33f15e-0075-4e3a-b05b-9ce3014cc337"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:30:07.780259768Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1972,"client":"0","":"downstairs","session_id":"0c33f15e-0075-4e3a-b05b-9ce3014cc337"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:30:07.7806311Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1972,"client":"1","":"downstairs","session_id":"0c33f15e-0075-4e3a-b05b-9ce3014cc337"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:30:07.780671568Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1972,"client":"1","":"downstairs","session_id":"0c33f15e-0075-4e3a-b05b-9ce3014cc337"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:30:07.780702837Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1972,"session_id":"0c33f15e-0075-4e3a-b05b-9ce3014cc337"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:30:07.780729595Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1972,"client":"2","":"downstairs","session_id":"0c33f15e-0075-4e3a-b05b-9ce3014cc337"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:30:07.780753334Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1972,"client":"2","":"downstairs","session_id":"0c33f15e-0075-4e3a-b05b-9ce3014cc337"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:30:07.780776273Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1972,"session_id":"0c33f15e-0075-4e3a-b05b-9ce3014cc337"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:30:07.780798012Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1972,"client":"0","":"downstairs","session_id":"0c33f15e-0075-4e3a-b05b-9ce3014cc337"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:30:07.780822541Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1972,"client":"0","":"downstairs","session_id":"0c33f15e-0075-4e3a-b05b-9ce3014cc337"} {"msg":"[0]R flush_numbers[0..12]: [339, 335, 329, 336, 331, 335, 331, 339, 338, 338, 326, 332]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:30:07.78084784Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1972,"":"downstairs","":"downstairs","session_id":"0c33f15e-0075-4e3a-b05b-9ce3014cc337"} {"msg":"[0]R generation[0..12]: [197, 195, 190, 196, 192, 195, 192, 197, 196, 196, 188, 193]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:30:07.780871799Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1972,"":"downstairs","":"downstairs","session_id":"0c33f15e-0075-4e3a-b05b-9ce3014cc337"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:30:07.780895557Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1972,"":"downstairs","":"downstairs","session_id":"0c33f15e-0075-4e3a-b05b-9ce3014cc337"} {"msg":"[1]R flush_numbers[0..12]: [339, 335, 329, 336, 331, 335, 331, 339, 338, 338, 326, 332]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:30:07.780921296Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1972,"":"downstairs","":"downstairs","session_id":"0c33f15e-0075-4e3a-b05b-9ce3014cc337"} {"msg":"[1]R generation[0..12]: [197, 195, 190, 196, 192, 195, 192, 197, 196, 196, 188, 193]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:30:07.780948255Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1972,"":"downstairs","":"downstairs","session_id":"0c33f15e-0075-4e3a-b05b-9ce3014cc337"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:30:07.780974394Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1972,"":"downstairs","":"downstairs","session_id":"0c33f15e-0075-4e3a-b05b-9ce3014cc337"} {"msg":"[2]R flush_numbers[0..12]: [339, 335, 329, 336, 331, 335, 331, 339, 338, 338, 326, 332]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:30:07.781005572Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1972,"":"downstairs","":"downstairs","session_id":"0c33f15e-0075-4e3a-b05b-9ce3014cc337"} {"msg":"[2]R generation[0..12]: [197, 195, 190, 196, 192, 195, 192, 197, 196, 196, 188, 193]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-08T20:30:07.781034951Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1972,"":"downstairs","":"downstairs","session_id":"0c33f15e-0075-4e3a-b05b-9ce3014cc337"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:30:07.781073758Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1972,"":"downstairs","":"downstairs","session_id":"0c33f15e-0075-4e3a-b05b-9ce3014cc337"} {"msg":"Max found gen is 198","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:30:07.781103267Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1972,"":"downstairs","":"downstairs","session_id":"0c33f15e-0075-4e3a-b05b-9ce3014cc337"} {"msg":"Generation requested: 198 >= found:198","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:30:07.781132276Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1972,"":"downstairs","":"downstairs","session_id":"0c33f15e-0075-4e3a-b05b-9ce3014cc337"} {"msg":"Next flush: 340","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:30:07.781160394Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1972,"":"downstairs","":"downstairs","session_id":"0c33f15e-0075-4e3a-b05b-9ce3014cc337"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:30:07.781187743Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1972,"":"downstairs","":"downstairs","session_id":"0c33f15e-0075-4e3a-b05b-9ce3014cc337"} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:30:07.781219442Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1972,"session_id":"0c33f15e-0075-4e3a-b05b-9ce3014cc337"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:30:07.78124622Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1972,"client":"0","":"downstairs","session_id":"0c33f15e-0075-4e3a-b05b-9ce3014cc337"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:30:07.781272969Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1972,"client":"1","":"downstairs","session_id":"0c33f15e-0075-4e3a-b05b-9ce3014cc337"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:30:07.781297578Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1972,"client":"2","":"downstairs","session_id":"0c33f15e-0075-4e3a-b05b-9ce3014cc337"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:30:07.781312257Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1972,"session_id":"0c33f15e-0075-4e3a-b05b-9ce3014cc337"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:30:07.781324866Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1972,"session_id":"0c33f15e-0075-4e3a-b05b-9ce3014cc337"} {"msg":"7cffe358-b65b-452a-9a7a-15ce7a78304f is now active with session: 0c33f15e-0075-4e3a-b05b-9ce3014cc337","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:30:07.781337876Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1972,"session_id":"0c33f15e-0075-4e3a-b05b-9ce3014cc337"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:30:07.781350015Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1972,"session_id":"0c33f15e-0075-4e3a-b05b-9ce3014cc337"} {"msg":"The guest has finished waiting for activation with:198","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:30:07.781365064Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1972} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 442 len 24576 data: 21 16 15 14 17 17 02/30 Write block 22 len 28672 data: 17 14 15 14 14 11 9 03/30 Read block 316 len 36864 04/30 Write block 331 len 24576 data: 13 14 16 16 15 14 05/30 Write block 141 len 20480 data: 15 14 17 18 19 06/30 Read block 359 len 40960 07/30 Write block 154 len 8192 data: 12 10 08/30 Write block 471 len 28672 data: 20 19 18 18 20 19 22 09/30 Read block 508 len 40960 10/30 Write block 42 len 4096 data: 17 11/30 Write block 86 len 20480 data: 14 14 15 14 15 12/30 Read block 447 len 16384 13/30 Read block 525 len 24576 14/30 Write block 508 len 20480 data: 16 14 15 17 15 15/30 Read block 5 len 36864 16/30 Read block 212 len 40960 17/30 Read block 554 len 20480 18/30 Read block 78 len 32768 19/30 Write block 548 len 16384 data: 16 12 12 9 20/30 Write block 67 len 16384 data: 11 11 9 13 21/30 Read block 420 len 20480 22/30 Write block 264 len 40960 data: 13 16 15 13 16 18 19 19 19 21 23/30 Read block 126 len 16384 24/30 Write block 25 len 16384 data: 15 15 12 10 25/30 Write block 73 len 12288 data: 15 16 17 26/30 Write block 138 len 24576 data: 14 14 11 16 15 18 27/30 Write block 126 len 20480 data: 14 12 11 9 12 28/30 Write block 472 len 12288 data: 20 19 19 29/30 Read block 163 len 20480 30/30 Read block 75 len 12288 ---------------------------------------------------------------- Crucible gen:198 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 6 Done Done Done false 1001 Acked Write 7 Done Done Done false 1002 Acked Read 9 Done Done Done false 1003 Acked Write 6 Done Done Done false 1004 Acked Write 5 Done Done Done false 1005 Acked Read 10 Done Done Done false 1006 Acked Write 2 Done Done Done false 1007 Acked Write 7 Done Done Done false 1008 Acked Read 10 Done Done Done false 1009 Acked Write 1 Done Done Done false 1010 Acked Write 5 Done Done Done false 1011 Acked Read 4 Done Done Done false 1012 Acked Read 6 Done Done Done false 1013 Acked Write 5 Done Done Done false 1014 Acked Read 9 Done Done Done false 1015 Acked Read 10 Done Done Done false 1016 Acked Read 5 Done Done Done false 1017 Acked Read 8 Done Done Done false 1018 Acked Write 4 Done Done Done false 1019 Acked Write 4 Done Done Done false 1020 Acked Read 5 Done Done Done false 1021 Acked Write 10 Done Done Done false 1022 Acked Read 4 Done Done Done false 1023 Acked Write 4 Done Done Done false 1024 Acked Write 3 Done Done Done false 1025 Acked Write 6 Done Done Done false 1026 Acked Write 5 Done Done Done false 1027 Acked Write 3 Done Done Done false 1028 Acked Read 5 Done Done Done false 1029 Acked Read 3 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 30 30 30 90 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 0 0 0 Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:30:07.800460811Z","hostname":"w-01JC6Q01W6ASASK2KCFV97BEDW","pid":1972} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 197 197 197 339 339 339 F F F 1 020-039 195 195 195 335 335 335 T T T 2 040-059 190 190 190 329 329 329 T T T 3 060-079 196 196 196 336 336 336 T T T 4 080-099 192 192 192 331 331 331 T T T 5 100-119 195 195 195 335 335 335 F F F 6 120-139 192 192 192 331 331 331 T T T 7 140-159 197 197 197 339 339 339 T T T 8 160-179 196 196 196 338 338 338 F F F 9 180-199 196 196 196 338 338 338 F F F 10 200-219 188 188 188 326 326 326 F F F 11 220-239 193 193 193 332 332 332 F F F 12 240-259 193 193 193 332 332 332 F F F 13 260-279 195 195 195 335 335 335 T T T 14 280-299 195 195 195 335 335 335 F F F 15 300-319 190 190 190 328 328 328 F F F 16 320-339 195 195 195 335 335 335 T T T 17 340-359 195 195 195 335 335 335 F F F 18 360-379 191 191 191 330 330 330 F F F 19 380-399 197 197 197 339 339 339 F F F 20 400-419 196 196 196 338 338 338 F F F 21 420-439 197 197 197 339 339 339 F F F 22 440-459 197 197 197 339 339 339 T T T 23 460-479 196 196 196 338 338 338 T T T 24 480-499 197 197 197 339 339 339 F F F 25 500-519 190 190 190 327 327 327 T T T 26 520-539 195 195 195 335 335 335 F F F 27 540-559 197 197 197 339 339 339 T T T 28 560-579 197 197 197 339 339 339 F F F 29 580-599 197 197 197 339 339 339 F F F Max gen: 197, Max flush: 339 A Difference in extent metadata is expected here On loop 99 Verifying data now No change after verify Loop: 99 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 197 197 197 339 339 339 F F F 1 020-039 198 198 198 340 340 340 F F F 2 040-059 198 198 198 340 340 340 F F F 3 060-079 198 198 198 340 340 340 F F F 4 080-099 198 198 198 340 340 340 F F F 5 100-119 195 195 195 335 335 335 F F F 6 120-139 198 198 198 340 340 340 F F F 7 140-159 198 198 198 340 340 340 F F F 8 160-179 196 196 196 338 338 338 F F F 9 180-199 196 196 196 338 338 338 F F F 10 200-219 188 188 188 326 326 326 F F F 11 220-239 193 193 193 332 332 332 F F F 12 240-259 193 193 193 332 332 332 F F F 13 260-279 198 198 198 340 340 340 F F F 14 280-299 195 195 195 335 335 335 F F F 15 300-319 190 190 190 328 328 328 F F F 16 320-339 198 198 198 340 340 340 F F F 17 340-359 195 195 195 335 335 335 F F F 18 360-379 191 191 191 330 330 330 F F F 19 380-399 197 197 197 339 339 339 F F F 20 400-419 196 196 196 338 338 338 F F F 21 420-439 197 197 197 339 339 339 F F F 22 440-459 198 198 198 340 340 340 F F F 23 460-479 198 198 198 340 340 340 F F F 24 480-499 197 197 197 339 339 339 F F F 25 500-519 198 198 198 340 340 340 F F F 26 520-539 195 195 195 335 335 335 F F F 27 540-559 198 198 198 340 340 340 F F F 28 560-579 197 197 197 339 339 339 F F F 29 580-599 197 197 197 339 339 339 F F F Max gen: 198, Max flush: 340 7:52 Test duration Test completed real 7:51.543265120 user 56.615587984 sys 19.542539253 trap 0.072562488 tflt 0.044127089 dflt 0.114354835 kflt 0.005484638 lock 12:18:49.577481635 slp 37:54.110637548 lat 5.743843895 stop 0.064365951