Turn off color for downstairs dump Creating region directory at: /var/tmp/test_repair Creating dsc directory at: /tmp/dsc Downstairs region 0 created at /var/tmp/test_repair/8810 in 0.39219 Downstairs region 1 created at /var/tmp/test_repair/8820 in 0.1429637 Downstairs region 2 created at /var/tmp/test_repair/8830 in 0.07656192 Created 3 regions /var/tmp/bins/crutest with -t 127.0.0.1:8810 -t 127.0.0.1:8820 -t 127.0.0.1:8830 1182 1183 1184 Dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:17.271757325Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1186,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:17.337461221Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1186,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:17.338416764Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1186} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"3818f0fc76fe787c6cc412af30fd59b930203885\",\n git_commit_timestamp: \"2024-11-01T16:08:30.000000000Z\",\n git_branch: \"renovate/hyper-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:17.338566483Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1186} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:17.338587866Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1186} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:17.338602978Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1186} {"msg":"Crucible ccda5e2b-73e4-42d5-bcaf-c3f338f1438c has session id: cd8955bf-7826-4741-a12b-15bdea5d3beb","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:17.339278565Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1186,"session_id":"cd8955bf-7826-4741-a12b-15bdea5d3beb"} {"msg":"Upstairs opts: Upstairs UUID: ccda5e2b-73e4-42d5-bcaf-c3f338f1438c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:17.339323291Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1186,"session_id":"cd8955bf-7826-4741-a12b-15bdea5d3beb"} {"msg":"Crucible stats registered with UUID: ccda5e2b-73e4-42d5-bcaf-c3f338f1438c","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:17.339339453Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1186,"session_id":"cd8955bf-7826-4741-a12b-15bdea5d3beb"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:17.371247504Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1186,"task":"crutest"} {"msg":"The guest has requested activation with gen:1","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:17.3712983Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1186} {"msg":"ccda5e2b-73e4-42d5-bcaf-c3f338f1438c active request set","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:17.37137058Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1186,"session_id":"cd8955bf-7826-4741-a12b-15bdea5d3beb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:17.371397673Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1186,"client":"0","":"downstairs","session_id":"cd8955bf-7826-4741-a12b-15bdea5d3beb"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:17.371474063Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1186,"client":"0","":"downstairs","session_id":"cd8955bf-7826-4741-a12b-15bdea5d3beb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:17.371493986Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1186,"client":"1","":"downstairs","session_id":"cd8955bf-7826-4741-a12b-15bdea5d3beb"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:17.371511238Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1186,"client":"1","":"downstairs","session_id":"cd8955bf-7826-4741-a12b-15bdea5d3beb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:17.37152811Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1186,"client":"2","":"downstairs","session_id":"cd8955bf-7826-4741-a12b-15bdea5d3beb"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:17.371555233Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1186,"client":"2","":"downstairs","session_id":"cd8955bf-7826-4741-a12b-15bdea5d3beb"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:17.371581827Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1186,"":"io task","client":"2","":"downstairs","session_id":"cd8955bf-7826-4741-a12b-15bdea5d3beb"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:17.371598109Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1186,"":"io task","client":"0","":"downstairs","session_id":"cd8955bf-7826-4741-a12b-15bdea5d3beb"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:17.371611821Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1186,"":"io task","client":"1","":"downstairs","session_id":"cd8955bf-7826-4741-a12b-15bdea5d3beb"} {"msg":"ds_connection connected from Ok(127.0.0.1:58318)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:17.371728086Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1186,"":"io task","client":"0","":"downstairs","session_id":"cd8955bf-7826-4741-a12b-15bdea5d3beb"} {"msg":"ds_connection connected from Ok(127.0.0.1:45622)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:17.371797505Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1186,"":"io task","client":"2","":"downstairs","session_id":"cd8955bf-7826-4741-a12b-15bdea5d3beb"} {"msg":"ds_connection connected from Ok(127.0.0.1:39603)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:17.37183528Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1186,"":"io task","client":"1","":"downstairs","session_id":"cd8955bf-7826-4741-a12b-15bdea5d3beb"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:17.373443937Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1186,"client":"1","":"downstairs","session_id":"cd8955bf-7826-4741-a12b-15bdea5d3beb"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:17.373477531Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1186,"client":"1","":"downstairs","session_id":"cd8955bf-7826-4741-a12b-15bdea5d3beb"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:17.373495563Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1186,"client":"2","":"downstairs","session_id":"cd8955bf-7826-4741-a12b-15bdea5d3beb"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:17.373512586Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1186,"client":"2","":"downstairs","session_id":"cd8955bf-7826-4741-a12b-15bdea5d3beb"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:17.373534969Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1186,"client":"0","":"downstairs","session_id":"cd8955bf-7826-4741-a12b-15bdea5d3beb"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:17.373565772Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1186,"client":"0","":"downstairs","session_id":"cd8955bf-7826-4741-a12b-15bdea5d3beb"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:17.374355484Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1186,"client":"1","":"downstairs","session_id":"cd8955bf-7826-4741-a12b-15bdea5d3beb"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:17.374488231Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1186,"client":"0","":"downstairs","session_id":"cd8955bf-7826-4741-a12b-15bdea5d3beb"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:17.37463342Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1186,"client":"2","":"downstairs","session_id":"cd8955bf-7826-4741-a12b-15bdea5d3beb"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:17.375393818Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1186,"client":"2","":"downstairs","session_id":"cd8955bf-7826-4741-a12b-15bdea5d3beb"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:17.375421292Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1186,"client":"2","":"downstairs","session_id":"cd8955bf-7826-4741-a12b-15bdea5d3beb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:17.375435653Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1186,"session_id":"cd8955bf-7826-4741-a12b-15bdea5d3beb"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:17.375448025Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1186,"client":"0","":"downstairs","session_id":"cd8955bf-7826-4741-a12b-15bdea5d3beb"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:17.375461357Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1186,"client":"0","":"downstairs","session_id":"cd8955bf-7826-4741-a12b-15bdea5d3beb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:17.375475039Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1186,"session_id":"cd8955bf-7826-4741-a12b-15bdea5d3beb"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:17.37548738Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1186,"client":"1","":"downstairs","session_id":"cd8955bf-7826-4741-a12b-15bdea5d3beb"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:17.375500962Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1186,"client":"1","":"downstairs","session_id":"cd8955bf-7826-4741-a12b-15bdea5d3beb"} {"msg":"[0]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:17.375514854Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1186,"":"downstairs","":"downstairs","session_id":"cd8955bf-7826-4741-a12b-15bdea5d3beb"} {"msg":"[0]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:17.375528636Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1186,"":"downstairs","":"downstairs","session_id":"cd8955bf-7826-4741-a12b-15bdea5d3beb"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:17.375546268Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1186,"":"downstairs","":"downstairs","session_id":"cd8955bf-7826-4741-a12b-15bdea5d3beb"} {"msg":"[1]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:17.375606435Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1186,"":"downstairs","":"downstairs","session_id":"cd8955bf-7826-4741-a12b-15bdea5d3beb"} {"msg":"[1]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:17.375625548Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1186,"":"downstairs","":"downstairs","session_id":"cd8955bf-7826-4741-a12b-15bdea5d3beb"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:17.37563969Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1186,"":"downstairs","":"downstairs","session_id":"cd8955bf-7826-4741-a12b-15bdea5d3beb"} {"msg":"[2]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:17.375653292Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1186,"":"downstairs","":"downstairs","session_id":"cd8955bf-7826-4741-a12b-15bdea5d3beb"} {"msg":"[2]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:17.375666633Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1186,"":"downstairs","":"downstairs","session_id":"cd8955bf-7826-4741-a12b-15bdea5d3beb"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:17.375680595Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1186,"":"downstairs","":"downstairs","session_id":"cd8955bf-7826-4741-a12b-15bdea5d3beb"} {"msg":"Max found gen is 1","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:17.375693487Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1186,"":"downstairs","":"downstairs","session_id":"cd8955bf-7826-4741-a12b-15bdea5d3beb"} {"msg":"Generation requested: 1 >= found:1","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:17.375706559Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1186,"":"downstairs","":"downstairs","session_id":"cd8955bf-7826-4741-a12b-15bdea5d3beb"} {"msg":"Next flush: 1","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:17.37571966Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1186,"":"downstairs","":"downstairs","session_id":"cd8955bf-7826-4741-a12b-15bdea5d3beb"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:17.37579745Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1186,"":"downstairs","":"downstairs","session_id":"cd8955bf-7826-4741-a12b-15bdea5d3beb"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:17.375815882Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1186,"session_id":"cd8955bf-7826-4741-a12b-15bdea5d3beb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:17.375828214Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1186,"client":"0","":"downstairs","session_id":"cd8955bf-7826-4741-a12b-15bdea5d3beb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:17.375841486Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1186,"client":"1","":"downstairs","session_id":"cd8955bf-7826-4741-a12b-15bdea5d3beb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:17.375854607Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1186,"client":"2","":"downstairs","session_id":"cd8955bf-7826-4741-a12b-15bdea5d3beb"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:17.37587054Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1186,"session_id":"cd8955bf-7826-4741-a12b-15bdea5d3beb"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO 2024-11-01T16:50:17.375883001Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1186,"session_id":"cd8955bf-7826-4741-a12b-15bdea5d3beb"} {"msg":"ccda5e2b-73e4-42d5-bcaf-c3f338f1438c is now active with session: cd8955bf-7826-4741-a12b-15bdea5d3beb","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:17.375900063Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1186,"session_id":"cd8955bf-7826-4741-a12b-15bdea5d3beb"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:17.375912185Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1186,"session_id":"cd8955bf-7826-4741-a12b-15bdea5d3beb"} {"msg":"The guest has finished waiting for activation with:1","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:17.375924256Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1186} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Fill test Read and Verify all blocks (0..600 range:false) Wrote out file "/tmp/test_repair_verify.data" CLIENT: Tests done. All submitted work has been ACK'd ---------------------------------------------------------------- Crucible gen:1 GIO:true work queues: Upstairs:0 downstairs:3 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1004 Acked Read 88 Done Done Done false 1005 Acked Read 256 Done Done Done false 1006 Acked Read 256 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1003 1003 1003 Downstairs last five completed: 1003 1002 1001 1000 Upstairs last five completed: 1006 1005 1004 1003 1002 CLIENT: Up:0 ds:3 act:3 ---------------------------------------------------------------- Crucible gen:1 GIO:true work queues: Upstairs:0 downstairs:0 Downstairs last five completed: 1007 1006 1005 1004 1003 Upstairs last five completed: 1007 1006 1005 1004 1003 CLIENT: Up:0 ds:0 act:3 CLIENT: All crucible jobs finished, exiting program {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:50:21.520197405Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1186} Begin loop 1 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:21.551960477Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1188,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:50:21.595993981Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1188,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:21.596393122Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1188,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:21.628803908Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1188,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:21.629313904Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1188} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"3818f0fc76fe787c6cc412af30fd59b930203885\",\n git_commit_timestamp: \"2024-11-01T16:08:30.000000000Z\",\n git_branch: \"renovate/hyper-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:21.629344088Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1188} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:21.62936112Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1188} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:21.629375872Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1188} {"msg":"Crucible aec8dc2f-a281-47b2-8c5d-c254cc8e7411 has session id: a8f93ea7-af94-4068-8de6-522c9f92e7f1","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:21.629892249Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1188,"session_id":"a8f93ea7-af94-4068-8de6-522c9f92e7f1"} {"msg":"Upstairs opts: Upstairs UUID: aec8dc2f-a281-47b2-8c5d-c254cc8e7411, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:21.629932814Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1188,"session_id":"a8f93ea7-af94-4068-8de6-522c9f92e7f1"} {"msg":"Crucible stats registered with UUID: aec8dc2f-a281-47b2-8c5d-c254cc8e7411","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:21.629956637Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1188,"session_id":"a8f93ea7-af94-4068-8de6-522c9f92e7f1"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:21.660999477Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1188,"task":"crutest"} {"msg":"The guest has requested activation with gen:2","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:21.661029981Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1188} {"msg":"aec8dc2f-a281-47b2-8c5d-c254cc8e7411 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:21.661077867Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1188,"session_id":"a8f93ea7-af94-4068-8de6-522c9f92e7f1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:21.661098489Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1188,"client":"0","":"downstairs","session_id":"a8f93ea7-af94-4068-8de6-522c9f92e7f1"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:21.661126703Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1188,"client":"0","":"downstairs","session_id":"a8f93ea7-af94-4068-8de6-522c9f92e7f1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:21.661144365Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1188,"client":"1","":"downstairs","session_id":"a8f93ea7-af94-4068-8de6-522c9f92e7f1"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:21.661161428Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1188,"client":"1","":"downstairs","session_id":"a8f93ea7-af94-4068-8de6-522c9f92e7f1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:21.661175029Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1188,"client":"2","":"downstairs","session_id":"a8f93ea7-af94-4068-8de6-522c9f92e7f1"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:21.661192141Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1188,"client":"2","":"downstairs","session_id":"a8f93ea7-af94-4068-8de6-522c9f92e7f1"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:21.661205853Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1188,"":"io task","client":"0","":"downstairs","session_id":"a8f93ea7-af94-4068-8de6-522c9f92e7f1"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:21.661219675Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1188,"":"io task","client":"2","":"downstairs","session_id":"a8f93ea7-af94-4068-8de6-522c9f92e7f1"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:21.661233507Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1188,"":"io task","client":"1","":"downstairs","session_id":"a8f93ea7-af94-4068-8de6-522c9f92e7f1"} {"msg":"ds_connection connected from Ok(127.0.0.1:57853)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:21.661303346Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1188,"":"io task","client":"0","":"downstairs","session_id":"a8f93ea7-af94-4068-8de6-522c9f92e7f1"} {"msg":"ds_connection connected from Ok(127.0.0.1:50616)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:21.661359993Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1188,"":"io task","client":"1","":"downstairs","session_id":"a8f93ea7-af94-4068-8de6-522c9f92e7f1"} {"msg":"ds_connection connected from Ok(127.0.0.1:61974)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:21.661394287Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1188,"":"io task","client":"2","":"downstairs","session_id":"a8f93ea7-af94-4068-8de6-522c9f92e7f1"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:21.662059523Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1188,"client":"2","":"downstairs","session_id":"a8f93ea7-af94-4068-8de6-522c9f92e7f1"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:21.662087227Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1188,"client":"2","":"downstairs","session_id":"a8f93ea7-af94-4068-8de6-522c9f92e7f1"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:21.662101529Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1188,"client":"1","":"downstairs","session_id":"a8f93ea7-af94-4068-8de6-522c9f92e7f1"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:21.662115191Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1188,"client":"1","":"downstairs","session_id":"a8f93ea7-af94-4068-8de6-522c9f92e7f1"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:21.662230775Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1188,"client":"0","":"downstairs","session_id":"a8f93ea7-af94-4068-8de6-522c9f92e7f1"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:21.662252998Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1188,"client":"0","":"downstairs","session_id":"a8f93ea7-af94-4068-8de6-522c9f92e7f1"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:21.662804949Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1188,"client":"1","":"downstairs","session_id":"a8f93ea7-af94-4068-8de6-522c9f92e7f1"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:21.662835773Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1188,"client":"0","":"downstairs","session_id":"a8f93ea7-af94-4068-8de6-522c9f92e7f1"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:21.662853596Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1188,"client":"2","":"downstairs","session_id":"a8f93ea7-af94-4068-8de6-522c9f92e7f1"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:21.663427469Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1188,"client":"1","":"downstairs","session_id":"a8f93ea7-af94-4068-8de6-522c9f92e7f1"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:21.663467495Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1188,"client":"1","":"downstairs","session_id":"a8f93ea7-af94-4068-8de6-522c9f92e7f1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:21.663497979Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1188,"session_id":"a8f93ea7-af94-4068-8de6-522c9f92e7f1"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:21.663522072Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1188,"client":"0","":"downstairs","session_id":"a8f93ea7-af94-4068-8de6-522c9f92e7f1"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:21.663545855Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1188,"client":"0","":"downstairs","session_id":"a8f93ea7-af94-4068-8de6-522c9f92e7f1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:21.663568958Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1188,"session_id":"a8f93ea7-af94-4068-8de6-522c9f92e7f1"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:21.663591441Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1188,"client":"2","":"downstairs","session_id":"a8f93ea7-af94-4068-8de6-522c9f92e7f1"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:21.663614824Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1188,"client":"2","":"downstairs","session_id":"a8f93ea7-af94-4068-8de6-522c9f92e7f1"} {"msg":"[0]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:21.663638197Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1188,"":"downstairs","":"downstairs","session_id":"a8f93ea7-af94-4068-8de6-522c9f92e7f1"} {"msg":"[0]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:21.66366133Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1188,"":"downstairs","":"downstairs","session_id":"a8f93ea7-af94-4068-8de6-522c9f92e7f1"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:21.663686433Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1188,"":"downstairs","":"downstairs","session_id":"a8f93ea7-af94-4068-8de6-522c9f92e7f1"} {"msg":"[1]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:21.663712606Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1188,"":"downstairs","":"downstairs","session_id":"a8f93ea7-af94-4068-8de6-522c9f92e7f1"} {"msg":"[1]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:21.66373869Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1188,"":"downstairs","":"downstairs","session_id":"a8f93ea7-af94-4068-8de6-522c9f92e7f1"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:21.663765393Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1188,"":"downstairs","":"downstairs","session_id":"a8f93ea7-af94-4068-8de6-522c9f92e7f1"} {"msg":"[2]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]"Wait for a query_work_queue command to finish before sending IO ,"v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:21.663805168Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1188,"":"downstairs","":"downstairs","session_id":"a8f93ea7-af94-4068-8de6-522c9f92e7f1"} {"msg":"[2]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:21.663836272Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1188,"":"downstairs","":"downstairs","session_id":"a8f93ea7-af94-4068-8de6-522c9f92e7f1"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:21.663866726Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1188,"":"downstairs","":"downstairs","session_id":"a8f93ea7-af94-4068-8de6-522c9f92e7f1"} {"msg":"Max found gen is 2","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:21.66389473Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1188,"":"downstairs","":"downstairs","session_id":"a8f93ea7-af94-4068-8de6-522c9f92e7f1"} {"msg":"Generation requested: 2 >= found:2","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:21.663923113Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1188,"":"downstairs","":"downstairs","session_id":"a8f93ea7-af94-4068-8de6-522c9f92e7f1"} {"msg":"Next flush: 2","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:21.663950437Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1188,"":"downstairs","":"downstairs","session_id":"a8f93ea7-af94-4068-8de6-522c9f92e7f1"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:21.663978981Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1188,"":"downstairs","":"downstairs","session_id":"a8f93ea7-af94-4068-8de6-522c9f92e7f1"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:21.664006804Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1188,"session_id":"a8f93ea7-af94-4068-8de6-522c9f92e7f1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 30,"time":"2024-11-01T16:50:21.664034768Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1188,"client":"0","":"downstairs","session_id":"a8f93ea7-af94-4068-8de6-522c9f92e7f1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:21.664064252Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1188,"client":"1","":"downstairs","session_id":"a8f93ea7-af94-4068-8de6-522c9f92e7f1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:21.664081094Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1188,"client":"2","":"downstairs","session_id":"a8f93ea7-af94-4068-8de6-522c9f92e7f1"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:21.664094735Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1188,"session_id":"a8f93ea7-af94-4068-8de6-522c9f92e7f1"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:21.664107367Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1188,"session_id":"a8f93ea7-af94-4068-8de6-522c9f92e7f1"} {"msg":"aec8dc2f-a281-47b2-8c5d-c254cc8e7411 is now active with session: a8f93ea7-af94-4068-8de6-522c9f92e7f1","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:21.664120189Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1188,"session_id":"a8f93ea7-af94-4068-8de6-522c9f92e7f1"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:21.66413235Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1188,"session_id":"a8f93ea7-af94-4068-8de6-522c9f92e7f1"} {"msg":"The guest has finished waiting for activation with:2","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:21.664144752Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1188} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 64 len 40960 data: 2 2 2 2 2 2 2 2 2 2 02/30 Write block 369 len 4096 data: 2 03/30 Read block 551 len 36864 04/30 Write block 210 len 12288 data: 2 2 2 05/30 Write block 6 len 16384 data: 2 2 2 2 06/30 Write block 355 len 20480 data: 2 2 2 2 2 07/30 Write block 149 len 32768 data: 2 2 2 2 2 2 2 2 08/30 Read block 425 len 40960 09/30 Read block 491 len 36864 10/30 Flush 11/30 Write block 345 len 16384 data: 2 2 2 2 12/30 Write block 259 len 32768 data: 2 2 2 2 2 2 2 2 13/30 Write block 418 len 24576 data: 2 2 2 2 2 2 14/30 Read block 398 len 16384 15/30 Write block 101 len 16384 data: 2 2 2 2 16/30 Write block 447 len 28672 data: 2 2 2 2 2 2 2 17/30 Read block 474 len 20480 18/30 Read block 467 len 36864 19/30 Read block 257 len 16384 20/30 Read block 177 len 16384 21/30 Write block 318 len 24576 data: 2 2 2 2 2 2 22/30 Write block 129 len 28672 data: 2 2 2 2 2 2 2 23/30 Read block 451 len 40960 24/30 Read block 200 len 12288 25/30 Write block 90 len 40960 data: 2 2 2 2 2 2 2 2 2 2 26/30 Read block 85 len 40960 27/30 Write block 546 len 24576 data: 2 2 2 2 2 2 28/30 Write block 329 len 28672 data: 2 2 2 2 2 2 2 29/30 Write block 131 len 16384 data: 3 3 3 3 30/30 Write block 64 len 32768 data: 3 3 3 3 3 3 3 3 ---------------------------------------------------------------- Crucible gen:2 GIO:true work queues: Upstairs:0 downstairs:20 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1010 Acked Write 4 Done Done Done false 1011 Acked Write 8 Done Done Done false 1012 Acked Write 6 Done Done Done false 1013 Acked Read 4 Done Done Done false 1014 Acked Write 4 Done Done Done false 1015 Acked Write 7 Done Done Done false 1016 Acked Read 5 Done Done Done false 1017 Acked Read 9 Done Done Done false 1018 Acked Read 4 Done Done Done false 1019 Acked Read 4 Done Done Done false 1020 Acked Write 6 Done Done Done false 1021 Acked Write 7 Done Done Done false 1022 Acked Read 10 Done Done Done false 1023 Acked Read 3 Done Done Done false 1024 Acked Write 10 Done Done Done false 1025 Acked Read 10 Done Done Done false 1026 Acked Write 6 Done Done Done false 1027 Acked Write 7 Done Sent Sent false 1028 Acked Write 4 Sent Sent Sent false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 3 3 8 Done 18 17 17 52 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1009 1009 1009 Downstairs last five completed: 1009 1008 1007 1006 1005 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:50:21.68524212Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1188} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 2 2 2 2 2 2 F F F 1 020-039 1 1 1 1 1 1 F F F 2 040-059 1 1 1 1 1 1 F F F 3 060-079 2 2 2 2 2 2 T F F <--- 4 080-099 1 1 1 1 1 1 T T T 5 100-119 1 1 1 1 1 1 T T T 6 120-139 1 1 1 1 1 1 T T T 7 140-159 2 2 2 2 2 2 F F F 8 160-179 1 1 1 1 1 1 F F F 9 180-199 1 1 1 1 1 1 F F F 10 200-219 2 2 2 2 2 2 F F F 11 220-239 1 1 1 1 1 1 F F F 12 240-259 1 1 1 1 1 1 T T T 13 260-279 1 1 1 1 1 1 T T T 14 280-299 1 1 1 1 1 1 F F F 15 300-319 1 1 1 1 1 1 T T T 16 320-339 1 1 1 1 1 1 T T T 17 340-359 2 2 2 2 2 2 T T T 18 360-379 2 2 2 2 2 2 F F F 19 380-399 1 1 1 1 1 1 F F F 20 400-419 1 1 1 1 1 1 T T T 21 420-439 1 1 1 1 1 1 T T T 22 440-459 1 1 1 1 1 1 T T T 23 460-479 1 1 1 1 1 1 F F F 24 480-499 1 1 1 1 1 1 F F F 25 500-519 1 1 1 1 1 1 F F F 26 520-539 1 1 1 1 1 1 F F F 27 540-559 1 1 1 1 1 1 T T T 28 560-579 1 1 1 1 1 1 F F F 29 580-599 1 1 1 1 1 1 F F F Max gen: 2, Max flush: 2 Error: Difference in extent metadata found! A Difference in extent metadata is expected here On loop 1 Verifying data now No change after verify Loop: 1 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 2 2 2 2 2 2 F F F 1 020-039 1 1 1 1 1 1 F F F 2 040-059 1 1 1 1 1 1 F F F 3 060-079 3 3 3 3 3 3 F F F 4 080-099 3 3 3 3 3 3 F F F 5 100-119 3 3 3 3 3 3 F F F 6 120-139 3 3 3 3 3 3 F F F 7 140-159 2 2 2 2 2 2 F F F 8 160-179 1 1 1 1 1 1 F F F 9 180-199 1 1 1 1 1 1 F F F 10 200-219 2 2 2 2 2 2 F F F 11 220-239 1 1 1 1 1 1 F F F 12 240-259 3 3 3 3 3 3 F F F 13 260-279 3 3 3 3 3 3 F F F 14 280-299 1 1 1 1 1 1 F F F 15 300-319 3 3 3 3 3 3 F F F 16 320-339 3 3 3 3 3 3 F F F 17 340-359 3 3 3 3 3 3 F F F 18 360-379 2 2 2 2 2 2 F F F 19 380-399 1 1 1 1 1 1 F F F 20 400-419 3 3 3 3 3 3 F F F 21 420-439 3 3 3 3 3 3 F F F 22 440-459 3 3 3 3 3 3 F F F 23 460-479 1 1 1 1 1 1 F F F 24 480-499 1 1 1 1 1 1 F F F 25 500-519 1 1 1 1 1 1 F F F 26 520-539 1 1 1 1 1 1 F F F 27 540-559 3 3 3 3 3 3 F F F 28 560-579 1 1 1 1 1 1 F F F 29 580-599 1 1 1 1 1 1 F F F Max gen: 3, Max flush: 3 Begin loop 2 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:26.147068984Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1196,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:26.191346769Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1196,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:26.191998683Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1196} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"3818f0fc76fe787c6cc412af30fd59b930203885\",\n git_commit_timestamp: \"2024-11-01T16:08:30.000000000Z\",\n git_branch: \"renovate/hyper-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:26.19204968Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1196} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:26.192078173Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1196} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:26.192113048Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1196} {"msg":"Crucible 406a6168-fa36-44e7-8baa-ca201e5ad290 has session id: 7ecafa4e-b01f-4016-9213-580caeb2ce27","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:26.192554885Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1196,"session_id":"7ecafa4e-b01f-4016-9213-580caeb2ce27"} {"msg":"Upstairs opts: Upstairs UUID: 406a6168-fa36-44e7-8baa-ca201e5ad290, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:26.19259762Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1196,"session_id":"7ecafa4e-b01f-4016-9213-580caeb2ce27"} {"msg":"Crucible stats registered with UUID: 406a6168-fa36-44e7-8baa-ca201e5ad290","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:26.192621923Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1196,"session_id":"7ecafa4e-b01f-4016-9213-580caeb2ce27"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:26.223915015Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1196,"task":"crutest"} {"msg":"The guest has requested activation with gen:4","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:26.223957911Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1196} {"msg":"406a6168-fa36-44e7-8baa-ca201e5ad290 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:26.223993075Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1196,"session_id":"7ecafa4e-b01f-4016-9213-580caeb2ce27"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:26.224023029Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1196,"client":"0","":"downstairs","session_id":"7ecafa4e-b01f-4016-9213-580caeb2ce27"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:26.224069065Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1196,"client":"0","":"downstairs","session_id":"7ecafa4e-b01f-4016-9213-580caeb2ce27"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:26.22410749Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1196,"client":"1","":"downstairs","session_id":"7ecafa4e-b01f-4016-9213-580caeb2ce27"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:26.224144285Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1196,"client":"1","":"downstairs","session_id":"7ecafa4e-b01f-4016-9213-580caeb2ce27"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:26.224295574Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1196,"client":"2","":"downstairs","session_id":"7ecafa4e-b01f-4016-9213-580caeb2ce27"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:26.22433671Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1196,"client":"2","":"downstairs","session_id":"7ecafa4e-b01f-4016-9213-580caeb2ce27"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:26.224437723Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1196,"":"io task","client":"0","":"downstairs","session_id":"7ecafa4e-b01f-4016-9213-580caeb2ce27"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:26.224491089Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1196,"":"io task","client":"2","":"downstairs","session_id":"7ecafa4e-b01f-4016-9213-580caeb2ce27"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:26.224528324Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1196,"":"io task","client":"1","":"downstairs","session_id":"7ecafa4e-b01f-4016-9213-580caeb2ce27"} {"msg":"ds_connection connected from Ok(127.0.0.1:53699)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:26.224561289Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1196,"":"io task","client":"0","":"downstairs","session_id":"7ecafa4e-b01f-4016-9213-580caeb2ce27"} {"msg":"ds_connection connected from Ok(127.0.0.1:38180)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:26.224591142Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1196,"":"io task","client":"1","":"downstairs","session_id":"7ecafa4e-b01f-4016-9213-580caeb2ce27"} {"msg":"ds_connection connected from Ok(127.0.0.1:37466)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:26.224621676Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1196,"":"io task","client":"2","":"downstairs","session_id":"7ecafa4e-b01f-4016-9213-580caeb2ce27"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:26.225056512Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1196,"client":"1","":"downstairs","session_id":"7ecafa4e-b01f-4016-9213-580caeb2ce27"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:26.225083336Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1196,"client":"1","":"downstairs","session_id":"7ecafa4e-b01f-4016-9213-580caeb2ce27"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:26.225098138Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1196,"client":"2","":"downstairs","session_id":"7ecafa4e-b01f-4016-9213-580caeb2ce27"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:26.225112009Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1196,"client":"2","":"downstairs","session_id":"7ecafa4e-b01f-4016-9213-580caeb2ce27"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:26.225125391Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1196,"client":"0","":"downstairs","session_id":"7ecafa4e-b01f-4016-9213-580caeb2ce27"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:26.225138723Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1196,"client":"0","":"downstairs","session_id":"7ecafa4e-b01f-4016-9213-580caeb2ce27"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:26.225696755Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1196,"client":"2","":"downstairs","session_id":"7ecafa4e-b01f-4016-9213-580caeb2ce27"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:26.22573759Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1196,"client":"0","":"downstairs","session_id":"7ecafa4e-b01f-4016-9213-580caeb2ce27"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:26.225772685Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1196,"client":"1","":"downstairs","session_id":"7ecafa4e-b01f-4016-9213-580caeb2ce27"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:26.226252787Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1196,"client":"1","":"downstairs","session_id":"7ecafa4e-b01f-4016-9213-580caeb2ce27"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:26.22628122Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1196,"client":"1","":"downstairs","session_id":"7ecafa4e-b01f-4016-9213-580caeb2ce27"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:26.226296702Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1196,"session_id":"7ecafa4e-b01f-4016-9213-580caeb2ce27"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:26.22636159Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1196,"client":"2","":"downstairs","session_id":"7ecafa4e-b01f-4016-9213-580caeb2ce27"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:26.226381563Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1196,"client":"2","":"downstairs","session_id":"7ecafa4e-b01f-4016-9213-580caeb2ce27"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:26.226395785Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1196,"session_id":"7ecafa4e-b01f-4016-9213-580caeb2ce27"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:26.226408216Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1196,"client":"0","":"downstairs","session_id":"7ecafa4e-b01f-4016-9213-580caeb2ce27"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:26.226421608Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1196,"client":"0","":"downstairs","session_id":"7ecafa4e-b01f-4016-9213-580caeb2ce27"} {"msg":"[0]R flush_numbers[0..12]: [2, 1, 1, 3, 3, 3, 3, 2, 1, 1, 2, 1]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:26.22643522Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1196,"":"downstairs","":"downstairs","session_id":"7ecafa4e-b01f-4016-9213-580caeb2ce27"} {"msg":"[0]R generation[0..12]: [2, 1, 1, 3, 3, 3, 3, 2, 1, 1, 2, 1]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:26.226448842Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1196,"":"downstairs","":"downstairs","session_id":"7ecafa4e-b01f-4016-9213-580caeb2ce27"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:26.226463264Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1196,"":"downstairs","":"downstairs","session_id":"7ecafa4e-b01f-4016-9213-580caeb2ce27"} {"msg":"[1]R flush_numbers[0..12]: [2, 1, 1, 3, 3, 3, 3, 2, 1, 1, 2, 1]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:26.226477185Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1196,"":"downstairs","":"downstairs","session_id":"7ecafa4e-b01f-4016-9213-580caeb2ce27"} {"msg":"[1]R generation[0..12]: [2, 1, 1, 3, 3, 3, 3, 2, 1, 1, 2, 1]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:26.226490667Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1196,"":"downstairs","":"downstairs","session_id":"7ecafa4e-b01f-4016-9213-580caeb2ce27"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:26.226504649Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1196,"":"downstairs","":"downstairs","session_id":"7ecafa4e-b01f-4016-9213-580caeb2ce27"} {"msg":"[2]R flush_numbers[0..12]: [2, 1, 1, 3, 3, 3, 3, 2, 1, 1, 2, 1]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:26.226518261Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1196,"":"downstairs","":"downstairs","session_id":"7ecafa4e-b01f-4016-9213-580caeb2ce27"} {"msg":"[2]R generation[0..12]: [2, 1, 1, 3, 3, 3, 3, 2, 1, 1, 2, 1]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:26.226534793Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1196,"":"downstairs","":"downstairs","session_id":"7ecafa4e-b01f-4016-9213-580caeb2ce27"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:26.226549145Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1196,"":"downstairs","":"downstairs","session_id":"7ecafa4e-b01f-4016-9213-580caeb2ce27"} {"msg":"Max found gen is 4","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:26.226562206Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1196,"":"downstairs","":"downstairs","session_id":"7ecafa4e-b01f-4016-9213-580caeb2ce27"} {"msg":"Generation requested: 4 >= found:4","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:26.226575358Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1196,"":"downstairs","":"downstairs","session_id":"7ecafa4e-b01f-4016-9213-580caeb2ce27"} {"msg":"Next flush: 4","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:26.22658845Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1196,"":"downstairs","":"downstairs","session_id":"7ecafa4e-b01f-4016-9213-580caeb2ce27"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:26.226601601Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1196,"":"downstairs","":"downstairs","session_id":"7ecafa4e-b01f-4016-9213-580caeb2ce27"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:26.226615123Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1196,"session_id":"7ecafa4e-b01f-4016-9213-580caeb2ce27"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:26.226627065Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1196,"client":"0","":"downstairs","session_id":"7ecafa4e-b01f-4016-9213-580caeb2ce27"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO 2024-11-01T16:50:26.226640457Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1196,"client":"1","":"downstairs","session_id":"7ecafa4e-b01f-4016-9213-580caeb2ce27"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:26.226664729Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1196,"client":"2","":"downstairs","session_id":"7ecafa4e-b01f-4016-9213-580caeb2ce27"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:26.226678251Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1196,"session_id":"7ecafa4e-b01f-4016-9213-580caeb2ce27"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:26.226690383Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1196,"session_id":"7ecafa4e-b01f-4016-9213-580caeb2ce27"} {"msg":"406a6168-fa36-44e7-8baa-ca201e5ad290 is now active with session: 7ecafa4e-b01f-4016-9213-580caeb2ce27","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:26.226703114Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1196,"session_id":"7ecafa4e-b01f-4016-9213-580caeb2ce27"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:26.226715216Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1196,"session_id":"7ecafa4e-b01f-4016-9213-580caeb2ce27"} {"msg":"The guest has finished waiting for activation with:4","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:26.226727338Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1196} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 234 len 20480 data: 2 2 2 2 2 02/30 Read block 538 len 32768 03/30 Write block 196 len 12288 data: 2 2 2 04/30 Write block 396 len 36864 data: 2 2 2 2 2 2 2 2 2 05/30 Read block 311 len 16384 06/30 Read block 297 len 32768 07/30 Flush 08/30 Write block 579 len 36864 data: 2 2 2 2 2 2 2 2 2 09/30 Flush 10/30 Write block 204 len 40960 data: 2 2 2 2 2 2 3 3 3 2 11/30 Write block 154 len 20480 data: 3 3 3 2 2 12/30 Read block 501 len 24576 13/30 Read block 380 len 40960 14/30 Read block 364 len 36864 15/30 Write block 135 len 32768 data: 3 2 2 2 2 2 2 2 16/30 Write block 154 len 40960 data: 4 4 4 3 3 2 2 2 2 2 17/30 Write block 568 len 8192 data: 2 2 18/30 Read block 346 len 40960 19/30 Write block 57 len 40960 data: 2 2 2 2 2 2 2 4 4 4 20/30 Read block 553 len 8192 21/30 Read block 93 len 24576 22/30 Read block 264 len 32768 23/30 Write block 440 len 32768 data: 2 2 2 2 2 2 2 3 24/30 Write block 488 len 32768 data: 2 2 2 2 2 2 2 2 25/30 Write block 308 len 36864 data: 2 2 2 2 2 2 2 2 2 26/30 Write block 397 len 12288 data: 3 3 3 27/30 Read block 320 len 28672 28/30 Write block 120 len 20480 data: 2 2 2 2 2 29/30 Read block 123 len 12288 30/30 Read block 182 len 40960 ---------------------------------------------------------------- Crucible gen:4 GIO:true work queues: Upstairs:0 downstairs:21 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1009 Acked Write 10 Done Done Done false 1010 Acked Write 5 Done Done Done false 1011 Acked Read 6 Done Done Done false 1012 Acked Read 10 Done Done Done false 1013 Acked Read 9 Done Done Done false 1014 Acked Write 8 Done Done Done false 1015 Acked Write 10 Done Done Done false 1016 Acked Write 2 Done Done Done false 1017 Acked Read 10 Done Done Done false 1018 Acked Write 10 Done Done Done false 1019 Acked Read 2 Done Done Done false 1020 Acked Read 6 Done Done Done false 1021 Acked Read 8 Done Done Done false 1022 Acked Write 8 Done Done Done false 1023 Acked Write 8 Done Done Done false 1024 Acked Write 9 Done Done Done false 1025 Acked Write 3 Done Done Done false 1026 Acked Read 7 Done Done Done false 1027 Acked Write 5 Done Done Done false 1028 Acked Read 3 Done Done Done false 1029 Acked Read 10 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 21 20 20 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1008 1008 1008 Downstairs last five completed: 1008 1007 1006 1005 1004 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:50:26.252023527Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1196} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 2 2 2 2 2 2 F F F 1 020-039 1 1 1 1 1 1 F F F 2 040-059 1 1 1 1 1 1 T T T 3 060-079 3 3 3 3 3 3 T T T 4 080-099 3 3 3 3 3 3 F F F 5 100-119 3 3 3 3 3 3 F F F 6 120-139 3 3 3 3 3 3 T T T 7 140-159 2 2 2 2 2 2 T T T 8 160-179 1 1 1 1 1 1 T T T 9 180-199 4 4 4 4 4 4 F F F 10 200-219 2 2 2 2 2 2 T T T 11 220-239 4 4 4 4 4 4 F F F 12 240-259 3 3 3 3 3 3 F F F 13 260-279 3 3 3 3 3 3 F F F 14 280-299 1 1 1 1 1 1 F F F 15 300-319 3 3 3 3 3 3 T T T 16 320-339 3 3 3 3 3 3 F F F 17 340-359 3 3 3 3 3 3 F F F 18 360-379 2 2 2 2 2 2 F F F 19 380-399 4 4 4 4 4 4 T T T 20 400-419 4 4 4 4 4 4 F F F 21 420-439 3 3 3 3 3 3 F F F 22 440-459 3 3 3 3 3 3 T T T 23 460-479 1 1 1 1 1 1 F F F 24 480-499 1 1 1 1 1 1 T T T 25 500-519 1 1 1 1 1 1 F F F 26 520-539 1 1 1 1 1 1 F F F 27 540-559 3 3 3 3 3 3 F F F 28 560-579 4 4 4 5 5 5 T T T 29 580-599 4 4 4 5 5 5 F F F Max gen: 4, Max flush: 5 A Difference in extent metadata is expected here On loop 2 Verifying data now No change after verify Loop: 2 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 2 2 2 2 2 2 F F F 1 020-039 1 1 1 1 1 1 F F F 2 040-059 5 5 5 6 6 6 F F F 3 060-079 5 5 5 6 6 6 F F F 4 080-099 3 3 3 3 3 3 F F F 5 100-119 3 3 3 3 3 3 F F F 6 120-139 5 5 5 6 6 6 F F F 7 140-159 5 5 5 6 6 6 F F F 8 160-179 5 5 5 6 6 6 F F F 9 180-199 4 4 4 4 4 4 F F F 10 200-219 5 5 5 6 6 6 F F F 11 220-239 4 4 4 4 4 4 F F F 12 240-259 3 3 3 3 3 3 F F F 13 260-279 3 3 3 3 3 3 F F F 14 280-299 1 1 1 1 1 1 F F F 15 300-319 5 5 5 6 6 6 F F F 16 320-339 3 3 3 3 3 3 F F F 17 340-359 3 3 3 3 3 3 F F F 18 360-379 2 2 2 2 2 2 F F F 19 380-399 5 5 5 6 6 6 F F F 20 400-419 4 4 4 4 4 4 F F F 21 420-439 3 3 3 3 3 3 F F F 22 440-459 5 5 5 6 6 6 F F F 23 460-479 1 1 1 1 1 1 F F F 24 480-499 5 5 5 6 6 6 F F F 25 500-519 1 1 1 1 1 1 F F F 26 520-539 1 1 1 1 1 1 F F F 27 540-559 3 3 3 3 3 3 F F F 28 560-579 5 5 5 6 6 6 F F F 29 580-599 4 4 4 5 5 5 F F F Max gen: 5, Max flush: 6 Begin loop 3 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:30.622926906Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1204,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:50:30.666168948Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1204,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:30.666522464Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1204,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:30.698800192Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1204,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:30.69932639Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1204} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"3818f0fc76fe787c6cc412af30fd59b930203885\",\n git_commit_timestamp: \"2024-11-01T16:08:30.000000000Z\",\n git_branch: \"renovate/hyper-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:30.699363345Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1204} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:30.699381937Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1204} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:30.699396999Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1204} {"msg":"Crucible a877bd89-5c46-468e-8523-ae9df6c81df5 has session id: 516367c7-6dc3-4482-bae0-110705b088d8","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:30.699850348Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1204,"session_id":"516367c7-6dc3-4482-bae0-110705b088d8"} {"msg":"Upstairs opts: Upstairs UUID: a877bd89-5c46-468e-8523-ae9df6c81df5, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:30.699890623Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1204,"session_id":"516367c7-6dc3-4482-bae0-110705b088d8"} {"msg":"Crucible stats registered with UUID: a877bd89-5c46-468e-8523-ae9df6c81df5","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:30.699913956Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1204,"session_id":"516367c7-6dc3-4482-bae0-110705b088d8"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:30.731236742Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1204,"task":"crutest"} {"msg":"The guest has requested activation with gen:6","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:30.731269766Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1204} {"msg":"a877bd89-5c46-468e-8523-ae9df6c81df5 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:30.73130454Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1204,"session_id":"516367c7-6dc3-4482-bae0-110705b088d8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:30.731322183Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1204,"client":"0","":"downstairs","session_id":"516367c7-6dc3-4482-bae0-110705b088d8"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:30.731350786Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1204,"client":"0","":"downstairs","session_id":"516367c7-6dc3-4482-bae0-110705b088d8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:30.731368289Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1204,"client":"1","":"downstairs","session_id":"516367c7-6dc3-4482-bae0-110705b088d8"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:30.731385501Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1204,"client":"1","":"downstairs","session_id":"516367c7-6dc3-4482-bae0-110705b088d8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:30.731402453Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1204,"client":"2","":"downstairs","session_id":"516367c7-6dc3-4482-bae0-110705b088d8"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:30.731430817Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1204,"client":"2","":"downstairs","session_id":"516367c7-6dc3-4482-bae0-110705b088d8"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:30.731467481Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1204,"":"io task","client":"2","":"downstairs","session_id":"516367c7-6dc3-4482-bae0-110705b088d8"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:30.731498636Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1204,"":"io task","client":"0","":"downstairs","session_id":"516367c7-6dc3-4482-bae0-110705b088d8"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:30.731519028Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1204,"":"io task","client":"1","":"downstairs","session_id":"516367c7-6dc3-4482-bae0-110705b088d8"} {"msg":"ds_connection connected from Ok(127.0.0.1:39047)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:30.73153667Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1204,"":"io task","client":"0","":"downstairs","session_id":"516367c7-6dc3-4482-bae0-110705b088d8"} {"msg":"ds_connection connected from Ok(127.0.0.1:64634)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:30.731565524Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1204,"":"io task","client":"2","":"downstairs","session_id":"516367c7-6dc3-4482-bae0-110705b088d8"} {"msg":"ds_connection connected from Ok(127.0.0.1:56715)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:30.731599709Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1204,"":"io task","client":"1","":"downstairs","session_id":"516367c7-6dc3-4482-bae0-110705b088d8"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:30.73223287Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1204,"client":"1","":"downstairs","session_id":"516367c7-6dc3-4482-bae0-110705b088d8"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:30.732265464Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1204,"client":"1","":"downstairs","session_id":"516367c7-6dc3-4482-bae0-110705b088d8"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:30.732283767Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1204,"client":"2","":"downstairs","session_id":"516367c7-6dc3-4482-bae0-110705b088d8"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:30.732300619Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1204,"client":"2","":"downstairs","session_id":"516367c7-6dc3-4482-bae0-110705b088d8"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:30.732317481Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1204,"client":"0","":"downstairs","session_id":"516367c7-6dc3-4482-bae0-110705b088d8"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:30.732338624Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1204,"client":"0","":"downstairs","session_id":"516367c7-6dc3-4482-bae0-110705b088d8"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:30.732922819Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1204,"client":"1","":"downstairs","session_id":"516367c7-6dc3-4482-bae0-110705b088d8"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:30.732952203Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1204,"client":"2","":"downstairs","session_id":"516367c7-6dc3-4482-bae0-110705b088d8"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:30.732968185Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1204,"client":"0","":"downstairs","session_id":"516367c7-6dc3-4482-bae0-110705b088d8"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:30.733510905Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1204,"client":"1","":"downstairs","session_id":"516367c7-6dc3-4482-bae0-110705b088d8"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:30.733538408Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1204,"client":"1","":"downstairs","session_id":"516367c7-6dc3-4482-bae0-110705b088d8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:30.73355282Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1204,"session_id":"516367c7-6dc3-4482-bae0-110705b088d8"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:30.733566442Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1204,"client":"0","":"downstairs","session_id":"516367c7-6dc3-4482-bae0-110705b088d8"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:30.733580044Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1204,"client":"0","":"downstairs","session_id":"516367c7-6dc3-4482-bae0-110705b088d8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:30.733593275Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1204,"session_id":"516367c7-6dc3-4482-bae0-110705b088d8"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:30.733605407Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1204,"client":"2","":"downstairs","session_id":"516367c7-6dc3-4482-bae0-110705b088d8"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:30.733618649Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1204,"client":"2","":"downstairs","session_id":"516367c7-6dc3-4482-bae0-110705b088d8"} {"msg":"[0]R flush_numbers[0..12]: [2, 1, 6, 6, 3, 3, 6, 6, 6, 4, 6, 4]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:30.733632331Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1204,"":"downstairs","":"downstairs","session_id":"516367c7-6dc3-4482-bae0-110705b088d8"} {"msg":"[0]R generation[0..12]: [2, 1, 5, 5, 3, 3, 5, 5, 5, 4, 5, 4]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:30.733646062Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1204,"":"downstairs","":"downstairs","session_id":"516367c7-6dc3-4482-bae0-110705b088d8"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:30.733660294Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1204,"":"downstairs","":"downstairs","session_id":"516367c7-6dc3-4482-bae0-110705b088d8"} {"msg":"[1]R flush_numbers[0..12]: [2, 1, 6, 6, 3, 3, 6, 6, 6, 4, 6, 4]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:30.733673976Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1204,"":"downstairs","":"downstairs","session_id":"516367c7-6dc3-4482-bae0-110705b088d8"} {"msg":"[1]R generation[0..12]: [2, 1, 5, 5, 3, 3, 5, 5, 5, 4, 5, 4]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:30.733687338Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1204,"":"downstairs","":"downstairs","session_id":"516367c7-6dc3-4482-bae0-110705b088d8"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:30.733701499Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1204,"":"downstairs","":"downstairs","session_id":"516367c7-6dc3-4482-bae0-110705b088d8"} {"msg":"[2]R flush_numbers[0..12]: [2, 1, 6, 6, 3, 3, 6, 6, 6, 4, 6, 4]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:30.733715181Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1204,"":"downstairs","":"downstairs","session_id":"516367c7-6dc3-4482-bae0-110705b088d8"} {"msg":"[2]R generation[0..12]: [2, 1, 5, 5, 3, 3, 5, 5, 5, 4, 5, 4]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:30.733731383Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1204,"":"downstairs","":"downstairs","session_id":"516367c7-6dc3-4482-bae0-110705b088d8"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:30.733745795Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1204,"":"downstairs","":"downstairs","session_id":"516367c7-6dc3-4482-bae0-110705b088d8"} {"msg":"Max found gen is 6","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:30.733758927Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1204,"":"downstairs","":"downstairs","session_id":"516367c7-6dc3-4482-bae0-110705b088d8"} {"msg":"Generation requested: 6 >= found:6","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-01T16:50:30.733772009Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1204,"":"downstairs","":"downstairs","session_id":"516367c7-6dc3-4482-bae0-110705b088d8"} {"msg":"Next flush: 7","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:30.733796012Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1204,"":"downstairs","":"downstairs","session_id":"516367c7-6dc3-4482-bae0-110705b088d8"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:30.733809153Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1204,"":"downstairs","":"downstairs","session_id":"516367c7-6dc3-4482-bae0-110705b088d8"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:30.733822445Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1204,"session_id":"516367c7-6dc3-4482-bae0-110705b088d8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:30.733834466Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1204,"client":"0","":"downstairs","session_id":"516367c7-6dc3-4482-bae0-110705b088d8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:30.733847658Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1204,"client":"1","":"downstairs","session_id":"516367c7-6dc3-4482-bae0-110705b088d8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:30.73386086Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1204,"client":"2","":"downstairs","session_id":"516367c7-6dc3-4482-bae0-110705b088d8"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:30.733874162Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1204,"session_id":"516367c7-6dc3-4482-bae0-110705b088d8"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:30.733886543Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1204,"session_id":"516367c7-6dc3-4482-bae0-110705b088d8"} {"msg":"a877bd89-5c46-468e-8523-ae9df6c81df5 is now active with session: 516367c7-6dc3-4482-bae0-110705b088d8","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:30.733899455Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1204,"session_id":"516367c7-6dc3-4482-bae0-110705b088d8"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:30.733911516Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1204,"session_id":"516367c7-6dc3-4482-bae0-110705b088d8"} {"msg":"The guest has finished waiting for activation with:6","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:30.733923688Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1204} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 70 len 20480 data: 4 4 3 3 2 02/30 Write block 130 len 24576 data: 3 4 4 4 4 4 03/30 Read block 166 len 20480 04/30 Write block 81 len 40960 data: 2 2 2 2 2 2 2 2 2 3 05/30 Write block 221 len 20480 data: 2 2 2 2 2 06/30 Read block 522 len 16384 07/30 Flush 08/30 Write block 551 len 12288 data: 3 2 2 09/30 Read block 587 len 16384 10/30 Read block 60 len 16384 11/30 Read block 314 len 20480 12/30 Write block 344 len 4096 data: 2 13/30 Read block 114 len 12288 14/30 Read block 244 len 16384 15/30 Read block 173 len 32768 16/30 Read block 20 len 8192 17/30 Write block 297 len 4096 data: 2 18/30 Write block 247 len 20480 data: 2 2 2 2 2 19/30 Flush 20/30 Write block 508 len 20480 data: 2 2 2 2 2 21/30 Flush 22/30 Write block 489 len 16384 data: 3 3 3 3 23/30 Write block 229 len 20480 data: 2 2 2 2 2 24/30 Write block 236 len 8192 data: 3 3 25/30 Write block 190 len 24576 data: 2 2 2 2 2 2 26/30 Flush 27/30 Write block 32 len 4096 data: 2 28/30 Write block 530 len 16384 data: 2 2 2 2 29/30 Write block 418 len 36864 data: 3 3 3 3 3 3 2 2 2 30/30 Read block 375 len 8192 ---------------------------------------------------------------- Crucible gen:6 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 1 Done Done Done false 1027 Acked Write 4 Done Done Done false 1028 Acked Write 9 Done Done Done false 1029 Acked Read 2 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:50:30.761417811Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1204} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 2 2 2 2 2 2 F F F 1 020-039 1 1 1 1 1 1 T T T 2 040-059 5 5 5 6 6 6 F F F 3 060-079 6 6 6 7 7 7 F F F 4 080-099 6 6 6 7 7 7 F F F 5 100-119 3 3 3 3 3 3 F F F 6 120-139 6 6 6 7 7 7 F F F 7 140-159 5 5 5 6 6 6 F F F 8 160-179 5 5 5 6 6 6 F F F 9 180-199 6 6 6 10 10 10 F F F 10 200-219 5 5 5 6 6 6 F F F 11 220-239 6 6 6 10 10 10 F F F 12 240-259 6 6 6 8 8 8 F F F 13 260-279 3 3 3 3 3 3 F F F 14 280-299 6 6 6 8 8 8 F F F 15 300-319 5 5 5 6 6 6 F F F 16 320-339 3 3 3 3 3 3 F F F 17 340-359 6 6 6 8 8 8 F F F 18 360-379 2 2 2 2 2 2 F F F 19 380-399 5 5 5 6 6 6 F F F 20 400-419 4 4 4 4 4 4 T T T 21 420-439 3 3 3 3 3 3 T T T 22 440-459 5 5 5 6 6 6 F F F 23 460-479 1 1 1 1 1 1 F F F 24 480-499 6 6 6 10 10 10 F F F 25 500-519 6 6 6 9 9 9 F F F 26 520-539 1 1 1 1 1 1 T T T 27 540-559 6 6 6 8 8 8 F F F 28 560-579 5 5 5 6 6 6 F F F 29 580-599 4 4 4 5 5 5 F F F Max gen: 6, Max flush: 10 A Difference in extent metadata is expected here On loop 3 Verifying data now No change after verify Loop: 3 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 2 2 2 2 2 2 F F F 1 020-039 7 7 7 11 11 11 F F F 2 040-059 5 5 5 6 6 6 F F F 3 060-079 6 6 6 7 7 7 F F F 4 080-099 6 6 6 7 7 7 F F F 5 100-119 3 3 3 3 3 3 F F F 6 120-139 6 6 6 7 7 7 F F F 7 140-159 5 5 5 6 6 6 F F F 8 160-179 5 5 5 6 6 6 F F F 9 180-199 6 6 6 10 10 10 F F F 10 200-219 5 5 5 6 6 6 F F F 11 220-239 6 6 6 10 10 10 F F F 12 240-259 6 6 6 8 8 8 F F F 13 260-279 3 3 3 3 3 3 F F F 14 280-299 6 6 6 8 8 8 F F F 15 300-319 5 5 5 6 6 6 F F F 16 320-339 3 3 3 3 3 3 F F F 17 340-359 6 6 6 8 8 8 F F F 18 360-379 2 2 2 2 2 2 F F F 19 380-399 5 5 5 6 6 6 F F F 20 400-419 7 7 7 11 11 11 F F F 21 420-439 7 7 7 11 11 11 F F F 22 440-459 5 5 5 6 6 6 F F F 23 460-479 1 1 1 1 1 1 F F F 24 480-499 6 6 6 10 10 10 F F F 25 500-519 6 6 6 9 9 9 F F F 26 520-539 7 7 7 11 11 11 F F F 27 540-559 6 6 6 8 8 8 F F F 28 560-579 5 5 5 6 6 6 F F F 29 580-599 4 4 4 5 5 5 F F F Max gen: 7, Max flush: 11 Begin loop 4 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:35.086264316Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1212,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:35.130984158Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1212,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:35.131831137Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1212} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"3818f0fc76fe787c6cc412af30fd59b930203885\",\n git_commit_timestamp: \"2024-11-01T16:08:30.000000000Z\",\n git_branch: \"renovate/hyper-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:35.131880983Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1212} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:35.131899765Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1212} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:35.131917598Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1212} {"msg":"Crucible dec464d3-cc5c-4002-ae87-926aaa8a78ec has session id: 73f215b8-9d9b-4445-a762-efc3b4d412ac","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:35.132339562Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1212,"session_id":"73f215b8-9d9b-4445-a762-efc3b4d412ac"} {"msg":"Upstairs opts: Upstairs UUID: dec464d3-cc5c-4002-ae87-926aaa8a78ec, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:35.132380117Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1212,"session_id":"73f215b8-9d9b-4445-a762-efc3b4d412ac"} {"msg":"Crucible stats registered with UUID: dec464d3-cc5c-4002-ae87-926aaa8a78ec","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:35.132407971Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1212,"session_id":"73f215b8-9d9b-4445-a762-efc3b4d412ac"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:35.163185927Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1212,"task":"crutest"} {"msg":"The guest has requested activation with gen:8","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:35.163230832Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1212} {"msg":"dec464d3-cc5c-4002-ae87-926aaa8a78ec active request set","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:35.163301671Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1212,"session_id":"73f215b8-9d9b-4445-a762-efc3b4d412ac"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:35.163335356Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1212,"client":"0","":"downstairs","session_id":"73f215b8-9d9b-4445-a762-efc3b4d412ac"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:35.16336694Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1212,"client":"0","":"downstairs","session_id":"73f215b8-9d9b-4445-a762-efc3b4d412ac"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:35.163385852Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1212,"client":"1","":"downstairs","session_id":"73f215b8-9d9b-4445-a762-efc3b4d412ac"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:35.163410776Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1212,"client":"1","":"downstairs","session_id":"73f215b8-9d9b-4445-a762-efc3b4d412ac"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:35.163440749Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1212,"client":"2","":"downstairs","session_id":"73f215b8-9d9b-4445-a762-efc3b4d412ac"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:35.163470673Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1212,"client":"2","":"downstairs","session_id":"73f215b8-9d9b-4445-a762-efc3b4d412ac"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:35.163501407Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1212,"":"io task","client":"2","":"downstairs","session_id":"73f215b8-9d9b-4445-a762-efc3b4d412ac"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:35.163649096Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1212,"":"io task","client":"0","":"downstairs","session_id":"73f215b8-9d9b-4445-a762-efc3b4d412ac"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:35.163711034Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1212,"":"io task","client":"1","":"downstairs","session_id":"73f215b8-9d9b-4445-a762-efc3b4d412ac"} {"msg":"ds_connection connected from Ok(127.0.0.1:37778)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:35.163749609Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1212,"":"io task","client":"2","":"downstairs","session_id":"73f215b8-9d9b-4445-a762-efc3b4d412ac"} {"msg":"ds_connection connected from Ok(127.0.0.1:61607)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:35.163787424Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1212,"":"io task","client":"0","":"downstairs","session_id":"73f215b8-9d9b-4445-a762-efc3b4d412ac"} {"msg":"ds_connection connected from Ok(127.0.0.1:44177)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:35.163823739Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1212,"":"io task","client":"1","":"downstairs","session_id":"73f215b8-9d9b-4445-a762-efc3b4d412ac"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:35.164352147Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1212,"client":"2","":"downstairs","session_id":"73f215b8-9d9b-4445-a762-efc3b4d412ac"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:35.16438053Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1212,"client":"2","":"downstairs","session_id":"73f215b8-9d9b-4445-a762-efc3b4d412ac"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:35.164396112Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1212,"client":"1","":"downstairs","session_id":"73f215b8-9d9b-4445-a762-efc3b4d412ac"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:35.164410374Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1212,"client":"1","":"downstairs","session_id":"73f215b8-9d9b-4445-a762-efc3b4d412ac"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:35.164423996Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1212,"client":"0","":"downstairs","session_id":"73f215b8-9d9b-4445-a762-efc3b4d412ac"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:35.164461811Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1212,"client":"0","":"downstairs","session_id":"73f215b8-9d9b-4445-a762-efc3b4d412ac"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:35.165055077Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1212,"client":"0","":"downstairs","session_id":"73f215b8-9d9b-4445-a762-efc3b4d412ac"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:35.165084191Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1212,"client":"2","":"downstairs","session_id":"73f215b8-9d9b-4445-a762-efc3b4d412ac"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:35.165110785Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1212,"client":"1","":"downstairs","session_id":"73f215b8-9d9b-4445-a762-efc3b4d412ac"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:35.165670437Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1212,"client":"1","":"downstairs","session_id":"73f215b8-9d9b-4445-a762-efc3b4d412ac"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:35.165709472Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1212,"client":"1","":"downstairs","session_id":"73f215b8-9d9b-4445-a762-efc3b4d412ac"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:35.165735085Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1212,"session_id":"73f215b8-9d9b-4445-a762-efc3b4d412ac"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:35.165756848Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1212,"client":"2","":"downstairs","session_id":"73f215b8-9d9b-4445-a762-efc3b4d412ac"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:35.165780151Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1212,"client":"2","":"downstairs","session_id":"73f215b8-9d9b-4445-a762-efc3b4d412ac"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:35.165807854Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1212,"session_id":"73f215b8-9d9b-4445-a762-efc3b4d412ac"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:35.165829787Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1212,"client":"0","":"downstairs","session_id":"73f215b8-9d9b-4445-a762-efc3b4d412ac"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:35.16585186Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1212,"client":"0","":"downstairs","session_id":"73f215b8-9d9b-4445-a762-efc3b4d412ac"} {"msg":"[0]R flush_numbers[0..12]: [2, 11, 6, 7, 7, 3, 7, 6, 6, 10, 6, 10]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:35.165875143Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1212,"":"downstairs","":"downstairs","session_id":"73f215b8-9d9b-4445-a762-efc3b4d412ac"} {"msg":"[0]R generation[0..12]: [2, 7, 5, 6, 6, 3, 6, 5, 5, 6, 5, 6]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:35.165900126Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1212,"":"downstairs","":"downstairs","session_id":"73f215b8-9d9b-4445-a762-efc3b4d412ac"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:35.16592699Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1212,"":"downstairs","":"downstairs","session_id":"73f215b8-9d9b-4445-a762-efc3b4d412ac"} {"msg":"[1]R flush_numbers[0..12]: [2, 11, 6, 7, 7, 3, 7, 6, 6, 10, 6, 10]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:35.165952473Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1212,"":"downstairs","":"downstairs","session_id":"73f215b8-9d9b-4445-a762-efc3b4d412ac"} {"msg":"[1]R generation[0..12]: [2, 7, 5, 6, 6, 3, 6, 5, 5, 6, 5, 6]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:35.165979247Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1212,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO 73f215b8-9d9b-4445-a762-efc3b4d412ac"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:35.166019622Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1212,"":"downstairs","":"downstairs","session_id":"73f215b8-9d9b-4445-a762-efc3b4d412ac"} {"msg":"[2]R flush_numbers[0..12]: [2, 11, 6, 7, 7, 3, 7, 6, 6, 10, 6, 10]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:35.166050496Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1212,"":"downstairs","":"downstairs","session_id":"73f215b8-9d9b-4445-a762-efc3b4d412ac"} {"msg":"[2]R generation[0..12]: [2, 7, 5, 6, 6, 3, 6, 5, 5, 6, 5, 6]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:35.166080169Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1212,"":"downstairs","":"downstairs","session_id":"73f215b8-9d9b-4445-a762-efc3b4d412ac"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:35.166109453Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1212,"":"downstairs","":"downstairs","session_id":"73f215b8-9d9b-4445-a762-efc3b4d412ac"} {"msg":"Max found gen is 8","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:35.166138187Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1212,"":"downstairs","":"downstairs","session_id":"73f215b8-9d9b-4445-a762-efc3b4d412ac"} {"msg":"Generation requested: 8 >= found:8","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:35.166165711Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1212,"":"downstairs","":"downstairs","session_id":"73f215b8-9d9b-4445-a762-efc3b4d412ac"} {"msg":"Next flush: 12"Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:35.166199695Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1212,"":"downstairs","":"downstairs","session_id":"73f215b8-9d9b-4445-a762-efc3b4d412ac"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:35.166223118Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1212,"":"downstairs","":"downstairs","session_id":"73f215b8-9d9b-4445-a762-efc3b4d412ac"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:35.16623734Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1212,"session_id":"73f215b8-9d9b-4445-a762-efc3b4d412ac"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:35.166249581Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1212,"client":"0","":"downstairs","session_id":"73f215b8-9d9b-4445-a762-efc3b4d412ac"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:35.166263023Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1212,"client":"1","":"downstairs","session_id":"73f215b8-9d9b-4445-a762-efc3b4d412ac"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:35.166276265Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1212,"client":"2","":"downstairs","session_id":"73f215b8-9d9b-4445-a762-efc3b4d412ac"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:35.166289656Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1212,"session_id":"73f215b8-9d9b-4445-a762-efc3b4d412ac"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:35.166301998Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1212,"session_id":"73f215b8-9d9b-4445-a762-efc3b4d412ac"} {"msg":"dec464d3-cc5c-4002-ae87-926aaa8a78ec is now active with session: 73f215b8-9d9b-4445-a762-efc3b4d412ac","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:35.16631467Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1212,"session_id":"73f215b8-9d9b-4445-a762-efc3b4d412ac"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:35.166326741Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1212,"session_id":"73f215b8-9d9b-4445-a762-efc3b4d412ac"} {"msg":"The guest has finished waiting for activation with:8","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:35.166339133Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1212} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 271 len 4096 data: 2 02/30 Read block 405 len 36864 03/30 Read block 220 len 24576 04/30 Write block 360 len 16384 data: 2 2 2 2 05/30 Read block 567 len 12288 06/30 Read block 108 len 24576 07/30 Write block 393 len 32768 data: 2 2 2 3 4 4 4 3 08/30 Read block 137 len 16384 09/30 Write block 199 len 40960 data: 2 2 2 2 2 3 3 3 3 3 10/30 Write block 274 len 12288 data: 2 2 2 11/30 Read block 6 len 12288 12/30 Read block 232 len 16384 13/30 Read block 261 len 40960 14/30 Write block 373 len 40960 data: 2 2 2 2 2 2 2 2 2 2 15/30 Write block 585 len 20480 data: 3 3 3 2 2 16/30 Write block 11 len 4096 data: 2 17/30 Write block 188 len 4096 data: 2 18/30 Write block 247 len 28672 data: 3 3 3 3 3 2 2 19/30 Write block 470 len 36864 data: 2 2 2 2 2 2 2 2 2 20/30 Read block 390 len 40960 21/30 Write block 478 len 12288 data: 3 2 2 22/30 Read block 304 len 12288 23/30 Read block 28 len 8192 24/30 Read block 9 len 36864 25/30 Flush 26/30 Write block 469 len 16384 data: 2 3 3 3 27/30 Read block 544 len 12288 28/30 Read block 122 len 4096 29/30 Write block 61 len 40960 data: 3 3 3 5 5 5 4 4 4 5 30/30 Write block 168 len 4096 data: 2 ---------------------------------------------------------------- Crucible gen:8 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 4 Done Done Done false 1026 Acked Read 3 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Write 10 Sent Sent Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:50:35.187698275Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1212} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 8 8 8 12 12 12 F F F 1 020-039 7 7 7 11 11 11 F F F 2 040-059 5 5 5 6 6 6 F F F 3 060-079 6 6 6 7 7 7 T T T 4 080-099 6 6 6 7 7 7 F F F 5 100-119 3 3 3 3 3 3 F F F 6 120-139 6 6 6 7 7 7 F F F 7 140-159 5 5 5 6 6 6 F F F 8 160-179 5 5 5 6 6 6 T T T 9 180-199 8 8 8 12 12 12 F F F 10 200-219 8 8 8 12 12 12 F F F 11 220-239 6 6 6 10 10 10 F F F 12 240-259 8 8 8 12 12 12 F F F 13 260-279 8 8 8 12 12 12 F F F 14 280-299 6 6 6 8 8 8 F F F 15 300-319 5 5 5 6 6 6 F F F 16 320-339 3 3 3 3 3 3 F F F 17 340-359 6 6 6 8 8 8 F F F 18 360-379 8 8 8 12 12 12 F F F 19 380-399 8 8 8 12 12 12 F F F 20 400-419 8 8 8 12 12 12 F F F 21 420-439 7 7 7 11 11 11 F F F 22 440-459 5 5 5 6 6 6 F F F 23 460-479 8 8 8 12 12 12 T T T 24 480-499 8 8 8 12 12 12 F F F 25 500-519 6 6 6 9 9 9 F F F 26 520-539 7 7 7 11 11 11 F F F 27 540-559 6 6 6 8 8 8 F F F 28 560-579 5 5 5 6 6 6 F F F 29 580-599 8 8 8 12 12 12 F F F Max gen: 8, Max flush: 12 A Difference in extent metadata is expected here On loop 4 Verifying data now No change after verify Loop: 4 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 8 8 8 12 12 12 F F F 1 020-039 7 7 7 11 11 11 F F F 2 040-059 5 5 5 6 6 6 F F F 3 060-079 9 9 9 13 13 13 F F F 4 080-099 6 6 6 7 7 7 F F F 5 100-119 3 3 3 3 3 3 F F F 6 120-139 6 6 6 7 7 7 F F F 7 140-159 5 5 5 6 6 6 F F F 8 160-179 9 9 9 13 13 13 F F F 9 180-199 8 8 8 12 12 12 F F F 10 200-219 8 8 8 12 12 12 F F F 11 220-239 6 6 6 10 10 10 F F F 12 240-259 8 8 8 12 12 12 F F F 13 260-279 8 8 8 12 12 12 F F F 14 280-299 6 6 6 8 8 8 F F F 15 300-319 5 5 5 6 6 6 F F F 16 320-339 3 3 3 3 3 3 F F F 17 340-359 6 6 6 8 8 8 F F F 18 360-379 8 8 8 12 12 12 F F F 19 380-399 8 8 8 12 12 12 F F F 20 400-419 8 8 8 12 12 12 F F F 21 420-439 7 7 7 11 11 11 F F F 22 440-459 5 5 5 6 6 6 F F F 23 460-479 9 9 9 13 13 13 F F F 24 480-499 8 8 8 12 12 12 F F F 25 500-519 6 6 6 9 9 9 F F F 26 520-539 7 7 7 11 11 11 F F F 27 540-559 6 6 6 8 8 8 F F F 28 560-579 5 5 5 6 6 6 F F F 29 580-599 8 8 8 12 12 12 F F F Max gen: 9, Max flush: 13 Begin loop 5 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:39.45961357Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1220,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:39.50362437Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1220,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:39.504226438Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1220} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"3818f0fc76fe787c6cc412af30fd59b930203885\",\n git_commit_timestamp: \"2024-11-01T16:08:30.000000000Z\",\n git_branch: \"renovate/hyper-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:39.504276514Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1220} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:39.504304238Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1220} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:39.504330541Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1220} {"msg":"Crucible b0fce403-dbaf-4ea0-90d5-c9199743773b has session id: 2b8ec891-3105-41f4-b65a-ec1e0e3068f9","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:39.504731753Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1220,"session_id":"2b8ec891-3105-41f4-b65a-ec1e0e3068f9"} {"msg":"Upstairs opts: Upstairs UUID: b0fce403-dbaf-4ea0-90d5-c9199743773b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:39.504773518Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1220,"session_id":"2b8ec891-3105-41f4-b65a-ec1e0e3068f9"} {"msg":"Crucible stats registered with UUID: b0fce403-dbaf-4ea0-90d5-c9199743773b","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:39.504794771Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1220,"session_id":"2b8ec891-3105-41f4-b65a-ec1e0e3068f9"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:39.535836581Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1220,"task":"crutest"} {"msg":"The guest has requested activation with gen:10","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:39.535869615Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1220} {"msg":"b0fce403-dbaf-4ea0-90d5-c9199743773b active request set","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:39.535928953Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1220,"session_id":"2b8ec891-3105-41f4-b65a-ec1e0e3068f9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:39.535960417Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1220,"client":"0","":"downstairs","session_id":"2b8ec891-3105-41f4-b65a-ec1e0e3068f9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:39.53598668Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1220,"client":"0","":"downstairs","session_id":"2b8ec891-3105-41f4-b65a-ec1e0e3068f9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:39.536002522Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1220,"client":"1","":"downstairs","session_id":"2b8ec891-3105-41f4-b65a-ec1e0e3068f9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:39.536016264Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1220,"client":"1","":"downstairs","session_id":"2b8ec891-3105-41f4-b65a-ec1e0e3068f9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:39.536029686Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1220,"client":"2","":"downstairs","session_id":"2b8ec891-3105-41f4-b65a-ec1e0e3068f9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:39.536043038Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1220,"client":"2","":"downstairs","session_id":"2b8ec891-3105-41f4-b65a-ec1e0e3068f9"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:39.536056189Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1220,"":"io task","client":"2","":"downstairs","session_id":"2b8ec891-3105-41f4-b65a-ec1e0e3068f9"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:39.536070051Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1220,"":"io task","client":"1","":"downstairs","session_id":"2b8ec891-3105-41f4-b65a-ec1e0e3068f9"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:39.536083823Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1220,"":"io task","client":"0","":"downstairs","session_id":"2b8ec891-3105-41f4-b65a-ec1e0e3068f9"} {"msg":"ds_connection connected from Ok(127.0.0.1:44389)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:39.536203768Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1220,"":"io task","client":"2","":"downstairs","session_id":"2b8ec891-3105-41f4-b65a-ec1e0e3068f9"} {"msg":"ds_connection connected from Ok(127.0.0.1:37068)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:39.536242203Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1220,"":"io task","client":"1","":"downstairs","session_id":"2b8ec891-3105-41f4-b65a-ec1e0e3068f9"} {"msg":"ds_connection connected from Ok(127.0.0.1:41233)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:39.536423776Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1220,"":"io task","client":"0","":"downstairs","session_id":"2b8ec891-3105-41f4-b65a-ec1e0e3068f9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:39.537033745Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1220,"client":"2","":"downstairs","session_id":"2b8ec891-3105-41f4-b65a-ec1e0e3068f9"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:39.537067299Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1220,"client":"2","":"downstairs","session_id":"2b8ec891-3105-41f4-b65a-ec1e0e3068f9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:39.537096523Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1220,"client":"1","":"downstairs","session_id":"2b8ec891-3105-41f4-b65a-ec1e0e3068f9"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:39.537123957Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1220,"client":"1","":"downstairs","session_id":"2b8ec891-3105-41f4-b65a-ec1e0e3068f9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:39.537156471Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1220,"client":"0","":"downstairs","session_id":"2b8ec891-3105-41f4-b65a-ec1e0e3068f9"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:39.537192275Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1220,"client":"0","":"downstairs","session_id":"2b8ec891-3105-41f4-b65a-ec1e0e3068f9"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:39.537763119Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1220,"client":"1","":"downstairs","session_id":"2b8ec891-3105-41f4-b65a-ec1e0e3068f9"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:39.537803744Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1220,"client":"0","":"downstairs","session_id":"2b8ec891-3105-41f4-b65a-ec1e0e3068f9"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:39.537834458Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1220,"client":"2","":"downstairs","session_id":"2b8ec891-3105-41f4-b65a-ec1e0e3068f9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:39.538335883Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1220,"client":"1","":"downstairs","session_id":"2b8ec891-3105-41f4-b65a-ec1e0e3068f9"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:39.538364146Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1220,"client":"1","":"downstairs","session_id":"2b8ec891-3105-41f4-b65a-ec1e0e3068f9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:39.538379628Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1220,"session_id":"2b8ec891-3105-41f4-b65a-ec1e0e3068f9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:39.538450718Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1220,"client":"0","":"downstairs","session_id":"2b8ec891-3105-41f4-b65a-ec1e0e3068f9"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:39.538472991Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1220,"client":"0","":"downstairs","session_id":"2b8ec891-3105-41f4-b65a-ec1e0e3068f9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:39.538486882Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1220,"session_id":"2b8ec891-3105-41f4-b65a-ec1e0e3068f9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:39.538499334Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1220,"client":"2","":"downstairs","session_id":"2b8ec891-3105-41f4-b65a-ec1e0e3068f9"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:39.538512746Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1220,"client":"2","":"downstairs","session_id":"2b8ec891-3105-41f4-b65a-ec1e0e3068f9"} {"msg":"[0]R flush_numbers[0..12]: [12, 11, 6, 13, 7, 3, 7, 6, 13, 12, 12, 10]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:39.538526607Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1220,"":"downstairs","":"downstairs","session_id":"2b8ec891-3105-41f4-b65a-ec1e0e3068f9"} {"msg":"[0]R generation[0..12]: [8, 7, 5, 9, 6, 3, 6, 5, 9, 8, 8, 6]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:39.538540209Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1220,"":"downstairs","":"downstairs","session_id":"2b8ec891-3105-41f4-b65a-ec1e0e3068f9"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:39.538557922Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1220,"":"downstairs","":"downstairs","session_id":"2b8ec891-3105-41f4-b65a-ec1e0e3068f9"} {"msg":"[1]R flush_numbers[0..12]: [12, 11, 6, 13, 7, 3, 7, 6, 13, 12, 12, 10]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:39.538571823Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1220,"":"downstairs","":"downstairs","session_id":"2b8ec891-3105-41f4-b65a-ec1e0e3068f9"} {"msg":"[1]R generation[0..12]: [8, 7, 5, 9, 6, 3, 6, 5, 9, 8, 8, 6]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:39.538585145Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1220,"":"downstairs","":"downstairs","session_id":"2b8ec891-3105-41f4-b65a-ec1e0e3068f9"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:39.538599227Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1220,"":"downstairs","":"downstairs","session_id":"2b8ec891-3105-41f4-b65a-ec1e0e3068f9"} {"msg":"[2]R flush_numbers[0..12]: [12, 11, 6, 13, 7, 3, 7, 6, 13, 12, 12, 10]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:39.538612969Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1220,"":"downstairs","":"downstairs","session_id":"2b8ec891-3105-41f4-b65a-ec1e0e3068f9"} {"msg":"[2]R generation[0..12]: [8, 7, 5, 9, 6, 3, 6, 5, 9, 8, 8, 6]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:39.53862642Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1220,"":"downstairs","":"downstairs","session_id":"2b8ec891-3105-41f4-b65a-ec1e0e3068f9"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:39.538640962Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1220,"":"downstairs","":"downstairs","session_id":"2b8ec891-3105-41f4-b65a-ec1e0e3068f9"} {"msg":"Max found gen is 10","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:39.538664985Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1220,"":"Wait for a query_work_queue command to finish before sending IO downstairs","":"downstairs","session_id":"2b8ec891-3105-41f4-b65a-ec1e0e3068f9"} {"msg":"Generation requested: 10 >= found:10","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:39.538698169Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1220,"":"downstairs","":"downstairs","session_id":"2b8ec891-3105-41f4-b65a-ec1e0e3068f9"} {"msg":"Next flush: 14","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:39.538718762Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1220,"":"downstairs","":"downstairs","session_id":"2b8ec891-3105-41f4-b65a-ec1e0e3068f9"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:39.538732564Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1220,"":"downstairs","":"downstairs","session_id":"2b8ec891-3105-41f4-b65a-ec1e0e3068f9"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:39.538745886Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1220,"session_id":"2b8ec891-3105-41f4-b65a-ec1e0e3068f9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:39.538757777Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1220,"client":"0","":"downstairs","session_id":"2b8ec891-3105-41f4-b65a-ec1e0e3068f9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:39.538770949Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1220,"client":"1","":"downstairs","session_id":"2b8ec891-3105-41f4-b65a-ec1e0e3068f9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:39.538784181Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1220,"client":"2","":"downstairs","session_id":"2b8ec891-3105-41f4-b65a-ec1e0e3068f9"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:39.538799883Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1220,"session_id":"2b8ec891-3105-41f4-b65a-ec1e0e3068f9"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:39.538812384Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1220,"session_id":"2b8ec891-3105-41f4-b65a-ec1e0e3068f9"} {"msg":"b0fce403-dbaf-4ea0-90d5-c9199743773b is now active with session: 2b8ec891-3105-41f4-b65a-ec1e0e3068f9","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:39.538825186Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1220,"session_id":"2b8ec891-3105-41f4-b65a-ec1e0e3068f9"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:39.538837187Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1220,"session_id":"2b8ec891-3105-41f4-b65a-ec1e0e3068f9"} {"msg":"The guest has finished waiting for activation with:10","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:39.538849429Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1220} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 396 len 32768 data: 4 5 5 5 4 3 3 3 02/30 Write block 177 len 24576 data: 2 2 2 2 2 2 03/30 Read block 93 len 12288 04/30 Write block 250 len 16384 data: 4 4 3 3 05/30 Read block 111 len 4096 06/30 Read block 381 len 36864 07/30 Read block 285 len 36864 08/30 Read block 528 len 24576 09/30 Read block 228 len 4096 10/30 Write block 150 len 36864 data: 3 3 3 3 5 5 5 4 4 11/30 Read block 524 len 4096 12/30 Read block 83 len 36864 13/30 Read block 210 len 32768 14/30 Write block 144 len 28672 data: 2 2 2 2 2 3 4 15/30 Read block 67 len 16384 16/30 Write block 223 len 28672 data: 3 3 3 2 2 2 3 17/30 Read block 567 len 24576 18/30 Read block 282 len 24576 19/30 Read block 36 len 40960 20/30 Read block 363 len 24576 21/30 Write block 66 len 8192 data: 6 5 22/30 Read block 359 len 28672 23/30 Read block 475 len 12288 24/30 Read block 15 len 28672 25/30 Flush 26/30 Flush 27/30 Write block 303 len 24576 data: 2 2 2 2 2 3 28/30 Write block 324 len 16384 data: 2 2 2 2 29/30 Read block 221 len 20480 30/30 Read block 474 len 12288 ---------------------------------------------------------------- Crucible gen:10 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 6 Done Done Done false 1027 Acked Write 4 Done Done Done false 1028 Acked Read 5 Done Done Done false 1029 Acked Read 3 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 4 3 3 10 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:50:39.56097863Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1220} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 8 8 8 12 12 12 F F F 1 020-039 7 7 7 11 11 11 F F F 2 040-059 5 5 5 6 6 6 F F F 3 060-079 10 10 10 14 14 14 F F F 4 080-099 6 6 6 7 7 7 F F F 5 100-119 3 3 3 3 3 3 F F F 6 120-139 6 6 6 7 7 7 F F F 7 140-159 10 10 10 14 14 14 F F F 8 160-179 10 10 10 14 14 14 F F F 9 180-199 10 10 10 14 14 14 F F F 10 200-219 8 8 8 12 12 12 F F F 11 220-239 10 10 10 14 14 14 F F F 12 240-259 10 10 10 14 14 14 F F F 13 260-279 8 8 8 12 12 12 F F F 14 280-299 6 6 6 8 8 8 F F F 15 300-319 5 5 5 6 6 6 T T T 16 320-339 3 3 3 3 3 3 T T T 17 340-359 6 6 6 8 8 8 F F F 18 360-379 8 8 8 12 12 12 F F F 19 380-399 10 10 10 14 14 14 F F F 20 400-419 10 10 10 14 14 14 F F F 21 420-439 7 7 7 11 11 11 F F F 22 440-459 5 5 5 6 6 6 F F F 23 460-479 9 9 9 13 13 13 F F F 24 480-499 8 8 8 12 12 12 F F F 25 500-519 6 6 6 9 9 9 F F F 26 520-539 7 7 7 11 11 11 F F F 27 540-559 6 6 6 8 8 8 F F F 28 560-579 5 5 5 6 6 6 F F F 29 580-599 8 8 8 12 12 12 F F F Max gen: 10, Max flush: 14 A Difference in extent metadata is expected here On loop 5 Verifying data now No change after verify Loop: 5 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 8 8 8 12 12 12 F F F 1 020-039 7 7 7 11 11 11 F F F 2 040-059 5 5 5 6 6 6 F F F 3 060-079 10 10 10 14 14 14 F F F 4 080-099 6 6 6 7 7 7 F F F 5 100-119 3 3 3 3 3 3 F F F 6 120-139 6 6 6 7 7 7 F F F 7 140-159 10 10 10 14 14 14 F F F 8 160-179 10 10 10 14 14 14 F F F 9 180-199 10 10 10 14 14 14 F F F 10 200-219 8 8 8 12 12 12 F F F 11 220-239 10 10 10 14 14 14 F F F 12 240-259 10 10 10 14 14 14 F F F 13 260-279 8 8 8 12 12 12 F F F 14 280-299 6 6 6 8 8 8 F F F 15 300-319 11 11 11 15 15 15 F F F 16 320-339 11 11 11 15 15 15 F F F 17 340-359 6 6 6 8 8 8 F F F 18 360-379 8 8 8 12 12 12 F F F 19 380-399 10 10 10 14 14 14 F F F 20 400-419 10 10 10 14 14 14 F F F 21 420-439 7 7 7 11 11 11 F F F 22 440-459 5 5 5 6 6 6 F F F 23 460-479 9 9 9 13 13 13 F F F 24 480-499 8 8 8 12 12 12 F F F 25 500-519 6 6 6 9 9 9 F F F 26 520-539 7 7 7 11 11 11 F F F 27 540-559 6 6 6 8 8 8 F F F 28 560-579 5 5 5 6 6 6 F F F 29 580-599 8 8 8 12 12 12 F F F Max gen: 11, Max flush: 15 Begin loop 6 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:43.825866569Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1228,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:50:43.868650631Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1228,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:43.869043012Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1228,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:43.901067268Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1228,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:43.901575134Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1228} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"3818f0fc76fe787c6cc412af30fd59b930203885\",\n git_commit_timestamp: \"2024-11-01T16:08:30.000000000Z\",\n git_branch: \"renovate/hyper-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:43.901611929Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1228} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:43.901628241Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1228} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:43.901658024Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1228} {"msg":"Crucible b74eb8eb-b711-4a4b-89d3-e9a289557999 has session id: d379e042-7089-46f3-a869-0cb01cec787a","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:43.902108553Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1228,"session_id":"d379e042-7089-46f3-a869-0cb01cec787a"} {"msg":"Upstairs opts: Upstairs UUID: b74eb8eb-b711-4a4b-89d3-e9a289557999, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:43.902144377Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1228,"session_id":"d379e042-7089-46f3-a869-0cb01cec787a"} {"msg":"Crucible stats registered with UUID: b74eb8eb-b711-4a4b-89d3-e9a289557999","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:43.902162769Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1228,"session_id":"d379e042-7089-46f3-a869-0cb01cec787a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:43.93344029Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1228,"task":"crutest"} {"msg":"The guest has requested activation with gen:12","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:43.933474804Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1228} {"msg":"b74eb8eb-b711-4a4b-89d3-e9a289557999 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:43.933509348Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1228,"session_id":"d379e042-7089-46f3-a869-0cb01cec787a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:43.933526441Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1228,"client":"0","":"downstairs","session_id":"d379e042-7089-46f3-a869-0cb01cec787a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:43.933556305Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1228,"client":"0","":"downstairs","session_id":"d379e042-7089-46f3-a869-0cb01cec787a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:43.933574827Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1228,"client":"1","":"downstairs","session_id":"d379e042-7089-46f3-a869-0cb01cec787a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:43.933592609Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1228,"client":"1","":"downstairs","session_id":"d379e042-7089-46f3-a869-0cb01cec787a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:43.933613902Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1228,"client":"2","":"downstairs","session_id":"d379e042-7089-46f3-a869-0cb01cec787a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:43.933631544Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1228,"client":"2","":"downstairs","session_id":"d379e042-7089-46f3-a869-0cb01cec787a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:43.933662938Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1228,"":"io task","client":"2","":"downstairs","session_id":"d379e042-7089-46f3-a869-0cb01cec787a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:43.933681081Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1228,"":"io task","client":"0","":"downstairs","session_id":"d379e042-7089-46f3-a869-0cb01cec787a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:43.933704954Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1228,"":"io task","client":"1","":"downstairs","session_id":"d379e042-7089-46f3-a869-0cb01cec787a"} {"msg":"ds_connection connected from Ok(127.0.0.1:37218)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:43.933743789Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1228,"":"io task","client":"2","":"downstairs","session_id":"d379e042-7089-46f3-a869-0cb01cec787a"} {"msg":"ds_connection connected from Ok(127.0.0.1:59190)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:43.933778543Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1228,"":"io task","client":"0","":"downstairs","session_id":"d379e042-7089-46f3-a869-0cb01cec787a"} {"msg":"ds_connection connected from Ok(127.0.0.1:53552)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:43.933815158Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1228,"":"io task","client":"1","":"downstairs","session_id":"d379e042-7089-46f3-a869-0cb01cec787a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:43.934619011Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1228,"client":"0","":"downstairs","session_id":"d379e042-7089-46f3-a869-0cb01cec787a"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:43.934647185Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1228,"client":"0","":"downstairs","session_id":"d379e042-7089-46f3-a869-0cb01cec787a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:43.934663537Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1228,"client":"2","":"downstairs","session_id":"d379e042-7089-46f3-a869-0cb01cec787a"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:43.934677419Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1228,"client":"2","":"downstairs","session_id":"d379e042-7089-46f3-a869-0cb01cec787a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:43.934691171Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1228,"client":"1","":"downstairs","session_id":"d379e042-7089-46f3-a869-0cb01cec787a"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:43.934704622Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1228,"client":"1","":"downstairs","session_id":"d379e042-7089-46f3-a869-0cb01cec787a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:43.935241712Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1228,"client":"1","":"downstairs","session_id":"d379e042-7089-46f3-a869-0cb01cec787a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:43.935285447Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1228,"client":"0","":"downstairs","session_id":"d379e042-7089-46f3-a869-0cb01cec787a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:43.935414624Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1228,"client":"2","":"downstairs","session_id":"d379e042-7089-46f3-a869-0cb01cec787a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:43.935839829Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1228,"client":"1","":"downstairs","session_id":"d379e042-7089-46f3-a869-0cb01cec787a"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:43.935864962Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1228,"client":"1","":"downstairs","session_id":"d379e042-7089-46f3-a869-0cb01cec787a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:43.935879384Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1228,"session_id":"d379e042-7089-46f3-a869-0cb01cec787a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:43.935955774Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1228,"client":"0","":"downstairs","session_id":"d379e042-7089-46f3-a869-0cb01cec787a"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:43.935989078Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1228,"client":"0","":"downstairs","session_id":"d379e042-7089-46f3-a869-0cb01cec787a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:43.936015631Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1228,"session_id":"d379e042-7089-46f3-a869-0cb01cec787a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:43.936030343Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1228,"client":"2","":"downstairs","session_id":"d379e042-7089-46f3-a869-0cb01cec787a"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:43.936044575Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1228,"client":"2","":"downstairs","session_id":"d379e042-7089-46f3-a869-0cb01cec787a"} {"msg":"[0]R flush_numbers[0..12]: [12, 11, 6, 14, 7, 3, 7, 14, 14, 14, 12, 14]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:43.936058407Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1228,"":"downstairs","":"downstairs","session_id":"d379e042-7089-46f3-a869-0cb01cec787a"} {"msg":"[0]R generation[0..12]: [8, 7, 5, 10, 6, 3, 6, 10, 10, 10, 8, 10]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:43.936072099Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1228,"":"downstairs","":"downstairs","session_id":"d379e042-7089-46f3-a869-0cb01cec787a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:43.936086251Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1228,"":"downstairs","":"downstairs","session_id":"d379e042-7089-46f3-a869-0cb01cec787a"} {"msg":"[1]R flush_numbers[0..12]: [12, 11, 6, 14, 7, 3, 7, 14, 14, 14, 12, 14]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:43.936100112Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1228,"":"downstairs","":"downstairs","session_id":"d379e042-7089-46f3-a869-0cb01cec787a"} {"msg":"[1]R generation[0..12]: [8, 7, 5, 10, 6, 3, 6, 10, 10, 10, 8, 10]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:43.936113674Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1228,"":"downstairs","":"downstairs","session_id":"d379e042-7089-46f3-a869-0cb01cec787a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:43.936131036Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1228,"":"downstairs","":"downstairs","session_id":"d379e042-7089-46f3-a869-0cb01cec787a"} {"msg":"[2]R flush_numbers[0..12]: [12, 11, 6, 14, 7, 3, 7, 14, 14, 14, 12, 14]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:43.936145608Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1228,"":"downstairs","":"downstairs","session_id":"d379e042-7089-46f3-a869-0cb01cec787a"} {"msg":"[2]R generation[0..12]: [8, 7, 5, 10, 6, 3, 6, 10, 10, 10, 8, 10]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:43.9361591Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":Wait for a query_work_queue command to finish before sending IO 1228,"":"downstairs","":"downstairs","session_id":"d379e042-7089-46f3-a869-0cb01cec787a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:43.936181083Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1228,"":"downstairs","":"downstairs","session_id":"d379e042-7089-46f3-a869-0cb01cec787a"} {"msg":"Max found gen is 12","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:43.936194294Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1228,"":"downstairs","":"downstairs","session_id":"d379e042-7089-46f3-a869-0cb01cec787a"} {"msg":"Generation requested: 12 >= found:12","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:43.936207626Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1228,"":"downstairs","":"downstairs","session_id":"d379e042-7089-46f3-a869-0cb01cec787a"} {"msg":"Next flush: 16","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:43.936220698Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1228,"":"downstairs","":"downstairs","session_id":"d379e042-7089-46f3-a869-0cb01cec787a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:43.93623943Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1228,"":"downstairs","":"downstairs","session_id":"d379e042-7089-46f3-a869-0cb01cec787a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:43.936255052Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1228,"session_id":"d379e042-7089-46f3-a869-0cb01cec787a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:43.936267394Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1228,"client":"0","":"downstairs","session_id":"d379e042-7089-46f3-a869-0cb01cec787a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:43.936281616Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1228,"client":"1","":"downstairs","session_id":"d379e042-7089-46f3-a869-0cb01cec787a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:43.936295237Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1228,"client":"2","":"downstairs","session_id":"d379e042-7089-46f3-a869-0cb01cec787a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:43.936308689Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1228,"session_id":"d379e042-7089-46f3-a869-0cb01cec787a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:43.936320921Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1228,"session_id":"d379e042-7089-46f3-a869-0cb01cec787a"} {"msg":"b74eb8eb-b711-4a4b-89d3-e9a289557999 is now active with session: d379e042-7089-46f3-a869-0cb01cec787a","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:43.936334132Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1228,"session_id":"d379e042-7089-46f3-a869-0cb01cec787a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:43.936346694Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1228,"session_id":"d379e042-7089-46f3-a869-0cb01cec787a"} {"msg":"The guest has finished waiting for activation with:12","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:43.936362116Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1228} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 450 len 28672 data: 3 3 3 3 2 2 2 02/30 Write block 221 len 24576 data: 3 3 4 4 4 3 03/30 Write block 41 len 24576 data: 2 2 2 2 2 2 04/30 Write block 105 len 40960 data: 2 2 2 2 2 2 2 2 2 2 05/30 Flush 06/30 Write block 593 len 24576 data: 2 2 2 2 2 2 07/30 Read block 271 len 12288 08/30 Write block 438 len 36864 data: 2 2 3 3 3 3 3 3 3 09/30 Write block 100 len 40960 data: 2 3 3 3 3 3 3 3 3 3 10/30 Read block 468 len 32768 11/30 Read block 237 len 20480 12/30 Read block 428 len 12288 13/30 Read block 526 len 36864 14/30 Read block 459 len 32768 15/30 Read block 412 len 16384 16/30 Read block 520 len 24576 17/30 Read block 122 len 24576 18/30 Write block 475 len 28672 data: 3 3 3 4 3 3 2 19/30 Write block 567 len 24576 data: 2 3 3 2 2 2 20/30 Write block 436 len 40960 data: 2 2 3 3 4 4 4 4 4 4 21/30 Read block 244 len 32768 22/30 Write block 79 len 8192 data: 2 2 23/30 Read block 160 len 32768 24/30 Write block 90 len 4096 data: 4 25/30 Read block 66 len 20480 26/30 Write block 286 len 8192 data: 2 2 27/30 Write block 205 len 36864 data: 4 4 4 4 3 4 4 4 3 28/30 Write block 417 len 24576 data: 2 4 4 4 4 4 29/30 Read block 16 len 4096 30/30 Write block 304 len 20480 data: 3 3 3 3 4 ---------------------------------------------------------------- Crucible gen:12 GIO:true work queues: Upstairs:0 downstairs:25 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1005 Acked Write 6 Done Done Done false 1006 Acked Read 3 Done Done Done false 1007 Acked Write 9 Done Done Done false 1008 Acked Write 10 Done Done Done false 1009 Acked Read 8 Done Done Done false 1010 Acked Read 5 Done Done Done false 1011 Acked Read 3 Done Done Done false 1012 Acked Read 9 Done Done Done false 1013 Acked Read 8 Done Done Done false 1014 Acked Read 4 Done Done Done false 1015 Acked Read 6 Done Done Done false 1016 Acked Read 6 Done Done Done false 1017 Acked Write 7 Done Done Done false 1018 Acked Write 6 Done Done Done false 1019 Acked Write 10 Done Done Done false 1020 Acked Read 8 Done Done Done false 1021 Acked Write 2 Done Done Done false 1022 Acked Read 8 Done Done Done false 1023 Acked Write 1 Done Done Done false 1024 Acked Read 5 Done Done Done false 1025 Acked Write 2 Done Done Done false 1026 Acked Write 9 Done Done Done false 1027 Acked Write 6 Done Done Done false 1028 Acked Read 1 Done Done Done false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 24 24 24 72 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1004 1004 1004 Downstairs last five completed: 1004 1003 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:50:43.956939887Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1228} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 8 8 8 12 12 12 F F F 1 020-039 7 7 7 11 11 11 F F F 2 040-059 12 12 12 16 16 16 F F F 3 060-079 10 10 10 14 14 14 T T T 4 080-099 6 6 6 7 7 7 T T T 5 100-119 12 12 12 16 16 16 T T T 6 120-139 6 6 6 7 7 7 F F F 7 140-159 10 10 10 14 14 14 F F F 8 160-179 10 10 10 14 14 14 F F F 9 180-199 10 10 10 14 14 14 F F F 10 200-219 8 8 8 12 12 12 T T T 11 220-239 12 12 12 16 16 16 F F F 12 240-259 10 10 10 14 14 14 F F F 13 260-279 8 8 8 12 12 12 F F F 14 280-299 6 6 6 8 8 8 T T T 15 300-319 11 11 11 15 15 15 T T T 16 320-339 11 11 11 15 15 15 F F F 17 340-359 6 6 6 8 8 8 F F F 18 360-379 8 8 8 12 12 12 F F F 19 380-399 10 10 10 14 14 14 F F F 20 400-419 10 10 10 14 14 14 T T T 21 420-439 7 7 7 11 11 11 T T T 22 440-459 12 12 12 16 16 16 T T T 23 460-479 9 9 9 13 13 13 T T T 24 480-499 8 8 8 12 12 12 T T T 25 500-519 6 6 6 9 9 9 F F F 26 520-539 7 7 7 11 11 11 F F F 27 540-559 6 6 6 8 8 8 F F F 28 560-579 5 5 5 6 6 6 T T T 29 580-599 8 8 8 12 12 12 T T T Max gen: 12, Max flush: 16 A Difference in extent metadata is expected here On loop 6 Verifying data now No change after verify Loop: 6 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 8 8 8 12 12 12 F F F 1 020-039 7 7 7 11 11 11 F F F 2 040-059 12 12 12 16 16 16 F F F 3 060-079 13 13 13 17 17 17 F F F 4 080-099 13 13 13 17 17 17 F F F 5 100-119 13 13 13 17 17 17 F F F 6 120-139 6 6 6 7 7 7 F F F 7 140-159 10 10 10 14 14 14 F F F 8 160-179 10 10 10 14 14 14 F F F 9 180-199 10 10 10 14 14 14 F F F 10 200-219 13 13 13 17 17 17 F F F 11 220-239 12 12 12 16 16 16 F F F 12 240-259 10 10 10 14 14 14 F F F 13 260-279 8 8 8 12 12 12 F F F 14 280-299 13 13 13 17 17 17 F F F 15 300-319 13 13 13 17 17 17 F F F 16 320-339 11 11 11 15 15 15 F F F 17 340-359 6 6 6 8 8 8 F F F 18 360-379 8 8 8 12 12 12 F F F 19 380-399 10 10 10 14 14 14 F F F 20 400-419 13 13 13 17 17 17 F F F 21 420-439 13 13 13 17 17 17 F F F 22 440-459 13 13 13 17 17 17 F F F 23 460-479 13 13 13 17 17 17 F F F 24 480-499 13 13 13 17 17 17 F F F 25 500-519 6 6 6 9 9 9 F F F 26 520-539 7 7 7 11 11 11 F F F 27 540-559 6 6 6 8 8 8 F F F 28 560-579 13 13 13 17 17 17 F F F 29 580-599 13 13 13 17 17 17 F F F Max gen: 13, Max flush: 17 Begin loop 7 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:48.394082011Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1236,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:50:48.436579417Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1236,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:48.436939793Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1236,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:48.468598132Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1236,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:48.469104287Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1236} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"3818f0fc76fe787c6cc412af30fd59b930203885\",\n git_commit_timestamp: \"2024-11-01T16:08:30.000000000Z\",\n git_branch: \"renovate/hyper-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:48.469136662Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1236} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:48.469151664Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1236} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:48.469165975Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1236} {"msg":"Crucible 313bc318-8024-45af-9c84-d282e353a0c0 has session id: 4dcf4efa-180a-431b-b181-2a1f6e848ba6","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:48.469574308Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1236,"session_id":"4dcf4efa-180a-431b-b181-2a1f6e848ba6"} {"msg":"Upstairs opts: Upstairs UUID: 313bc318-8024-45af-9c84-d282e353a0c0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:48.469614533Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1236,"session_id":"4dcf4efa-180a-431b-b181-2a1f6e848ba6"} {"msg":"Crucible stats registered with UUID: 313bc318-8024-45af-9c84-d282e353a0c0","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:48.469632166Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1236,"session_id":"4dcf4efa-180a-431b-b181-2a1f6e848ba6"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:48.500312589Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1236,"task":"crutest"} {"msg":"The guest has requested activation with gen:14","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:48.500344193Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1236} {"msg":"313bc318-8024-45af-9c84-d282e353a0c0 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:48.500369896Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1236,"session_id":"4dcf4efa-180a-431b-b181-2a1f6e848ba6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:48.500386288Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1236,"client":"0","":"downstairs","session_id":"4dcf4efa-180a-431b-b181-2a1f6e848ba6"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:48.500415682Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1236,"client":"0","":"downstairs","session_id":"4dcf4efa-180a-431b-b181-2a1f6e848ba6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:48.500437985Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1236,"client":"1","":"downstairs","session_id":"4dcf4efa-180a-431b-b181-2a1f6e848ba6"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:48.500455527Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1236,"client":"1","":"downstairs","session_id":"4dcf4efa-180a-431b-b181-2a1f6e848ba6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:48.500472039Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1236,"client":"2","":"downstairs","session_id":"4dcf4efa-180a-431b-b181-2a1f6e848ba6"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:48.500499543Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1236,"client":"2","":"downstairs","session_id":"4dcf4efa-180a-431b-b181-2a1f6e848ba6"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:48.500531617Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1236,"":"io task","client":"2","":"downstairs","session_id":"4dcf4efa-180a-431b-b181-2a1f6e848ba6"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:48.500565841Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1236,"":"io task","client":"0","":"downstairs","session_id":"4dcf4efa-180a-431b-b181-2a1f6e848ba6"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:48.500600006Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1236,"":"io task","client":"1","":"downstairs","session_id":"4dcf4efa-180a-431b-b181-2a1f6e848ba6"} {"msg":"ds_connection connected from Ok(127.0.0.1:36735)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:48.50063183Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1236,"":"io task","client":"2","":"downstairs","session_id":"4dcf4efa-180a-431b-b181-2a1f6e848ba6"} {"msg":"ds_connection connected from Ok(127.0.0.1:55015)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:48.500665034Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1236,"":"io task","client":"0","":"downstairs","session_id":"4dcf4efa-180a-431b-b181-2a1f6e848ba6"} {"msg":"ds_connection connected from Ok(127.0.0.1:47496)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:48.500697208Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1236,"":"io task","client":"1","":"downstairs","session_id":"4dcf4efa-180a-431b-b181-2a1f6e848ba6"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:48.501312208Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1236,"client":"1","":"downstairs","session_id":"4dcf4efa-180a-431b-b181-2a1f6e848ba6"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:48.501339761Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1236,"client":"1","":"downstairs","session_id":"4dcf4efa-180a-431b-b181-2a1f6e848ba6"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:48.501354513Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1236,"client":"2","":"downstairs","session_id":"4dcf4efa-180a-431b-b181-2a1f6e848ba6"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:48.501375736Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1236,"client":"2","":"downstairs","session_id":"4dcf4efa-180a-431b-b181-2a1f6e848ba6"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:48.501496771Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1236,"client":"0","":"downstairs","session_id":"4dcf4efa-180a-431b-b181-2a1f6e848ba6"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:48.501535456Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1236,"client":"0","":"downstairs","session_id":"4dcf4efa-180a-431b-b181-2a1f6e848ba6"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:48.50202997Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1236,"client":"1","":"downstairs","session_id":"4dcf4efa-180a-431b-b181-2a1f6e848ba6"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:48.502197682Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1236,"client":"0","":"downstairs","session_id":"4dcf4efa-180a-431b-b181-2a1f6e848ba6"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:48.502219274Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1236,"client":"2","":"downstairs","session_id":"4dcf4efa-180a-431b-b181-2a1f6e848ba6"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:48.502538915Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1236,"client":"1","":"downstairs","session_id":"4dcf4efa-180a-431b-b181-2a1f6e848ba6"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:48.502578311Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1236,"client":"1","":"downstairs","session_id":"4dcf4efa-180a-431b-b181-2a1f6e848ba6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:48.502606064Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1236,"session_id":"4dcf4efa-180a-431b-b181-2a1f6e848ba6"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:48.502757244Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1236,"client":"0","":"downstairs","session_id":"4dcf4efa-180a-431b-b181-2a1f6e848ba6"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:48.502784937Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1236,"client":"0","":"downstairs","session_id":"4dcf4efa-180a-431b-b181-2a1f6e848ba6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:48.502800449Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1236,"session_id":"4dcf4efa-180a-431b-b181-2a1f6e848ba6"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:48.502813531Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1236,"client":"2","":"downstairs","session_id":"4dcf4efa-180a-431b-b181-2a1f6e848ba6"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:48.502827353Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1236,"client":"2","":"downstairs","session_id":"4dcf4efa-180a-431b-b181-2a1f6e848ba6"} {"msg":"[0]R flush_numbers[0..12]: [12, 11, 16, 17, 17, 17, 7, 14, 14, 14, 17, 16]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:48.502841194Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1236,"":"downstairs","":"downstairs","session_id":"4dcf4efa-180a-431b-b181-2a1f6e848ba6"} {"msg":"[0]R generation[0..12]: [8, 7, 12, 13, 13, 13, 6, 10, 10, 10, 13, 12]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:48.502856056Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1236,"":"downstairs","":"downstairs","session_id":"4dcf4efa-180a-431b-b181-2a1f6e848ba6"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:48.502871408Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1236,"":"downstairs","":"downstairs","session_id":"4dcf4efa-180a-431b-b181-2a1f6e848ba6"} {"msg":"[1]R flush_numbers[0..12]: [12, 11, 16, 17, 17, 17, 7, 14, 14, 14, 17, 16]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:48.50288541Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1236,"":"downstairs","":"downstairs","session_id":"4dcf4efa-180a-431b-b181-2a1f6e848ba6"} {"msg":"[1]R generation[0..12]: [8, 7, 12, 13, 13, 13, 6, 10, 10, 10, 13, 12]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:48.502902772Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1236,"":"downstairs","":"downstairs","session_id":"4dcf4efa-180a-431b-b181-2a1f6e848ba6"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:48.502927076Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1236,"":"downstairs","":"downstairs","session_id":"4dcf4efa-180a-431b-b181-2a1f6e848ba6"} {"msg":"[2]R flush_numbers[0..12]: [12, 11, 16, 17, 17, 17, 7, 14, 14, 14, 17, 16]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:48.502954549Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1236,"":"downstairs","":"downstairs","session_id":"4dcf4efa-180a-431b-b181-2a1f6e848ba6"} {"msg":"[2]R generation[0..12]: [8, 7, 12, 13, 13, 13, 6, 10, 10, 10, 13, 12]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:48.502983013Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1236,"":"downstairs","":"downstairs","session_id":"4dcf4efa-180a-431b-b181-2a1f6e848ba6"} Wait for a query_work_queue command to finish before sending IO {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:48.503018137Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1236,"":"downstairs","":"downstairs","session_id":"4dcf4efa-180a-431b-b181-2a1f6e848ba6"} {"msg":"Max found gen is 14","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:48.503042611Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1236,"":"downstairs","":"downstairs","session_id":"4dcf4efa-180a-431b-b181-2a1f6e848ba6"} {"msg":"Generation requested: 14 >= found:14","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:48.503064713Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1236,"":"downstairs","":"downstairs","session_id":"4dcf4efa-180a-431b-b181-2a1f6e848ba6"} {"msg":"Next flush: 18","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:48.503089606Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1236,"":"downstairs","":"downstairs","session_id":"4dcf4efa-180a-431b-b181-2a1f6e848ba6"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:48.50311566Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1236,"":"downstairs","":"downstairs","session_id":"4dcf4efa-180a-431b-b181-2a1f6e848ba6"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:48.503135803Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1236,"session_id":"4dcf4efa-180a-431b-b181-2a1f6e848ba6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:48.503149214Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1236,"client":"0","":"downstairs","session_id":"4dcf4efa-180a-431b-b181-2a1f6e848ba6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:48.503172427Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1236,"client":"1","":"downstairs","session_id":"4dcf4efa-180a-431b-b181-2a1f6e848ba6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:48.503198151Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1236,"client":"2","":"downstairs","session_id":"4dcf4efa-180a-431b-b181-2a1f6e848ba6"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:48.503224434Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1236,"session_id":"4dcf4efa-180a-431b-b181-2a1f6e848ba6"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:48.503238916Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1236,"session_id":"4dcf4efa-180a-431b-b181-2a1f6e848ba6"} {"msg":"313bc318-8024-45af-9c84-d282e353a0c0 is now active with session: 4dcf4efa-180a-431b-b181-2a1f6e848ba6","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:48.503259568Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1236,"session_id":"4dcf4efa-180a-431b-b181-2a1f6e848ba6"} {"msg":"Set Active after no reconciliation","v":Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 0,"name":"crucible","level":30,"time":"2024-11-01T16:50:48.503285382Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1236,"session_id":"4dcf4efa-180a-431b-b181-2a1f6e848ba6"} {"msg":"The guest has finished waiting for activation with:14","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:48.503307105Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1236} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 443 len 36864 data: 5 5 5 4 4 3 3 4 4 02/30 Write block 244 len 28672 data: 2 2 2 4 4 4 5 03/30 Read block 554 len 24576 04/30 Read block 217 len 28672 05/30 Read block 173 len 40960 06/30 Write block 216 len 12288 data: 2 2 2 07/30 Read block 274 len 20480 08/30 Read block 456 len 4096 09/30 Write block 316 len 36864 data: 3 2 3 3 3 3 3 3 3 10/30 Read block 362 len 36864 11/30 Read block 562 len 20480 12/30 Flush 13/30 Write block 217 len 8192 data: 3 3 14/30 Flush 15/30 Flush 16/30 Write block 125 len 32768 data: 2 2 2 2 3 4 5 5 17/30 Read block 424 len 24576 18/30 Read block 125 len 24576 19/30 Write block 42 len 12288 data: 3 3 3 20/30 Write block 302 len 28672 data: 2 3 4 4 4 4 5 21/30 Read block 387 len 20480 22/30 Write block 329 len 40960 data: 3 3 3 3 3 3 3 2 2 2 23/30 Read block 500 len 4096 24/30 Write block 218 len 12288 data: 4 2 2 25/30 Read block 134 len 12288 26/30 Write block 268 len 40960 data: 2 2 2 3 2 2 3 3 3 2 27/30 Write block 595 len 4096 data: 3 28/30 Read block 69 len 4096 29/30 Read block 502 len 40960 30/30 Read block 448 len 4096 ---------------------------------------------------------------- Crucible gen:14 GIO:true work queues: Upstairs:0 downstairs:15 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1015 Acked Write 8 Done Done Done false 1016 Acked Read 6 Done Done Done false 1017 Acked Read 6 Done Done Done false 1018 Acked Write 3 Done Done Done false 1019 Acked Write 7 Done Done Done false 1020 Acked Read 5 Done Done Done false 1021 Acked Write 10 Done Done Done false 1022 Acked Read 1 Done Done Done false 1023 Acked Write 3 Done Done Done false 1024 Acked Read 3 Done Done Done false 1025 Acked Write 10 Done Done Done false 1026 Acked Write 1 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Read 10 Done Done Done false 1029 Acked Read 1 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 15 15 15 45 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1014 1014 1014 Downstairs last five completed: 1014 1013 1012 1011 1010 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:50:48.528001746Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1236} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 8 8 8 12 12 12 F F F 1 020-039 7 7 7 11 11 11 F F F 2 040-059 12 12 12 16 16 16 T T T 3 060-079 13 13 13 17 17 17 F F F 4 080-099 13 13 13 17 17 17 F F F 5 100-119 13 13 13 17 17 17 F F F 6 120-139 6 6 6 7 7 7 T T T 7 140-159 10 10 10 14 14 14 F F F 8 160-179 10 10 10 14 14 14 F F F 9 180-199 10 10 10 14 14 14 F F F 10 200-219 14 14 14 19 19 19 T T T 11 220-239 12 12 12 16 16 16 T T T 12 240-259 14 14 14 18 18 18 F F F 13 260-279 8 8 8 12 12 12 T T T 14 280-299 13 13 13 17 17 17 F F F 15 300-319 14 14 14 18 18 18 T T T 16 320-339 14 14 14 18 18 18 T T T 17 340-359 6 6 6 8 8 8 F F F 18 360-379 8 8 8 12 12 12 F F F 19 380-399 10 10 10 14 14 14 F F F 20 400-419 13 13 13 17 17 17 F F F 21 420-439 13 13 13 17 17 17 F F F 22 440-459 14 14 14 18 18 18 F F F 23 460-479 13 13 13 17 17 17 F F F 24 480-499 13 13 13 17 17 17 F F F 25 500-519 6 6 6 9 9 9 F F F 26 520-539 7 7 7 11 11 11 F F F 27 540-559 6 6 6 8 8 8 F F F 28 560-579 13 13 13 17 17 17 F F F 29 580-599 13 13 13 17 17 17 T T T Max gen: 14, Max flush: 19 A Difference in extent metadata is expected here On loop 7 Verifying data now No change after verify Loop: 7 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 8 8 8 12 12 12 F F F 1 020-039 7 7 7 11 11 11 F F F 2 040-059 15 15 15 20 20 20 F F F 3 060-079 13 13 13 17 17 17 F F F 4 080-099 13 13 13 17 17 17 F F F 5 100-119 13 13 13 17 17 17 F F F 6 120-139 15 15 15 20 20 20 F F F 7 140-159 10 10 10 14 14 14 F F F 8 160-179 10 10 10 14 14 14 F F F 9 180-199 10 10 10 14 14 14 F F F 10 200-219 15 15 15 20 20 20 F F F 11 220-239 15 15 15 20 20 20 F F F 12 240-259 14 14 14 18 18 18 F F F 13 260-279 15 15 15 20 20 20 F F F 14 280-299 13 13 13 17 17 17 F F F 15 300-319 15 15 15 20 20 20 F F F 16 320-339 15 15 15 20 20 20 F F F 17 340-359 6 6 6 8 8 8 F F F 18 360-379 8 8 8 12 12 12 F F F 19 380-399 10 10 10 14 14 14 F F F 20 400-419 13 13 13 17 17 17 F F F 21 420-439 13 13 13 17 17 17 F F F 22 440-459 14 14 14 18 18 18 F F F 23 460-479 13 13 13 17 17 17 F F F 24 480-499 13 13 13 17 17 17 F F F 25 500-519 6 6 6 9 9 9 F F F 26 520-539 7 7 7 11 11 11 F F F 27 540-559 6 6 6 8 8 8 F F F 28 560-579 13 13 13 17 17 17 F F F 29 580-599 15 15 15 20 20 20 F F F Max gen: 15, Max flush: 20 Begin loop 8 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:52.904197096Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1244,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:50:52.94683587Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1244,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:52.947214979Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1244,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:52.978930045Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1244,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:52.979441262Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1244} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"3818f0fc76fe787c6cc412af30fd59b930203885\",\n git_commit_timestamp: \"2024-11-01T16:08:30.000000000Z\",\n git_branch: \"renovate/hyper-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:52.979476046Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1244} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:52.979492098Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1244} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:52.97950585Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1244} {"msg":"Crucible 214e3b42-bd02-4c95-85d0-6a219b95b7df has session id: bfc997aa-0b13-4b1e-9a10-33ae535e14b0","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:52.979963819Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1244,"session_id":"bfc997aa-0b13-4b1e-9a10-33ae535e14b0"} {"msg":"Upstairs opts: Upstairs UUID: 214e3b42-bd02-4c95-85d0-6a219b95b7df, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:52.979991152Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1244,"session_id":"bfc997aa-0b13-4b1e-9a10-33ae535e14b0"} {"msg":"Crucible stats registered with UUID: 214e3b42-bd02-4c95-85d0-6a219b95b7df","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:52.980004994Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1244,"session_id":"bfc997aa-0b13-4b1e-9a10-33ae535e14b0"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:53.010541579Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1244,"task":"crutest"} {"msg":"The guest has requested activation with gen:16","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:53.010569732Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1244} {"msg":"214e3b42-bd02-4c95-85d0-6a219b95b7df active request set","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:53.010648683Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1244,"session_id":"bfc997aa-0b13-4b1e-9a10-33ae535e14b0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:53.010665545Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1244,"client":"0","":"downstairs","session_id":"bfc997aa-0b13-4b1e-9a10-33ae535e14b0"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:53.010690278Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1244,"client":"0","":"downstairs","session_id":"bfc997aa-0b13-4b1e-9a10-33ae535e14b0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:53.01070496Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1244,"client":"1","":"downstairs","session_id":"bfc997aa-0b13-4b1e-9a10-33ae535e14b0"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:53.010718972Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1244,"client":"1","":"downstairs","session_id":"bfc997aa-0b13-4b1e-9a10-33ae535e14b0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:53.010735634Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1244,"client":"2","":"downstairs","session_id":"bfc997aa-0b13-4b1e-9a10-33ae535e14b0"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:53.010749716Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1244,"client":"2","":"downstairs","session_id":"bfc997aa-0b13-4b1e-9a10-33ae535e14b0"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:53.010763037Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1244,"":"io task","client":"2","":"downstairs","session_id":"bfc997aa-0b13-4b1e-9a10-33ae535e14b0"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:53.010852579Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1244,"":"io task","client":"0","":"downstairs","session_id":"bfc997aa-0b13-4b1e-9a10-33ae535e14b0"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:53.011079328Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1244,"":"io task","client":"1","":"downstairs","session_id":"bfc997aa-0b13-4b1e-9a10-33ae535e14b0"} {"msg":"ds_connection connected from Ok(127.0.0.1:58837)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:53.011109932Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1244,"":"io task","client":"2","":"downstairs","session_id":"bfc997aa-0b13-4b1e-9a10-33ae535e14b0"} {"msg":"ds_connection connected from Ok(127.0.0.1:54730)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:53.011135325Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1244,"":"io task","client":"1","":"downstairs","session_id":"bfc997aa-0b13-4b1e-9a10-33ae535e14b0"} {"msg":"ds_connection connected from Ok(127.0.0.1:46540)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:53.011159318Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1244,"":"io task","client":"0","":"downstairs","session_id":"bfc997aa-0b13-4b1e-9a10-33ae535e14b0"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:53.011577062Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1244,"client":"1","":"downstairs","session_id":"bfc997aa-0b13-4b1e-9a10-33ae535e14b0"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:53.011604796Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1244,"client":"1","":"downstairs","session_id":"bfc997aa-0b13-4b1e-9a10-33ae535e14b0"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:53.011746644Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1244,"client":"0","":"downstairs","session_id":"bfc997aa-0b13-4b1e-9a10-33ae535e14b0"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:53.011788179Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1244,"client":"0","":"downstairs","session_id":"bfc997aa-0b13-4b1e-9a10-33ae535e14b0"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:53.011821854Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1244,"client":"2","":"downstairs","session_id":"bfc997aa-0b13-4b1e-9a10-33ae535e14b0"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:53.011852578Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1244,"client":"2","":"downstairs","session_id":"bfc997aa-0b13-4b1e-9a10-33ae535e14b0"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:53.012235177Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1244,"client":"1","":"downstairs","session_id":"bfc997aa-0b13-4b1e-9a10-33ae535e14b0"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:53.012267811Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1244,"client":"2","":"downstairs","session_id":"bfc997aa-0b13-4b1e-9a10-33ae535e14b0"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:53.012553148Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1244,"client":"0","":"downstairs","session_id":"bfc997aa-0b13-4b1e-9a10-33ae535e14b0"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:53.012815432Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1244,"client":"2","":"downstairs","session_id":"bfc997aa-0b13-4b1e-9a10-33ae535e14b0"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:53.012844515Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1244,"client":"2","":"downstairs","session_id":"bfc997aa-0b13-4b1e-9a10-33ae535e14b0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:53.012859587Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1244,"session_id":"bfc997aa-0b13-4b1e-9a10-33ae535e14b0"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:53.012898662Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1244,"client":"1","":"downstairs","session_id":"bfc997aa-0b13-4b1e-9a10-33ae535e14b0"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:53.012915444Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1244,"client":"1","":"downstairs","session_id":"bfc997aa-0b13-4b1e-9a10-33ae535e14b0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:53.012929236Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1244,"session_id":"bfc997aa-0b13-4b1e-9a10-33ae535e14b0"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:53.013026009Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1244,"client":"0","":"downstairs","session_id":"bfc997aa-0b13-4b1e-9a10-33ae535e14b0"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:53.013045311Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1244,"client":"0","":"downstairs","session_id":"bfc997aa-0b13-4b1e-9a10-33ae535e14b0"} {"msg":"[0]R flush_numbers[0..12]: [12, 11, 20, 17, 17, 17, 20, 14, 14, 14, 20, 20]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:53.013059303Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1244,"":"downstairs","":"downstairs","session_id":"bfc997aa-0b13-4b1e-9a10-33ae535e14b0"} {"msg":"[0]R generation[0..12]: [8, 7, 15, 13, 13, 13, 15, 10, 10, 10, 15, 15]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:53.013073095Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1244,"":"downstairs","":"downstairs","session_id":"bfc997aa-0b13-4b1e-9a10-33ae535e14b0"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:53.013087377Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1244,"":"downstairs","":"downstairs","session_id":"bfc997aa-0b13-4b1e-9a10-33ae535e14b0"} {"msg":"[1]R flush_numbers[0..12]: [12, 11, 20, 17, 17, 17, 20, 14, 14, 14, 20, 20]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:53.013101059Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1244,"":"downstairs","":"downstairs","session_id":"bfc997aa-0b13-4b1e-9a10-33ae535e14b0"} {"msg":"[1]R generation[0..12]: [8, 7, 15, 13, 13, 13, 15, 10, 10, 10, 15, 15]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:53.01311484Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1244,"":"downstairs","":"downstairs","session_id":"bfc997aa-0b13-4b1e-9a10-33ae535e14b0"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:53.013132732Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1244,"":"downstairs","":"downstairs","session_id":"bfc997aa-0b13-4b1e-9a10-33ae535e14b0"} {"msg":"[2]R flush_numbers[0..12]: [12, 11, 20, 17, 17, 17, 20, 14, 14, 14, 20, 20]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:53.013146904Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1244,"":"downstairs","":"downstairs","session_id":"bfc997aa-0b13-4b1e-9a10-33ae535e14b0"} {"msg":"[2]R generation[0..12]: [8, 7, 15, 13, 13, 13, 15, 10, 10, 10, 15, 15]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:53.013160666Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1244,"":"downstairs","":"downstairs","session_id":"bfc997aa-0b13-4b1e-9a10-33ae535e14b0"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:53.013175418Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1244,"":"downstairs","":"downstairs","session_id":"bfc997aa-0b13-4b1e-9a10-33ae535e14b0"} {"msg":"Max found gen is 16","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:53.01318862Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1244,"":"downstairs","":"downstairs","session_id":"bfc997aa-0b13-4b1e-9a10-33ae535e14b0"} {"msg":"Generation requested: 16 >= found:16","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:53.013201892Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1244,"":"downstairs","":"downstairs","session_id":"bfc997aa-0b13-4b1e-9a10-33ae535e14b0"} {"msg":"Next flush: 21","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:53.013214943Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1244,"":"downstairs","":"downstairs","session_id":"bfc997aa-0b13-4b1e-9a10-33ae535e14b0"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:53.013227945Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1244,"Wait for a query_work_queue command to finish before sending IO ":"downstairs","":"downstairs","session_id":"bfc997aa-0b13-4b1e-9a10-33ae535e14b0"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:53.013250278Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1244,"session_id":"bfc997aa-0b13-4b1e-9a10-33ae535e14b0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:53.013262519Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1244,"client":"0","":"downstairs","session_id":"bfc997aa-0b13-4b1e-9a10-33ae535e14b0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:53.013275841Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1244,"client":"1","":"downstairs","session_id":"bfc997aa-0b13-4b1e-9a10-33ae535e14b0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:53.013289023Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1244,"client":"2","":"downstairs","session_id":"bfc997aa-0b13-4b1e-9a10-33ae535e14b0"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:53.013302334Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1244,"session_id":"bfc997aa-0b13-4b1e-9a10-33ae535e14b0"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:53.013314586Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1244,"session_id":"bfc997aa-0b13-4b1e-9a10-33ae535e14b0"} {"msg":"214e3b42-bd02-4c95-85d0-6a219b95b7df is now active with session: bfc997aa-0b13-4b1e-9a10-33ae535e14b0","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:53.013327237Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1244,"session_id":"bfc997aa-0b13-4b1e-9a10-33ae535e14b0"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:53.013341039Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1244,"session_id":"bfc997aa-0b13-4b1e-9a10-33ae535e14b0"} {"msg":"The guest has finished waiting for activation with:16","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:53.013353251Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1244} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 49 len 32768 data: 2 2 2 2 2 2 2 2 02/30 Read block 61 len 32768 03/30 Read block 363 len 40960 04/30 Read block 518 len 40960 05/30 Read block 17 len 24576 06/30 Read block 137 len 28672 07/30 Read block 140 len 28672 08/30 Write block 315 len 36864 data: 3 4 3 4 4 4 4 4 4 09/30 Read block 469 len 36864 10/30 Write block 79 len 8192 data: 3 3 11/30 Read block 562 len 4096 12/30 Write block 519 len 32768 data: 2 2 2 2 2 2 2 2 13/30 Read block 333 len 28672 14/30 Read block 243 len 4096 15/30 Write block 132 len 4096 data: 6 16/30 Write block 176 len 24576 data: 2 3 3 3 3 3 17/30 Write block 99 len 36864 data: 3 3 4 4 4 4 4 4 4 18/30 Read block 457 len 4096 19/30 Read block 116 len 8192 20/30 Flush 21/30 Write block 395 len 36864 data: 3 5 6 6 6 5 4 4 4 22/30 Flush 23/30 Write block 34 len 24576 data: 2 2 2 2 2 2 24/30 Write block 528 len 28672 data: 2 2 3 3 3 3 2 25/30 Write block 479 len 40960 data: 4 4 3 2 2 2 2 2 2 3 26/30 Read block 520 len 40960 27/30 Write block 43 len 20480 data: 4 4 3 3 2 28/30 Write block 553 len 28672 data: 3 2 2 2 2 2 2 29/30 Write block 183 len 32768 data: 2 2 2 2 2 3 2 3 30/30 Read block 392 len 40960 ---------------------------------------------------------------- Crucible gen:16 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 6 Done Done Done false 1023 Acked Write 7 Done Done Done false 1024 Acked Write 10 Done Done Done false 1025 Acked Read 10 Done Done Done false 1026 Acked Write 5 Done Done Done false 1027 Acked Write 7 Done Done Done false 1028 Acked Write 8 Sent Done Done false 1029 Acked Read 10 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 0 1 3 Done 6 8 7 21 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1021 1021 1021 Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:50:53.037685616Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1244} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 8 8 8 12 12 12 F F F 1 020-039 7 7 7 11 11 11 T T T 2 040-059 16 16 16 21 21 21 T T T 3 060-079 16 16 16 21 21 21 F F F 4 080-099 16 16 16 21 21 21 F F F 5 100-119 16 16 16 21 21 21 F F F 6 120-139 16 16 16 21 21 21 F F F 7 140-159 10 10 10 14 14 14 F F F 8 160-179 16 16 16 21 21 21 F F F 9 180-199 16 16 16 21 21 21 T T T 10 200-219 15 15 15 20 20 20 F F F 11 220-239 15 15 15 20 20 20 F F F 12 240-259 14 14 14 18 18 18 F F F 13 260-279 15 15 15 20 20 20 F F F 14 280-299 13 13 13 17 17 17 F F F 15 300-319 16 16 16 21 21 21 F F F 16 320-339 16 16 16 21 21 21 F F F 17 340-359 6 6 6 8 8 8 F F F 18 360-379 8 8 8 12 12 12 F F F 19 380-399 16 16 16 22 22 22 F F F 20 400-419 16 16 16 22 22 22 F F F 21 420-439 13 13 13 17 17 17 F F F 22 440-459 14 14 14 18 18 18 F F F 23 460-479 13 13 13 17 17 17 T T T 24 480-499 13 13 13 17 17 17 T T T 25 500-519 16 16 16 21 21 21 F F F 26 520-539 16 16 16 21 21 21 T T T 27 540-559 6 6 6 8 8 8 T T T 28 560-579 13 13 13 17 17 17 F F F 29 580-599 15 15 15 20 20 20 F F F Max gen: 16, Max flush: 22 A Difference in extent metadata is expected here On loop 8 Verifying data now No change after verify Loop: 8 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 8 8 8 12 12 12 F F F 1 020-039 17 17 17 23 23 23 F F F 2 040-059 17 17 17 23 23 23 F F F 3 060-079 16 16 16 21 21 21 F F F 4 080-099 16 16 16 21 21 21 F F F 5 100-119 16 16 16 21 21 21 F F F 6 120-139 16 16 16 21 21 21 F F F 7 140-159 10 10 10 14 14 14 F F F 8 160-179 16 16 16 21 21 21 F F F 9 180-199 17 17 17 23 23 23 F F F 10 200-219 15 15 15 20 20 20 F F F 11 220-239 15 15 15 20 20 20 F F F 12 240-259 14 14 14 18 18 18 F F F 13 260-279 15 15 15 20 20 20 F F F 14 280-299 13 13 13 17 17 17 F F F 15 300-319 16 16 16 21 21 21 F F F 16 320-339 16 16 16 21 21 21 F F F 17 340-359 6 6 6 8 8 8 F F F 18 360-379 8 8 8 12 12 12 F F F 19 380-399 16 16 16 22 22 22 F F F 20 400-419 16 16 16 22 22 22 F F F 21 420-439 13 13 13 17 17 17 F F F 22 440-459 14 14 14 18 18 18 F F F 23 460-479 17 17 17 23 23 23 F F F 24 480-499 17 17 17 23 23 23 F F F 25 500-519 16 16 16 21 21 21 F F F 26 520-539 17 17 17 23 23 23 F F F 27 540-559 17 17 17 23 23 23 F F F 28 560-579 13 13 13 17 17 17 F F F 29 580-599 15 15 15 20 20 20 F F F Max gen: 17, Max flush: 23 Begin loop 9 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:57.396610803Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1252,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:50:57.439079455Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1252,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:57.439442522Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1252,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:57.471207785Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1252,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:57.471823924Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1252} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"3818f0fc76fe787c6cc412af30fd59b930203885\",\n git_commit_timestamp: \"2024-11-01T16:08:30.000000000Z\",\n git_branch: \"renovate/hyper-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:57.471854488Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1252} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:57.47187228Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1252} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:57.471887052Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1252} {"msg":"Crucible 7bd9bef9-7e6b-4c35-b6b7-639174b66d21 has session id: b31b39b8-c1b4-489e-a3a1-c6a00ae394ec","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:57.472312517Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1252,"session_id":"b31b39b8-c1b4-489e-a3a1-c6a00ae394ec"} {"msg":"Upstairs opts: Upstairs UUID: 7bd9bef9-7e6b-4c35-b6b7-639174b66d21, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:57.472351962Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1252,"session_id":"b31b39b8-c1b4-489e-a3a1-c6a00ae394ec"} {"msg":"Crucible stats registered with UUID: 7bd9bef9-7e6b-4c35-b6b7-639174b66d21","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:57.472377495Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1252,"session_id":"b31b39b8-c1b4-489e-a3a1-c6a00ae394ec"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:57.50298825Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1252,"task":"crutest"} {"msg":"The guest has requested activation with gen:18","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:57.503018394Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1252} {"msg":"7bd9bef9-7e6b-4c35-b6b7-639174b66d21 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:57.503095373Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1252,"session_id":"b31b39b8-c1b4-489e-a3a1-c6a00ae394ec"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:57.503118376Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1252,"client":"0","":"downstairs","session_id":"b31b39b8-c1b4-489e-a3a1-c6a00ae394ec"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:57.50314899Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1252,"client":"0","":"downstairs","session_id":"b31b39b8-c1b4-489e-a3a1-c6a00ae394ec"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:57.503166753Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1252,"client":"1","":"downstairs","session_id":"b31b39b8-c1b4-489e-a3a1-c6a00ae394ec"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:57.503189905Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1252,"client":"1","":"downstairs","session_id":"b31b39b8-c1b4-489e-a3a1-c6a00ae394ec"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:57.503220849Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1252,"client":"2","":"downstairs","session_id":"b31b39b8-c1b4-489e-a3a1-c6a00ae394ec"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:57.503252664Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1252,"client":"2","":"downstairs","session_id":"b31b39b8-c1b4-489e-a3a1-c6a00ae394ec"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:57.503289208Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1252,"":"io task","client":"2","":"downstairs","session_id":"b31b39b8-c1b4-489e-a3a1-c6a00ae394ec"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:57.503323133Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1252,"":"io task","client":"0","":"downstairs","session_id":"b31b39b8-c1b4-489e-a3a1-c6a00ae394ec"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:57.503353497Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1252,"":"io task","client":"1","":"downstairs","session_id":"b31b39b8-c1b4-489e-a3a1-c6a00ae394ec"} {"msg":"ds_connection connected from Ok(127.0.0.1:48524)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:57.503384001Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1252,"":"io task","client":"2","":"downstairs","session_id":"b31b39b8-c1b4-489e-a3a1-c6a00ae394ec"} {"msg":"ds_connection connected from Ok(127.0.0.1:57920)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:57.503415585Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1252,"":"io task","client":"1","":"downstairs","session_id":"b31b39b8-c1b4-489e-a3a1-c6a00ae394ec"} {"msg":"ds_connection connected from Ok(127.0.0.1:58976)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:57.503550242Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1252,"":"io task","client":"0","":"downstairs","session_id":"b31b39b8-c1b4-489e-a3a1-c6a00ae394ec"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:57.504206216Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1252,"client":"2","":"downstairs","session_id":"b31b39b8-c1b4-489e-a3a1-c6a00ae394ec"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:57.5042342Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1252,"client":"2","":"downstairs","session_id":"b31b39b8-c1b4-489e-a3a1-c6a00ae394ec"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:57.504249782Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1252,"client":"1","":"downstairs","session_id":"b31b39b8-c1b4-489e-a3a1-c6a00ae394ec"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:57.504263994Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1252,"client":"1","":"downstairs","session_id":"b31b39b8-c1b4-489e-a3a1-c6a00ae394ec"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:57.504410123Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1252,"client":"0","":"downstairs","session_id":"b31b39b8-c1b4-489e-a3a1-c6a00ae394ec"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:57.504439297Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1252,"client":"0","":"downstairs","session_id":"b31b39b8-c1b4-489e-a3a1-c6a00ae394ec"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:57.50485582Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1252,"client":"2","":"downstairs","session_id":"b31b39b8-c1b4-489e-a3a1-c6a00ae394ec"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:57.504980766Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1252,"client":"1","":"downstairs","session_id":"b31b39b8-c1b4-489e-a3a1-c6a00ae394ec"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:57.505019871Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1252,"client":"0","":"downstairs","session_id":"b31b39b8-c1b4-489e-a3a1-c6a00ae394ec"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:57.505488942Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1252,"client":"2","":"downstairs","session_id":"b31b39b8-c1b4-489e-a3a1-c6a00ae394ec"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:57.505517565Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1252,"client":"2","":"downstairs","session_id":"b31b39b8-c1b4-489e-a3a1-c6a00ae394ec"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:57.505533167Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1252,"session_id":"b31b39b8-c1b4-489e-a3a1-c6a00ae394ec"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:57.505561271Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1252,"client":"1","":"downstairs","session_id":"b31b39b8-c1b4-489e-a3a1-c6a00ae394ec"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:57.505575493Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1252,"client":"1","":"downstairs","session_id":"b31b39b8-c1b4-489e-a3a1-c6a00ae394ec"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:57.505588845Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1252,"session_id":"b31b39b8-c1b4-489e-a3a1-c6a00ae394ec"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:57.505607477Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1252,"client":"0","":"downstairs","session_id":"b31b39b8-c1b4-489e-a3a1-c6a00ae394ec"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:57.505620779Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1252,"client":"0","":"downstairs","session_id":"b31b39b8-c1b4-489e-a3a1-c6a00ae394ec"} {"msg":"[0]R flush_numbers[0..12]: [12, 23, 23, 21, 21, 21, 21, 14, 21, 23, 20, 20]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:57.505645112Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1252,"":"downstairs","":"downstairs","session_id":"b31b39b8-c1b4-489e-a3a1-c6a00ae394ec"} {"msg":"[0]R generation[0..12]: [8, 17, 17, 16, 16, 16, 16, 10, 16, 17, 15, 15]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:57.505659254Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1252,"":"downstairs","":"downstairs","session_id":"b31b39b8-c1b4-489e-a3a1-c6a00ae394ec"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:57.505673706Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1252,"":"downstairs","":"downstairs","session_id":"b31b39b8-c1b4-489e-a3a1-c6a00ae394ec"} {"msg":"[1]R flush_numbers[0..12]: [12, 23, 23, 21, 21, 21, 21, 14, 21, 23, 20, 20]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:57.505687537Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1252,"":"downstairs","":"downstairs","session_id":"b31b39b8-c1b4-489e-a3a1-c6a00ae394ec"} {"msg":"[1]R generation[0..12]: [8, 17, 17, 16, 16, 16, 16, 10, 16, 17, 15, 15]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:57.505701069Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1252,"":"downstairs","":"downstairs","session_id":"b31b39b8-c1b4-489e-a3a1-c6a00ae394ec"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:57.505714771Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1252,"":"downstairs","":"downstairs","session_id":"b31b39b8-c1b4-489e-a3a1-c6a00ae394ec"} {"msg":"[2]R flush_numbers[0..12]: [12, 23, 23, 21, 21, 21, 21, 14, 21, 23, 20, 20]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:57.505732423Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1252,"":"downstairs","":"downstairs","session_id":"b31b39b8-c1b4-489e-a3a1-c6a00ae394ec"} {"msg":"[2]R generation[0..12]: [8, 17, 17, 16, 16, 16, 16, 10, 16, 17, 15, 15]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:57.505746805Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1252,"":"downstairs","":"downstairs","session_id":"b31b39b8-c1b4-489e-a3a1-c6a00ae394ec"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:57.505761017Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1252,"":"downstairs","":"downstairs","session_id":"b31b39b8-c1b4-489e-a3a1-c6a00ae394ec"} {"msg":"Max found gen is 18","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:57.505774369Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1252,"":"downstairs","":"downstairs","session_id":"b31b39b8-c1b4-489e-a3a1-c6a00ae394ec"} {"msg":"Generation requested: 18 >= found:18","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:57.5057879Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1252,"":"downstairs","":"downstairs","session_id":"b31b39b8-c1b4-489e-a3a1-c6a00ae394ec"} {"msg":"Next flush: 24","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:57.505801002Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1252,"":"downstairs","":"downstairs","session_id":"b31b39b8-c1b4-489e-a3a1-c6a00ae394ec"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:57.505814174Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1252,"":"downstairs","":"downstairs","session_id":"b31b39b8-c1b4-489e-a3a1-c6a00ae394ec"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:57.505827275Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1252,"session_id":"b31b39b8-c1b4-489e-a3a1-c6a00ae394ec"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:57.505839097Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1252,"client":"0","":"downstairs","session_id":"b31b39b8-c1b4-489e-a3a1-c6a00ae394ec"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:57.505852028Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1252,"client":"1","":"downstairs","session_id":Wait for a query_work_queue command to finish before sending IO "b31b39b8-c1b4-489e-a3a1-c6a00ae394ec"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:57.505876782Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1252,"client":"2","":"downstairs","session_id":"b31b39b8-c1b4-489e-a3a1-c6a00ae394ec"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:57.505890104Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1252,"session_id":"b31b39b8-c1b4-489e-a3a1-c6a00ae394ec"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:57.505902305Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1252,"session_id":"b31b39b8-c1b4-489e-a3a1-c6a00ae394ec"} {"msg":"7bd9bef9-7e6b-4c35-b6b7-639174b66d21 is now active with session: b31b39b8-c1b4-489e-a3a1-c6a00ae394ec","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:57.505915007Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1252,"session_id":"b31b39b8-c1b4-489e-a3a1-c6a00ae394ec"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:57.505927018Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1252,"session_id":"b31b39b8-c1b4-489e-a3a1-c6a00ae394ec"} {"msg":"The guest has finished waiting for activation with:18","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:50:57.50593912Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1252} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 85 len 4096 data: 3 02/30 Write block 273 len 12288 data: 3 4 4 03/30 Flush 04/30 Write block 293 len 32768 data: 2 2 2 2 3 2 2 2 05/30 Read block 3 len 20480 06/30 Write block 497 len 24576 data: 2 2 2 2 2 2 07/30 Read block 547 len 40960 08/30 Write block 23 len 24576 data: 2 2 2 2 2 2 09/30 Read block 23 len 28672 10/30 Write block 5 len 8192 data: 2 3 11/30 Write block 574 len 36864 data: 2 2 2 2 2 3 3 3 3 12/30 Read block 280 len 8192 13/30 Write block 120 len 16384 data: 3 3 3 3 14/30 Write block 149 len 4096 data: 4 15/30 Write block 1 len 12288 data: 2 2 2 16/30 Flush 17/30 Write block 399 len 16384 data: 7 6 5 5 18/30 Read block 304 len 4096 19/30 Flush 20/30 Write block 66 len 40960 data: 7 6 5 5 6 5 4 4 3 2 21/30 Flush 22/30 Write block 551 len 8192 data: 4 3 23/30 Write block 201 len 8192 data: 3 3 24/30 Read block 323 len 32768 25/30 Write block 205 len 12288 data: 5 5 5 26/30 Read block 156 len 40960 27/30 Read block 190 len 12288 28/30 Read block 299 len 12288 29/30 Read block 281 len 32768 30/30 Write block 461 len 16384 data: 2 2 2 2 ---------------------------------------------------------------- Crucible gen:18 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 2 Done Done Done false 1022 Acked Write 2 Done Done Done false 1023 Acked Read 8 Done Done Done false 1024 Acked Write 3 Done Done Done false 1025 Acked Read 10 Done Done Done false 1026 Acked Read 3 Done Done Done false 1027 Acked Read 3 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 8 8 8 24 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1020 1020 1020 Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:50:57.530749136Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1252} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 18 18 18 25 25 25 F F F 1 020-039 18 18 18 25 25 25 F F F 2 040-059 17 17 17 23 23 23 F F F 3 060-079 18 18 18 27 27 27 F F F 4 080-099 18 18 18 24 24 24 F F F 5 100-119 16 16 16 21 21 21 F F F 6 120-139 18 18 18 25 25 25 F F F 7 140-159 18 18 18 25 25 25 F F F 8 160-179 16 16 16 21 21 21 F F F 9 180-199 17 17 17 23 23 23 F F F 10 200-219 15 15 15 20 20 20 T T T 11 220-239 15 15 15 20 20 20 F F F 12 240-259 14 14 14 18 18 18 F F F 13 260-279 18 18 18 24 24 24 F F F 14 280-299 18 18 18 25 25 25 F F F 15 300-319 18 18 18 25 25 25 F F F 16 320-339 16 16 16 21 21 21 F F F 17 340-359 6 6 6 8 8 8 F F F 18 360-379 8 8 8 12 12 12 F F F 19 380-399 18 18 18 26 26 26 F F F 20 400-419 18 18 18 26 26 26 F F F 21 420-439 13 13 13 17 17 17 F F F 22 440-459 14 14 14 18 18 18 F F F 23 460-479 17 17 17 23 23 23 T T T 24 480-499 18 18 18 25 25 25 F F F 25 500-519 18 18 18 25 25 25 F F F 26 520-539 17 17 17 23 23 23 F F F 27 540-559 17 17 17 23 23 23 T T T 28 560-579 18 18 18 25 25 25 F F F 29 580-599 18 18 18 25 25 25 F F F Max gen: 18, Max flush: 27 A Difference in extent metadata is expected here On loop 9 Verifying data now No change after verify Loop: 9 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 18 18 18 25 25 25 F F F 1 020-039 18 18 18 25 25 25 F F F 2 040-059 17 17 17 23 23 23 F F F 3 060-079 18 18 18 27 27 27 F F F 4 080-099 18 18 18 24 24 24 F F F 5 100-119 16 16 16 21 21 21 F F F 6 120-139 18 18 18 25 25 25 F F F 7 140-159 18 18 18 25 25 25 F F F 8 160-179 16 16 16 21 21 21 F F F 9 180-199 17 17 17 23 23 23 F F F 10 200-219 19 19 19 28 28 28 F F F 11 220-239 15 15 15 20 20 20 F F F 12 240-259 14 14 14 18 18 18 F F F 13 260-279 18 18 18 24 24 24 F F F 14 280-299 18 18 18 25 25 25 F F F 15 300-319 18 18 18 25 25 25 F F F 16 320-339 16 16 16 21 21 21 F F F 17 340-359 6 6 6 8 8 8 F F F 18 360-379 8 8 8 12 12 12 F F F 19 380-399 18 18 18 26 26 26 F F F 20 400-419 18 18 18 26 26 26 F F F 21 420-439 13 13 13 17 17 17 F F F 22 440-459 14 14 14 18 18 18 F F F 23 460-479 19 19 19 28 28 28 F F F 24 480-499 18 18 18 25 25 25 F F F 25 500-519 18 18 18 25 25 25 F F F 26 520-539 17 17 17 23 23 23 F F F 27 540-559 19 19 19 28 28 28 F F F 28 560-579 18 18 18 25 25 25 F F F 29 580-599 18 18 18 25 25 25 F F F Max gen: 19, Max flush: 28 Begin loop 10 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:01.841689048Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1260,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:51:01.885235799Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1260,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:01.885665575Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1260,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:01.918214129Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1260,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:01.918771671Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1260} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"3818f0fc76fe787c6cc412af30fd59b930203885\",\n git_commit_timestamp: \"2024-11-01T16:08:30.000000000Z\",\n git_branch: \"renovate/hyper-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:01.918814716Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1260} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:01.918832098Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1260} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:01.918854951Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1260} {"msg":"Crucible 3fc15bc5-fad3-4f4a-9b4e-ec20b7dc7472 has session id: 93a3c7c9-139f-4b5b-91c7-68926e7c6adf","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:01.919288637Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1260,"session_id":"93a3c7c9-139f-4b5b-91c7-68926e7c6adf"} {"msg":"Upstairs opts: Upstairs UUID: 3fc15bc5-fad3-4f4a-9b4e-ec20b7dc7472, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:01.919342374Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1260,"session_id":"93a3c7c9-139f-4b5b-91c7-68926e7c6adf"} {"msg":"Crucible stats registered with UUID: 3fc15bc5-fad3-4f4a-9b4e-ec20b7dc7472","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:01.919371558Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1260,"session_id":"93a3c7c9-139f-4b5b-91c7-68926e7c6adf"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:01.950781085Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1260,"task":"crutest"} {"msg":"The guest has requested activation with gen:20","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:01.950810969Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1260} {"msg":"3fc15bc5-fad3-4f4a-9b4e-ec20b7dc7472 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:01.950885388Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1260,"session_id":"93a3c7c9-139f-4b5b-91c7-68926e7c6adf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:01.950906971Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1260,"client":"0","":"downstairs","session_id":"93a3c7c9-139f-4b5b-91c7-68926e7c6adf"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:01.950936245Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1260,"client":"0","":"downstairs","session_id":"93a3c7c9-139f-4b5b-91c7-68926e7c6adf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:01.950954377Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1260,"client":"1","":"downstairs","session_id":"93a3c7c9-139f-4b5b-91c7-68926e7c6adf"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:01.950971489Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1260,"client":"1","":"downstairs","session_id":"93a3c7c9-139f-4b5b-91c7-68926e7c6adf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:01.950988102Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1260,"client":"2","":"downstairs","session_id":"93a3c7c9-139f-4b5b-91c7-68926e7c6adf"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:01.951005194Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1260,"client":"2","":"downstairs","session_id":"93a3c7c9-139f-4b5b-91c7-68926e7c6adf"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:01.951021576Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1260,"":"io task","client":"2","":"downstairs","session_id":"93a3c7c9-139f-4b5b-91c7-68926e7c6adf"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:01.951039058Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1260,"":"io task","client":"0","":"downstairs","session_id":"93a3c7c9-139f-4b5b-91c7-68926e7c6adf"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:01.951060001Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1260,"":"io task","client":"1","":"downstairs","session_id":"93a3c7c9-139f-4b5b-91c7-68926e7c6adf"} {"msg":"ds_connection connected from Ok(127.0.0.1:34726)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:01.951086164Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1260,"":"io task","client":"2","":"downstairs","session_id":"93a3c7c9-139f-4b5b-91c7-68926e7c6adf"} {"msg":"ds_connection connected from Ok(127.0.0.1:39539)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:01.951276809Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1260,"":"io task","client":"1","":"downstairs","session_id":"93a3c7c9-139f-4b5b-91c7-68926e7c6adf"} {"msg":"ds_connection connected from Ok(127.0.0.1:34625)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:01.951413036Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1260,"":"io task","client":"0","":"downstairs","session_id":"93a3c7c9-139f-4b5b-91c7-68926e7c6adf"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:01.951918791Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1260,"client":"1","":"downstairs","session_id":"93a3c7c9-139f-4b5b-91c7-68926e7c6adf"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:01.951946155Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1260,"client":"1","":"downstairs","session_id":"93a3c7c9-139f-4b5b-91c7-68926e7c6adf"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:01.951961457Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1260,"client":"2","":"downstairs","session_id":"93a3c7c9-139f-4b5b-91c7-68926e7c6adf"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:01.951975219Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1260,"client":"2","":"downstairs","session_id":"93a3c7c9-139f-4b5b-91c7-68926e7c6adf"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:01.951989091Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1260,"client":"0","":"downstairs","session_id":"93a3c7c9-139f-4b5b-91c7-68926e7c6adf"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:01.952002692Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1260,"client":"0","":"downstairs","session_id":"93a3c7c9-139f-4b5b-91c7-68926e7c6adf"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:01.952564105Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1260,"client":"1","":"downstairs","session_id":"93a3c7c9-139f-4b5b-91c7-68926e7c6adf"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:01.952730456Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1260,"client":"2","":"downstairs","session_id":"93a3c7c9-139f-4b5b-91c7-68926e7c6adf"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:01.952887736Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1260,"client":"0","":"downstairs","session_id":"93a3c7c9-139f-4b5b-91c7-68926e7c6adf"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:01.953209848Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1260,"client":"1","":"downstairs","session_id":"93a3c7c9-139f-4b5b-91c7-68926e7c6adf"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:01.9532288Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1260,"client":"1","":"downstairs","session_id":"93a3c7c9-139f-4b5b-91c7-68926e7c6adf"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:01.953245852Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1260,"session_id":"93a3c7c9-139f-4b5b-91c7-68926e7c6adf"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:01.95330593Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1260,"client":"2","":"downstairs","session_id":"93a3c7c9-139f-4b5b-91c7-68926e7c6adf"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:01.953347635Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1260,"client":"2","":"downstairs","session_id":"93a3c7c9-139f-4b5b-91c7-68926e7c6adf"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:01.953376299Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1260,"session_id":"93a3c7c9-139f-4b5b-91c7-68926e7c6adf"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:01.953402353Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1260,"client":"0","":"downstairs","session_id":"93a3c7c9-139f-4b5b-91c7-68926e7c6adf"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:01.953426676Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1260,"client":"0","":"downstairs","session_id":"93a3c7c9-139f-4b5b-91c7-68926e7c6adf"} {"msg":"[0]R flush_numbers[0..12]: [25, 25, 23, 27, 24, 21, 25, 25, 21, 23, 28, 20]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:01.953451169Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1260,"":"downstairs","":"downstairs","session_id":"93a3c7c9-139f-4b5b-91c7-68926e7c6adf"} {"msg":"[0]R generation[0..12]: [18, 18, 17, 18, 18, 16, 18, 18, 16, 17, 19, 15]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:01.953475442Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1260,"":"downstairs","":"downstairs","session_id":"93a3c7c9-139f-4b5b-91c7-68926e7c6adf"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:01.953501125Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1260,"":"downstairs","":"downstairs","session_id":"93a3c7c9-139f-4b5b-91c7-68926e7c6adf"} {"msg":"[1]R flush_numbers[0..12]: [25, 25, 23, 27, 24, 21, 25, 25, 21, 23, 28, 20]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:01.953525409Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1260,"":"downstairs","":"downstairs","session_id":"93a3c7c9-139f-4b5b-91c7-68926e7c6adf"} {"msg":"[1]R generation[0..12]: [18, 18, 17, 18, 18, 16, 18, 18, 16, 17, 19, 15]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:01.953548612Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1260,"":"downstairs","":"downstairs","session_id":"93a3c7c9-139f-4b5b-91c7-68926e7c6adf"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:01.953572624Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1260,"":"downstairs","":"downstairs","session_id":"93a3c7c9-139f-4b5b-91c7-68926e7c6adf"} {"msg":"[2]R flush_numbers[0..12]: [25, 25, 23, 27, 24, 21, 25, 25, 21, 23, 28, 20]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:01.953597748Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1260,"":"downstairs","":"downstairs","session_id":"93a3c7c9-139f-4b5b-91c7-68926e7c6adf"} {"msg":"[2]R generation[0..12]: [18, 18, 17, 18, 18, 16, 18, 18, 16, 17, 19, 15]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:01.953624061Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1260,"":"downstairs","":"downstairs","session_id":"93a3c7c9-139f-4b5b-91c7-68926e7c6adf"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:01.953653915Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1260,"":"downstairs","":"downstairs","session_id":"93a3c7c9-139f-4b5b-91c7-68926e7c6adf"} {"msg":"Max found gen is 20","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:01.953680738Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1260,"":"downstairs","":"downstairs","session_id":"93a3c7c9-139f-4b5b-91c7-68926e7c6adf"} {"msg":"Generation requested: 20 >= found:20","v":0,"name":"crucible","Wait for a query_work_queue command to finish before sending IO level":30,"time":"2024-11-01T16:51:01.953716413Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1260,"":"downstairs","":"downstairs","session_id":"93a3c7c9-139f-4b5b-91c7-68926e7c6adf"} {"msg":"Next flush: 29","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:01.953744987Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1260,"":"downstairs","":"downstairs","session_id":"93a3c7c9-139f-4b5b-91c7-68926e7c6adf"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:01.95377204Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1260,"":"downstairs","":"downstairs","session_id":"93a3c7c9-139f-4b5b-91c7-68926e7c6adf"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:01.953799524Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1260,"session_id":"93a3c7c9-139f-4b5b-91c7-68926e7c6adf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:01.953825157Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1260,"client":"0","":"downstairs","session_id":"93a3c7c9-139f-4b5b-91c7-68926e7c6adf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:01.95385228Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1260,"client":"1","":"downstairs","session_id":"93a3c7c9-139f-4b5b-91c7-68926e7c6adf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 time":"2024-11-01T16:51:01.953879584Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1260,"client":"2","":"downstairs","session_id":"93a3c7c9-139f-4b5b-91c7-68926e7c6adf"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:01.953913078Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1260,"session_id":"93a3c7c9-139f-4b5b-91c7-68926e7c6adf"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:01.953933161Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1260,"session_id":"93a3c7c9-139f-4b5b-91c7-68926e7c6adf"} {"msg":"3fc15bc5-fad3-4f4a-9b4e-ec20b7dc7472 is now active with session: 93a3c7c9-139f-4b5b-91c7-68926e7c6adf","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:01.953947253Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1260,"session_id":"93a3c7c9-139f-4b5b-91c7-68926e7c6adf"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:01.953959724Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1260,"session_id":"93a3c7c9-139f-4b5b-91c7-68926e7c6adf"} {"msg":"The guest has finished waiting for activation with:20","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:01.953972186Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1260} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 19 len 8192 data: 2 2 02/30 Read block 109 len 4096 03/30 Read block 444 len 20480 04/30 Write block 261 len 4096 data: 3 05/30 Flush 06/30 Flush 07/30 Write block 264 len 20480 data: 3 3 3 2 3 08/30 Write block 315 len 32768 data: 4 5 4 5 5 5 5 5 09/30 Read block 417 len 8192 10/30 Read block 451 len 8192 11/30 Read block 133 len 12288 12/30 Read block 257 len 4096 13/30 Read block 588 len 36864 14/30 Write block 491 len 4096 data: 4 15/30 Read block 45 len 28672 16/30 Read block 483 len 28672 17/30 Flush 18/30 Write block 246 len 40960 data: 3 5 5 5 6 5 4 4 2 2 19/30 Read block 565 len 4096 20/30 Flush 21/30 Write block 512 len 8192 data: 3 2 22/30 Write block 417 len 20480 data: 3 5 5 5 5 23/30 Write block 256 len 4096 data: 2 24/30 Write block 571 len 40960 data: 3 3 2 3 3 3 3 3 4 4 25/30 Write block 66 len 28672 data: 8 7 6 6 7 6 5 26/30 Write block 271 len 16384 data: 4 3 4 5 27/30 Read block 243 len 32768 28/30 Read block 567 len 24576 29/30 Read block 556 len 20480 30/30 Read block 27 len 8192 ---------------------------------------------------------------- Crucible gen:20 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1020 Acked Write 2 Done Done Done false 1021 Acked Write 5 Done Done Done false 1022 Acked Write 1 Done Done Done false 1023 Acked Write 10 Done Done Done false 1024 Acked Write 7 Done Sent Done false 1025 Acked Write 4 Done Sent Sent false 1026 Acked Read 8 Done Sent Sent false 1027 Acked Read 6 Done Sent Sent false 1028 Acked Read 5 Done Sent Sent false 1029 Acked Read 2 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 6 5 11 Done 10 4 5 19 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1019 1019 1019 Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:51:01.976837482Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1260} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 20 20 20 29 29 29 F F F 1 020-039 20 20 20 29 29 29 F F F 2 040-059 17 17 17 23 23 23 F F F 3 060-079 18 18 18 27 27 27 T T T 4 080-099 18 18 18 24 24 24 F F F 5 100-119 16 16 16 21 21 21 F F F 6 120-139 18 18 18 25 25 25 F F F 7 140-159 18 18 18 25 25 25 F F F 8 160-179 16 16 16 21 21 21 F F F 9 180-199 17 17 17 23 23 23 F F F 10 200-219 19 19 19 28 28 28 F F F 11 220-239 15 15 15 20 20 20 F F F 12 240-259 20 20 20 32 32 32 T T T 13 260-279 20 20 20 31 31 31 T T T 14 280-299 18 18 18 25 25 25 F F F 15 300-319 20 20 20 31 31 31 F F F 16 320-339 20 20 20 31 31 31 F F F 17 340-359 6 6 6 8 8 8 F F F 18 360-379 8 8 8 12 12 12 F F F 19 380-399 18 18 18 26 26 26 F F F 20 400-419 18 18 18 26 26 26 T T T 21 420-439 13 13 13 17 17 17 T T T 22 440-459 14 14 14 18 18 18 F F F 23 460-479 19 19 19 28 28 28 F F F 24 480-499 20 20 20 31 31 31 F F F 25 500-519 18 18 18 25 25 25 T T T 26 520-539 17 17 17 23 23 23 F F F 27 540-559 19 19 19 28 28 28 F F F 28 560-579 18 18 18 25 25 25 T T T 29 580-599 18 18 18 25 25 25 T T T Max gen: 20, Max flush: 32 A Difference in extent metadata is expected here On loop 10 Verifying data now No change after verify Loop: 10 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 20 20 20 29 29 29 F F F 1 020-039 20 20 20 29 29 29 F F F 2 040-059 17 17 17 23 23 23 F F F 3 060-079 21 21 21 33 33 33 F F F 4 080-099 18 18 18 24 24 24 F F F 5 100-119 16 16 16 21 21 21 F F F 6 120-139 18 18 18 25 25 25 F F F 7 140-159 18 18 18 25 25 25 F F F 8 160-179 16 16 16 21 21 21 F F F 9 180-199 17 17 17 23 23 23 F F F 10 200-219 19 19 19 28 28 28 F F F 11 220-239 15 15 15 20 20 20 F F F 12 240-259 21 21 21 33 33 33 F F F 13 260-279 21 21 21 33 33 33 F F F 14 280-299 18 18 18 25 25 25 F F F 15 300-319 20 20 20 31 31 31 F F F 16 320-339 20 20 20 31 31 31 F F F 17 340-359 6 6 6 8 8 8 F F F 18 360-379 8 8 8 12 12 12 F F F 19 380-399 18 18 18 26 26 26 F F F 20 400-419 21 21 21 33 33 33 F F F 21 420-439 21 21 21 33 33 33 F F F 22 440-459 14 14 14 18 18 18 F F F 23 460-479 19 19 19 28 28 28 F F F 24 480-499 20 20 20 31 31 31 F F F 25 500-519 21 21 21 33 33 33 F F F 26 520-539 17 17 17 23 23 23 F F F 27 540-559 19 19 19 28 28 28 F F F 28 560-579 21 21 21 33 33 33 F F F 29 580-599 21 21 21 33 33 33 F F F Max gen: 21, Max flush: 33 Begin loop 11 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:06.344211217Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1268,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:06.388226308Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1268,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:06.388813184Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1268} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"3818f0fc76fe787c6cc412af30fd59b930203885\",\n git_commit_timestamp: \"2024-11-01T16:08:30.000000000Z\",\n git_branch: \"renovate/hyper-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:06.388853069Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1268} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:06.388869371Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1268} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:06.388884463Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1268} {"msg":"Crucible 1af2ef72-9910-4eef-b7eb-022011f657d4 has session id: b5c47d11-2472-4043-9283-ae81daa6bd3d","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:06.389317439Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1268,"session_id":"b5c47d11-2472-4043-9283-ae81daa6bd3d"} {"msg":"Upstairs opts: Upstairs UUID: 1af2ef72-9910-4eef-b7eb-022011f657d4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:06.389364515Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1268,"session_id":"b5c47d11-2472-4043-9283-ae81daa6bd3d"} {"msg":"Crucible stats registered with UUID: 1af2ef72-9910-4eef-b7eb-022011f657d4","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:06.389397099Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1268,"session_id":"b5c47d11-2472-4043-9283-ae81daa6bd3d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:06.420324424Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1268,"task":"crutest"} {"msg":"The guest has requested activation with gen:22","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:06.420355168Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1268} {"msg":"1af2ef72-9910-4eef-b7eb-022011f657d4 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:06.420440809Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1268,"session_id":"b5c47d11-2472-4043-9283-ae81daa6bd3d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:06.420463542Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1268,"client":"0","":"downstairs","session_id":"b5c47d11-2472-4043-9283-ae81daa6bd3d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:06.420491576Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1268,"client":"0","":"downstairs","session_id":"b5c47d11-2472-4043-9283-ae81daa6bd3d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:06.420509148Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1268,"client":"1","":"downstairs","session_id":"b5c47d11-2472-4043-9283-ae81daa6bd3d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:06.42052599Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1268,"client":"1","":"downstairs","session_id":"b5c47d11-2472-4043-9283-ae81daa6bd3d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:06.420542992Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1268,"client":"2","":"downstairs","session_id":"b5c47d11-2472-4043-9283-ae81daa6bd3d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:06.420570946Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1268,"client":"2","":"downstairs","session_id":"b5c47d11-2472-4043-9283-ae81daa6bd3d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:06.420599929Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1268,"":"io task","client":"2","":"downstairs","session_id":"b5c47d11-2472-4043-9283-ae81daa6bd3d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:06.420629993Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1268,"":"io task","client":"0","":"downstairs","session_id":"b5c47d11-2472-4043-9283-ae81daa6bd3d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:06.420661168Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1268,"":"io task","client":"1","":"downstairs","session_id":"b5c47d11-2472-4043-9283-ae81daa6bd3d"} {"msg":"ds_connection connected from Ok(127.0.0.1:64523)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:06.420695292Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1268,"":"io task","client":"2","":"downstairs","session_id":"b5c47d11-2472-4043-9283-ae81daa6bd3d"} {"msg":"ds_connection connected from Ok(127.0.0.1:39750)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:06.420883286Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1268,"":"io task","client":"1","":"downstairs","session_id":"b5c47d11-2472-4043-9283-ae81daa6bd3d"} {"msg":"ds_connection connected from Ok(127.0.0.1:56399)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:06.420931822Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1268,"":"io task","client":"0","":"downstairs","session_id":"b5c47d11-2472-4043-9283-ae81daa6bd3d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:06.421468541Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1268,"client":"0","":"downstairs","session_id":"b5c47d11-2472-4043-9283-ae81daa6bd3d"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:06.421494225Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1268,"client":"0","":"downstairs","session_id":"b5c47d11-2472-4043-9283-ae81daa6bd3d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:06.421629212Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1268,"client":"1","":"downstairs","session_id":"b5c47d11-2472-4043-9283-ae81daa6bd3d"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:06.421677868Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1268,"client":"1","":"downstairs","session_id":"b5c47d11-2472-4043-9283-ae81daa6bd3d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:06.421704742Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1268,"client":"2","":"downstairs","session_id":"b5c47d11-2472-4043-9283-ae81daa6bd3d"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:06.421731045Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1268,"client":"2","":"downstairs","session_id":"b5c47d11-2472-4043-9283-ae81daa6bd3d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:06.422092982Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1268,"client":"0","":"downstairs","session_id":"b5c47d11-2472-4043-9283-ae81daa6bd3d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:06.422237451Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1268,"client":"1","":"downstairs","session_id":"b5c47d11-2472-4043-9283-ae81daa6bd3d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:06.422328712Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1268,"client":"2","":"downstairs","session_id":"b5c47d11-2472-4043-9283-ae81daa6bd3d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:06.422708031Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1268,"client":"0","":"downstairs","session_id":"b5c47d11-2472-4043-9283-ae81daa6bd3d"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:06.422732264Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1268,"client":"0","":"downstairs","session_id":"b5c47d11-2472-4043-9283-ae81daa6bd3d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:06.422746376Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1268,"session_id":"b5c47d11-2472-4043-9283-ae81daa6bd3d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:06.422892775Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1268,"client":"1","":"downstairs","session_id":"b5c47d11-2472-4043-9283-ae81daa6bd3d"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:06.422922339Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1268,"client":"1","":"downstairs","session_id":"b5c47d11-2472-4043-9283-ae81daa6bd3d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:06.422939491Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1268,"session_id":"b5c47d11-2472-4043-9283-ae81daa6bd3d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:06.422956583Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1268,"client":"2","":"downstairs","session_id":"b5c47d11-2472-4043-9283-ae81daa6bd3d"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:06.422971665Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1268,"client":"2","":"downstairs","session_id":"b5c47d11-2472-4043-9283-ae81daa6bd3d"} {"msg":"[0]R flush_numbers[0..12]: [29, 29, 23, 33, 24, 21, 25, 25, 21, 23, 28, 20]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:06.422985987Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1268,"":"downstairs","":"downstairs","session_id":"b5c47d11-2472-4043-9283-ae81daa6bd3d"} {"msg":"[0]R generation[0..12]: [20, 20, 17, 21, 18, 16, 18, 18, 16, 17, 19, 15]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:06.423015751Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1268,"":"downstairs","":"downstairs","session_id":"b5c47d11-2472-4043-9283-ae81daa6bd3d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:06.423031393Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1268,"":"downstairs","":"downstairs","session_id":"b5c47d11-2472-4043-9283-ae81daa6bd3d"} {"msg":"[1]R flush_numbers[0..12]: [29, 29, 23, 33, 24, 21, 25, 25, 21, 23, 28, 20]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:06.423045305Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1268,"":"downstairs","":"downstairs","session_id":"b5c47d11-2472-4043-9283-ae81daa6bd3d"} {"msg":"[1]R generation[0..12]: [20, 20, 17, 21, 18, 16, 18, 18, 16, 17, 19, 15]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:06.423059217Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1268,"":"downstairs","":"downstairs","session_id":"b5c47d11-2472-4043-9283-ae81daa6bd3d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:06.423074579Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1268,"":"downstairs","":"downstairs","session_id":"b5c47d11-2472-4043-9283-ae81daa6bd3d"} {"msg":"[2]R flush_numbers[0..12]: [29, 29, 23, 33, 24, 21, 25, 25, 21, 23, 28, 20]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:06.42308838Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1268,"":"downstairs","":"downstairs","session_id":"b5c47d11-2472-4043-9283-ae81daa6bd3d"} {"msg":"[2]R generation[0..12]: [20, 20, 17, 21, 18, 16, 18, 18, 16, 17, 19, 15]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:06.423103232Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1268,"":"downstairs","":"downstairs","session_id":"b5c47d11-2472-4043-9283-ae81daa6bd3d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:06.423117724Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1268,"":"downstairs","":"downstairs","session_id":"b5c47d11-2472-4043-9283-ae81daa6bd3d"} {"msg":"Max found gen is 22","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:06.423131156Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1268,"":"downstairs","":"downstairs","session_id":"b5c47d11-2472-4043-9283-ae81daa6bd3d"} {"msg":"Generation requested: 22 >= found:22","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:06.423144437Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","Wait for a query_work_queue command to finish before sending IO pid":1268,"":"downstairs","":"downstairs","session_id":"b5c47d11-2472-4043-9283-ae81daa6bd3d"} {"msg":"Next flush: 34","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:06.42316567Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1268,"":"downstairs","":"downstairs","session_id":"b5c47d11-2472-4043-9283-ae81daa6bd3d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:06.423181212Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1268,"":"downstairs","":"downstairs","session_id":"b5c47d11-2472-4043-9283-ae81daa6bd3d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:06.423194814Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1268,"session_id":"b5c47d11-2472-4043-9283-ae81daa6bd3d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:06.423206965Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1268,"client":"0","":"downstairs","session_id":"b5c47d11-2472-4043-9283-ae81daa6bd3d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:06.423220397Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1268,"client":"1","":"downstairs","session_id":"b5c47d11-2472-4043-9283-ae81daa6bd3d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:06.423233629Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1268,"client":"2","":"downstairs","session_id":"b5c47d11-2472-4043-9283-ae81daa6bd3d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:06.423246831Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1268,"session_id":"b5c47d11-2472-4043-9283-ae81daa6bd3d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:06.423259412Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1268,"session_id":"b5c47d11-2472-4043-9283-ae81daa6bd3d"} {"msg":"1af2ef72-9910-4eef-b7eb-022011f657d4 is now active with session: b5c47d11-2472-4043-9283-ae81daa6bd3d","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:06.423272394Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1268,"session_id":"b5c47d11-2472-4043-9283-ae81daa6bd3d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2024-11-01T16:51:06.423284546Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1268,"session_id":"b5c47d11-2472-4043-9283-ae81daa6bd3d"} {"msg":"The guest has finished waiting for activation with:22","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:06.423300337Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1268} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 527 len 36864 data: 2 3 3 4 4 4 4 3 2 03/30 Read block 514 len 20480 04/30 Read block 480 len 12288 05/30 Write block 468 len 16384 data: 2 3 4 4 06/30 Write block 389 len 4096 data: 2 07/30 Write block 393 len 32768 data: 3 3 4 6 7 7 8 7 08/30 Read block 217 len 20480 09/30 Read block 406 len 20480 10/30 Write block 406 len 40960 data: 2 2 2 2 2 2 2 2 2 2 11/30 Read block 98 len 36864 12/30 Write block 422 len 40960 data: 5 4 3 3 3 2 2 2 2 2 13/30 Read block 116 len 8192 14/30 Write block 590 len 32768 data: 2 2 2 3 3 4 3 3 15/30 Read block 169 len 4096 16/30 Write block 216 len 4096 data: 3 17/30 Write block 352 len 16384 data: 2 2 2 3 18/30 Read block 268 len 40960 19/30 Read block 142 len 20480 20/30 Write block 349 len 4096 data: 2 21/30 Write block 296 len 4096 data: 3 22/30 Write block 254 len 40960 data: 3 3 3 2 2 3 3 4 3 3 23/30 Read block 493 len 40960 24/30 Read block 268 len 28672 25/30 Read block 494 len 16384 26/30 Write block 356 len 8192 data: 3 3 27/30 Read block 317 len 16384 28/30 Read block 435 len 32768 29/30 Write block 348 len 28672 data: 3 3 2 2 3 3 3 30/30 Write block 443 len 36864 data: 6 6 6 5 5 4 4 5 5 ---------------------------------------------------------------- Crucible gen:22 GIO:true work queues: Upstairs:0 downstairs:29 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1001 Acked Write 9 Done Done Done false 1002 Acked Read 5 Done Done Done false 1003 Acked Read 3 Done Done Done false 1004 Acked Write 4 Done Done Done false 1005 Acked Write 1 Done Done Done false 1006 Acked Write 8 Done Done Done false 1007 Acked Read 5 Done Done Done false 1008 Acked Read 5 Done Done Done false 1009 Acked Write 10 Done Done Done false 1010 Acked Read 9 Done Done Done false 1011 Acked Write 10 Done Done Done false 1012 Acked Read 2 Done Done Done false 1013 Acked Write 8 Done Done Done false 1014 Acked Read 1 Done Done Done false 1015 Acked Write 1 Done Done Done false 1016 Acked Write 4 Done Done Done false 1017 Acked Read 10 Done Done Done false 1018 Acked Read 5 Done Done Done false 1019 Acked Write 1 Done Done Done false 1020 Acked Write 1 Done Done Done false 1021 Acked Write 10 Done Done Done false 1022 Acked Read 10 Done Done Done false 1023 Acked Read 7 Done Done Done false 1024 Acked Read 4 Done Done Done false 1025 Acked Write 2 Done Done Done false 1026 Acked Read 4 Done Done Done false 1027 Acked Read 8 Done Done Done false 1028 Acked Write 7 Sent Sent Sent false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 27 27 27 81 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1000 1000 1000 Downstairs last five completed: 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:51:06.443367803Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1268} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 20 20 20 29 29 29 F F F 1 020-039 20 20 20 29 29 29 F F F 2 040-059 17 17 17 23 23 23 F F F 3 060-079 21 21 21 33 33 33 F F F 4 080-099 18 18 18 24 24 24 F F F 5 100-119 16 16 16 21 21 21 F F F 6 120-139 18 18 18 25 25 25 F F F 7 140-159 18 18 18 25 25 25 F F F 8 160-179 16 16 16 21 21 21 F F F 9 180-199 17 17 17 23 23 23 F F F 10 200-219 19 19 19 28 28 28 T T T 11 220-239 15 15 15 20 20 20 F F F 12 240-259 21 21 21 33 33 33 T T T 13 260-279 21 21 21 33 33 33 T T T 14 280-299 18 18 18 25 25 25 T T T 15 300-319 20 20 20 31 31 31 F F F 16 320-339 20 20 20 31 31 31 F F F 17 340-359 6 6 6 8 8 8 T T T 18 360-379 8 8 8 12 12 12 F F F 19 380-399 18 18 18 26 26 26 T T T 20 400-419 21 21 21 33 33 33 T T T 21 420-439 21 21 21 33 33 33 T T T 22 440-459 14 14 14 18 18 18 T T T 23 460-479 19 19 19 28 28 28 T T T 24 480-499 20 20 20 31 31 31 F F F 25 500-519 21 21 21 33 33 33 F F F 26 520-539 17 17 17 23 23 23 T T T 27 540-559 19 19 19 28 28 28 F F F 28 560-579 21 21 21 33 33 33 F F F 29 580-599 21 21 21 33 33 33 T T T Max gen: 21, Max flush: 33 A Difference in extent metadata is expected here On loop 11 Verifying data now No change after verify Loop: 11 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 20 20 20 29 29 29 F F F 1 020-039 20 20 20 29 29 29 F F F 2 040-059 17 17 17 23 23 23 F F F 3 060-079 21 21 21 33 33 33 F F F 4 080-099 18 18 18 24 24 24 F F F 5 100-119 16 16 16 21 21 21 F F F 6 120-139 18 18 18 25 25 25 F F F 7 140-159 18 18 18 25 25 25 F F F 8 160-179 16 16 16 21 21 21 F F F 9 180-199 17 17 17 23 23 23 F F F 10 200-219 22 22 22 34 34 34 F F F 11 220-239 15 15 15 20 20 20 F F F 12 240-259 22 22 22 34 34 34 F F F 13 260-279 22 22 22 34 34 34 F F F 14 280-299 22 22 22 34 34 34 F F F 15 300-319 20 20 20 31 31 31 F F F 16 320-339 20 20 20 31 31 31 F F F 17 340-359 22 22 22 34 34 34 F F F 18 360-379 8 8 8 12 12 12 F F F 19 380-399 22 22 22 34 34 34 F F F 20 400-419 22 22 22 34 34 34 F F F 21 420-439 22 22 22 34 34 34 F F F 22 440-459 22 22 22 34 34 34 F F F 23 460-479 22 22 22 34 34 34 F F F 24 480-499 20 20 20 31 31 31 F F F 25 500-519 21 21 21 33 33 33 F F F 26 520-539 22 22 22 34 34 34 F F F 27 540-559 19 19 19 28 28 28 F F F 28 560-579 21 21 21 33 33 33 F F F 29 580-599 22 22 22 34 34 34 F F F Max gen: 22, Max flush: 34 Begin loop 12 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:10.847289247Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1276,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:10.891287356Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1276,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:10.89186151Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1276} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"3818f0fc76fe787c6cc412af30fd59b930203885\",\n git_commit_timestamp: \"2024-11-01T16:08:30.000000000Z\",\n git_branch: \"renovate/hyper-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:10.891904236Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1276} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:10.891920018Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1276} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:10.891934349Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1276} {"msg":"Crucible 846115bf-0f14-4e57-bd7f-bb9be2694822 has session id: 430fd606-1e23-4bdf-94c2-5e8cdea19758","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:10.892453706Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1276,"session_id":"430fd606-1e23-4bdf-94c2-5e8cdea19758"} {"msg":"Upstairs opts: Upstairs UUID: 846115bf-0f14-4e57-bd7f-bb9be2694822, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:10.892509033Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1276,"session_id":"430fd606-1e23-4bdf-94c2-5e8cdea19758"} {"msg":"Crucible stats registered with UUID: 846115bf-0f14-4e57-bd7f-bb9be2694822","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:10.892538417Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1276,"session_id":"430fd606-1e23-4bdf-94c2-5e8cdea19758"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:10.923486605Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1276,"task":"crutest"} {"msg":"The guest has requested activation with gen:24","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:10.923516609Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1276} {"msg":"846115bf-0f14-4e57-bd7f-bb9be2694822 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:10.923577386Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1276,"session_id":"430fd606-1e23-4bdf-94c2-5e8cdea19758"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:10.923612201Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1276,"client":"0","":"downstairs","session_id":"430fd606-1e23-4bdf-94c2-5e8cdea19758"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:10.923660667Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1276,"client":"0","":"downstairs","session_id":"430fd606-1e23-4bdf-94c2-5e8cdea19758"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:10.923698672Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1276,"client":"1","":"downstairs","session_id":"430fd606-1e23-4bdf-94c2-5e8cdea19758"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:10.923733306Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1276,"client":"1","":"downstairs","session_id":"430fd606-1e23-4bdf-94c2-5e8cdea19758"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:10.923767091Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1276,"client":"2","":"downstairs","session_id":"430fd606-1e23-4bdf-94c2-5e8cdea19758"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:10.923802895Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1276,"client":"2","":"downstairs","session_id":"430fd606-1e23-4bdf-94c2-5e8cdea19758"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:10.923831669Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1276,"":"io task","client":"2","":"downstairs","session_id":"430fd606-1e23-4bdf-94c2-5e8cdea19758"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:10.923907229Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1276,"":"io task","client":"0","":"downstairs","session_id":"430fd606-1e23-4bdf-94c2-5e8cdea19758"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:10.923951745Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1276,"":"io task","client":"1","":"downstairs","session_id":"430fd606-1e23-4bdf-94c2-5e8cdea19758"} {"msg":"ds_connection connected from Ok(127.0.0.1:43201)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:10.923986659Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1276,"":"io task","client":"0","":"downstairs","session_id":"430fd606-1e23-4bdf-94c2-5e8cdea19758"} {"msg":"ds_connection connected from Ok(127.0.0.1:57654)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:10.924041586Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1276,"":"io task","client":"1","":"downstairs","session_id":"430fd606-1e23-4bdf-94c2-5e8cdea19758"} {"msg":"ds_connection connected from Ok(127.0.0.1:37659)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:10.92406975Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1276,"":"io task","client":"2","":"downstairs","session_id":"430fd606-1e23-4bdf-94c2-5e8cdea19758"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:10.924566374Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1276,"client":"2","":"downstairs","session_id":"430fd606-1e23-4bdf-94c2-5e8cdea19758"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:10.924594488Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1276,"client":"2","":"downstairs","session_id":"430fd606-1e23-4bdf-94c2-5e8cdea19758"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:10.924720044Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1276,"client":"1","":"downstairs","session_id":"430fd606-1e23-4bdf-94c2-5e8cdea19758"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:10.924750548Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1276,"client":"1","":"downstairs","session_id":"430fd606-1e23-4bdf-94c2-5e8cdea19758"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:10.92477206Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1276,"client":"0","":"downstairs","session_id":"430fd606-1e23-4bdf-94c2-5e8cdea19758"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:10.924798394Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1276,"client":"0","":"downstairs","session_id":"430fd606-1e23-4bdf-94c2-5e8cdea19758"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:10.925241661Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1276,"client":"2","":"downstairs","session_id":"430fd606-1e23-4bdf-94c2-5e8cdea19758"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:10.92531061Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1276,"client":"1","":"downstairs","session_id":"430fd606-1e23-4bdf-94c2-5e8cdea19758"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:10.925351585Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1276,"client":"0","":"downstairs","session_id":"430fd606-1e23-4bdf-94c2-5e8cdea19758"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:10.925749876Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1276,"client":"2","":"downstairs","session_id":"430fd606-1e23-4bdf-94c2-5e8cdea19758"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:10.925789942Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1276,"client":"2","":"downstairs","session_id":"430fd606-1e23-4bdf-94c2-5e8cdea19758"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:10.925819895Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1276,"session_id":"430fd606-1e23-4bdf-94c2-5e8cdea19758"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:10.925880143Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1276,"client":"1","":"downstairs","session_id":"430fd606-1e23-4bdf-94c2-5e8cdea19758"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:10.925908627Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1276,"client":"1","":"downstairs","session_id":"430fd606-1e23-4bdf-94c2-5e8cdea19758"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:10.925926289Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1276,"session_id":"430fd606-1e23-4bdf-94c2-5e8cdea19758"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:10.925939381Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1276,"client":"0","":"downstairs","session_id":"430fd606-1e23-4bdf-94c2-5e8cdea19758"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:10.925952983Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1276,"client":"0","":"downstairs","session_id":"430fd606-1e23-4bdf-94c2-5e8cdea19758"} {"msg":"[0]R flush_numbers[0..12]: [29, 29, 23, 33, 24, 21, 25, 25, 21, 23, 34, 20]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:10.925967885Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1276,"":"downstairs","":"downstairs","session_id":"430fd606-1e23-4bdf-94c2-5e8cdea19758"} {"msg":"[0]R generation[0..12]: [20, 20, 17, 21, 18, 16, 18, 18, 16, 17, 22, 15]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:10.925981896Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1276,"":"downstairs","":"downstairs","session_id":"430fd606-1e23-4bdf-94c2-5e8cdea19758"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:10.925996078Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1276,"":"downstairs","":"downstairs","session_id":"430fd606-1e23-4bdf-94c2-5e8cdea19758"} {"msg":"[1]R flush_numbers[0..12]: [29, 29, 23, 33, 24, 21, 25, 25, 21, 23, 34, 20]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:10.92601002Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1276,"":"downstairs","":"downstairs","session_id":"430fd606-1e23-4bdf-94c2-5e8cdea19758"} {"msg":"[1]R generation[0..12]: [20, 20, 17, 21, 18, 16, 18, 18, 16, 17, 22, 15]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:10.926023762Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1276,"":"downstairs","":"downstairs","session_id":"430fd606-1e23-4bdf-94c2-5e8cdea19758"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:10.926037674Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1276,"":"downstairs","":"downstairs","session_id":"430fd606-1e23-4bdf-94c2-5e8cdea19758"} {"msg":"[2]R flush_numbers[0..12]: [29, 29, 23, 33, 24, 21, 25, 25, 21, 23, 34, 20]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:10.926051235Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1276,"":"downstairs","":"downstairs","session_id":"430fd606-1e23-4bdf-94c2-5e8cdea19758"} {"msg":"[2]R generation[0..12]: [20, 20, 17, 21, 18, 16, 18, 18, 16, 17, 22, 15]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:10.926064777Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1276,"":"downstairs","":"downstairs","session_id":"430fd606-1e23-4bdf-94c2-5e8cdea19758"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:10.926078699Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1276,"":"downstairs","":"downstairs","session_id":"430fd606-1e23-4bdf-94c2-5e8cdea19758"} Wait for a query_work_queue command to finish before sending IO {"msg":"Max found gen is 23","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:10.926099041Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1276,"":"downstairs","":"downstairs","session_id":"430fd606-1e23-4bdf-94c2-5e8cdea19758"} {"msg":"Generation requested: 24 >= found:23","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:10.926112453Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1276,"":"downstairs","":"downstairs","session_id":"430fd606-1e23-4bdf-94c2-5e8cdea19758"} {"msg":"Next flush: 35","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:10.926125595Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1276,"":"downstairs","":"downstairs","session_id":"430fd606-1e23-4bdf-94c2-5e8cdea19758"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:10.926140277Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1276,"":"downstairs","":"downstairs","session_id":"430fd606-1e23-4bdf-94c2-5e8cdea19758"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:10.926153758Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1276,"session_id":"430fd606-1e23-4bdf-94c2-5e8cdea19758"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:10.92616592Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1276,"client":"0","":"downstairs","session_id":"430fd606-1e23-4bdf-94c2-5e8cdea19758"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:10.926179312Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1276,"client":"1","":"downstairs","session_id":"430fd606-1e23-4bdf-94c2-5e8cdea19758"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:10.926192463Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1276,"client":"2","":"downstairs","session_id":"430fd606-1e23-4bdf-94c2-5e8cdea19758"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:10.926205855Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1276,"session_id":"430fd606-1e23-4bdf-94c2-5e8cdea19758"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:10.926218387Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1276,"session_id":"430fd606-1e23-4bdf-94c2-5e8cdea19758"} {"msg":"846115bf-0f14-4e57-bd7f-bb9be2694822 is now active with session: 430fd606-1e23-4bdf-94c2-5e8cdea19758","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:10.926231379Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1276,"session_id":"430fd606-1e23-4bdf-94c2-5e8cdea19758"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:10.92624342Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1276,"session_id":"430fd606-1e23-4bdf-94c2-5e8cdea19758"} {"msg":"The guest has finished waiting for activation with:24","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:10.926256062Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1276} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 439 len 4096 data: 4 02/30 Read block 560 len 32768 03/30 Read block 227 len 8192 04/30 Read block 232 len 32768 05/30 Write block 288 len 4096 data: 2 06/30 Read block 147 len 36864 07/30 Read block 481 len 12288 08/30 Flush 09/30 Write block 311 len 24576 data: 3 3 3 3 5 6 10/30 Read block 166 len 4096 11/30 Write block 93 len 40960 data: 3 3 3 3 3 3 4 4 5 5 12/30 Write block 346 len 32768 data: 3 3 4 4 3 3 4 4 13/30 Write block 42 len 20480 data: 4 5 5 4 4 14/30 Read block 432 len 8192 15/30 Write block 22 len 8192 data: 2 3 16/30 Write block 499 len 40960 data: 3 3 3 3 2 2 2 2 2 3 17/30 Read block 366 len 28672 18/30 Read block 93 len 28672 19/30 Read block 416 len 28672 20/30 Write block 453 len 12288 data: 4 3 3 21/30 Read block 40 len 40960 22/30 Read block 211 len 4096 23/30 Flush 24/30 Write block 466 len 40960 data: 2 2 3 4 5 5 4 3 3 4 25/30 Flush 26/30 Write block 68 len 32768 data: 7 7 8 7 6 5 4 3 27/30 Read block 77 len 12288 28/30 Write block 49 len 4096 data: 3 29/30 Read block 244 len 12288 30/30 Write block 262 len 32768 data: 4 4 4 4 4 3 4 3 ---------------------------------------------------------------- Crucible gen:24 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 8 Done Done Done false 1026 Acked Read 3 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Read 3 Done Done Done false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:51:10.950751878Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1276} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 20 20 20 29 29 29 F F F 1 020-039 24 24 24 36 36 36 F F F 2 040-059 24 24 24 36 36 36 T T T 3 060-079 21 21 21 33 33 33 T T T 4 080-099 24 24 24 36 36 36 F F F 5 100-119 24 24 24 36 36 36 F F F 6 120-139 18 18 18 25 25 25 F F F 7 140-159 18 18 18 25 25 25 F F F 8 160-179 16 16 16 21 21 21 F F F 9 180-199 17 17 17 23 23 23 F F F 10 200-219 22 22 22 34 34 34 F F F 11 220-239 15 15 15 20 20 20 F F F 12 240-259 22 22 22 34 34 34 F F F 13 260-279 22 22 22 34 34 34 T T T 14 280-299 24 24 24 35 35 35 F F F 15 300-319 24 24 24 36 36 36 F F F 16 320-339 20 20 20 31 31 31 F F F 17 340-359 24 24 24 36 36 36 F F F 18 360-379 8 8 8 12 12 12 F F F 19 380-399 22 22 22 34 34 34 F F F 20 400-419 22 22 22 34 34 34 F F F 21 420-439 24 24 24 35 35 35 F F F 22 440-459 24 24 24 36 36 36 F F F 23 460-479 24 24 24 37 37 37 F F F 24 480-499 24 24 24 36 36 36 F F F 25 500-519 24 24 24 36 36 36 F F F 26 520-539 22 22 22 34 34 34 F F F 27 540-559 19 19 19 28 28 28 F F F 28 560-579 21 21 21 33 33 33 F F F 29 580-599 22 22 22 34 34 34 F F F Max gen: 24, Max flush: 37 A Difference in extent metadata is expected here On loop 12 Verifying data now No change after verify Loop: 12 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 20 20 20 29 29 29 F F F 1 020-039 24 24 24 36 36 36 F F F 2 040-059 25 25 25 38 38 38 F F F 3 060-079 25 25 25 38 38 38 F F F 4 080-099 24 24 24 36 36 36 F F F 5 100-119 24 24 24 36 36 36 F F F 6 120-139 18 18 18 25 25 25 F F F 7 140-159 18 18 18 25 25 25 F F F 8 160-179 16 16 16 21 21 21 F F F 9 180-199 17 17 17 23 23 23 F F F 10 200-219 22 22 22 34 34 34 F F F 11 220-239 15 15 15 20 20 20 F F F 12 240-259 22 22 22 34 34 34 F F F 13 260-279 25 25 25 38 38 38 F F F 14 280-299 24 24 24 35 35 35 F F F 15 300-319 24 24 24 36 36 36 F F F 16 320-339 20 20 20 31 31 31 F F F 17 340-359 24 24 24 36 36 36 F F F 18 360-379 8 8 8 12 12 12 F F F 19 380-399 22 22 22 34 34 34 F F F 20 400-419 22 22 22 34 34 34 F F F 21 420-439 24 24 24 35 35 35 F F F 22 440-459 24 24 24 36 36 36 F F F 23 460-479 24 24 24 37 37 37 F F F 24 480-499 24 24 24 36 36 36 F F F 25 500-519 24 24 24 36 36 36 F F F 26 520-539 22 22 22 34 34 34 F F F 27 540-559 19 19 19 28 28 28 F F F 28 560-579 21 21 21 33 33 33 F F F 29 580-599 22 22 22 34 34 34 F F F Max gen: 25, Max flush: 38 Begin loop 13 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:15.22963621Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1284,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:15.273748334Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1284,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:15.274662912Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1284} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"3818f0fc76fe787c6cc412af30fd59b930203885\",\n git_commit_timestamp: \"2024-11-01T16:08:30.000000000Z\",\n git_branch: \"renovate/hyper-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:15.274706468Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1284} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:15.27472184Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1284} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:15.274736102Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1284} {"msg":"Crucible fddb7a84-3384-425a-b8f3-465c39cab92b has session id: b0161ead-c25c-43a2-8898-b5e6fb42f284","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:15.275057643Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1284,"session_id":"b0161ead-c25c-43a2-8898-b5e6fb42f284"} {"msg":"Upstairs opts: Upstairs UUID: fddb7a84-3384-425a-b8f3-465c39cab92b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:15.275097338Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1284,"session_id":"b0161ead-c25c-43a2-8898-b5e6fb42f284"} {"msg":"Crucible stats registered with UUID: fddb7a84-3384-425a-b8f3-465c39cab92b","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:15.275122111Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1284,"session_id":"b0161ead-c25c-43a2-8898-b5e6fb42f284"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:15.305777251Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1284,"task":"crutest"} {"msg":"The guest has requested activation with gen:26","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:15.305814176Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1284} {"msg":"fddb7a84-3384-425a-b8f3-465c39cab92b active request set","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:15.305881845Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1284,"session_id":"b0161ead-c25c-43a2-8898-b5e6fb42f284"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:15.305900587Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1284,"client":"0","":"downstairs","session_id":"b0161ead-c25c-43a2-8898-b5e6fb42f284"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:15.3059271Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1284,"client":"0","":"downstairs","session_id":"b0161ead-c25c-43a2-8898-b5e6fb42f284"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:15.305942642Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1284,"client":"1","":"downstairs","session_id":"b0161ead-c25c-43a2-8898-b5e6fb42f284"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:15.305956484Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1284,"client":"1","":"downstairs","session_id":"b0161ead-c25c-43a2-8898-b5e6fb42f284"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:15.305969916Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1284,"client":"2","":"downstairs","session_id":"b0161ead-c25c-43a2-8898-b5e6fb42f284"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:15.305983108Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1284,"client":"2","":"downstairs","session_id":"b0161ead-c25c-43a2-8898-b5e6fb42f284"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:15.3059965Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1284,"":"io task","client":"2","":"downstairs","session_id":"b0161ead-c25c-43a2-8898-b5e6fb42f284"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:15.306010391Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1284,"":"io task","client":"0","":"downstairs","session_id":"b0161ead-c25c-43a2-8898-b5e6fb42f284"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:15.306024133Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1284,"":"io task","client":"1","":"downstairs","session_id":"b0161ead-c25c-43a2-8898-b5e6fb42f284"} {"msg":"ds_connection connected from Ok(127.0.0.1:39463)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:15.306133397Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1284,"":"io task","client":"2","":"downstairs","session_id":"b0161ead-c25c-43a2-8898-b5e6fb42f284"} {"msg":"ds_connection connected from Ok(127.0.0.1:62801)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:15.306182383Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1284,"":"io task","client":"1","":"downstairs","session_id":"b0161ead-c25c-43a2-8898-b5e6fb42f284"} {"msg":"ds_connection connected from Ok(127.0.0.1:45020)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:15.306322771Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1284,"":"io task","client":"0","":"downstairs","session_id":"b0161ead-c25c-43a2-8898-b5e6fb42f284"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:15.306968155Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1284,"client":"1","":"downstairs","session_id":"b0161ead-c25c-43a2-8898-b5e6fb42f284"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:15.307029193Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1284,"client":"1","":"downstairs","session_id":"b0161ead-c25c-43a2-8898-b5e6fb42f284"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:15.307064737Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1284,"client":"0","":"downstairs","session_id":"b0161ead-c25c-43a2-8898-b5e6fb42f284"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:15.3070888Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1284,"client":"0","":"downstairs","session_id":"b0161ead-c25c-43a2-8898-b5e6fb42f284"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:15.307120174Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1284,"client":"2","":"downstairs","session_id":"b0161ead-c25c-43a2-8898-b5e6fb42f284"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:15.307146978Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1284,"client":"2","":"downstairs","session_id":"b0161ead-c25c-43a2-8898-b5e6fb42f284"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:15.307598426Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1284,"client":"1","":"downstairs","session_id":"b0161ead-c25c-43a2-8898-b5e6fb42f284"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:15.307633771Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1284,"client":"0","":"downstairs","session_id":"b0161ead-c25c-43a2-8898-b5e6fb42f284"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:15.307797692Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1284,"client":"2","":"downstairs","session_id":"b0161ead-c25c-43a2-8898-b5e6fb42f284"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:15.308145816Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1284,"client":"1","":"downstairs","session_id":"b0161ead-c25c-43a2-8898-b5e6fb42f284"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:15.308169029Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1284,"client":"1","":"downstairs","session_id":"b0161ead-c25c-43a2-8898-b5e6fb42f284"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:15.308183531Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1284,"session_id":"b0161ead-c25c-43a2-8898-b5e6fb42f284"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:15.308246079Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1284,"client":"0","":"downstairs","session_id":"b0161ead-c25c-43a2-8898-b5e6fb42f284"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:15.308264722Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1284,"client":"0","":"downstairs","session_id":"b0161ead-c25c-43a2-8898-b5e6fb42f284"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:15.308278174Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1284,"session_id":"b0161ead-c25c-43a2-8898-b5e6fb42f284"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:15.308316138Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1284,"client":"2","":"downstairs","session_id":"b0161ead-c25c-43a2-8898-b5e6fb42f284"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:15.30833281Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1284,"client":"2","":"downstairs","session_id":"b0161ead-c25c-43a2-8898-b5e6fb42f284"} {"msg":"[0]R flush_numbers[0..12]: [29, 36, 38, 38, 36, 36, 25, 25, 21, 23, 34, 20]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:15.308347732Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1284,"":"downstairs","":"downstairs","session_id":"b0161ead-c25c-43a2-8898-b5e6fb42f284"} {"msg":"[0]R generation[0..12]: [20, 24, 25, 25, 24, 24, 18, 18, 16, 17, 22, 15]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:15.308361694Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1284,"":"downstairs","":"downstairs","session_id":"b0161ead-c25c-43a2-8898-b5e6fb42f284"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:15.308380107Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1284,"":"downstairs","":"downstairs","session_id":"b0161ead-c25c-43a2-8898-b5e6fb42f284"} {"msg":"[1]R flush_numbers[0..12]: [29, 36, 38, 38, 36, 36, 25, 25, 21, 23, 34, 20]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:15.308394349Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1284,"":"downstairs","":"downstairs","session_id":"b0161ead-c25c-43a2-8898-b5e6fb42f284"} {"msg":"[1]R generation[0..12]: [20, 24, 25, 25, 24, 24, 18, 18, 16, 17, 22, 15]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:15.30840797Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1284,"":"downstairs","":"downstairs","session_id":"b0161ead-c25c-43a2-8898-b5e6fb42f284"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:15.308421902Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1284,"":"downstairs","":"downstairs","session_id":"b0161ead-c25c-43a2-8898-b5e6fb42f284"} {"msg":"[2]R flush_numbers[0..12]: [29, 36, 38, 38, 36, 36, 25, 25, 21, 23, 34, 20]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:15.308435464Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1284,"":"downstairs","":"downstairs","session_id":"b0161ead-c25c-43a2-8898-b5e6fb42f284"} {"msg":"[2]R generation[0..12]: [20, 24, 25, 25, 24, 24, 18, 18, 16, 17, 22, 15]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:15.308449065Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1284,"":"downstairs","":"downstairs","session_id":"b0161ead-c25c-43a2-8898-b5e6fb42f284"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:15.308462867Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1284,"":"downstairs","":"downstairs","session_id":"b0161ead-c25c-43a2-8898-b5e6fb42f284"} {"msg":"Max found gen is 26","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:15.308475929Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1284,"":"downstairs","":"downstairs","session_id":"b0161ead-c25c-43a2-8898-b5e6fb42f284"} {"msg":"Generation requested: 26 >= found:26","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:15.308489381Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1284,"":"downstairs","":"downstairs","session_id":"b0161ead-c25c-43a2-8898-b5e6fb42f284"} {"msg":"Next flush: 39","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:15.308502512Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1284,"":"downstairs","":"downstairs","session_id":"b0161ead-c25c-43a2-8898-b5e6fb42f284"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-01T16:51:15.308515584Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1284,"":"downstairs","":"downstairs","session_id":"b0161ead-c25c-43a2-8898-b5e6fb42f284"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:15.308538437Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1284,"session_id":"b0161ead-c25c-43a2-8898-b5e6fb42f284"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:15.30856143Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1284,"client":"0","":"downstairs","session_id":"b0161ead-c25c-43a2-8898-b5e6fb42f284"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:15.308583573Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1284,"client":"1","":"downstairs","session_id":"b0161ead-c25c-43a2-8898-b5e6fb42f284"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:15.308597775Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1284,"client":"2","":"downstairs","session_id":"b0161ead-c25c-43a2-8898-b5e6fb42f284"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:15.308614267Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1284,"session_id":"b0161ead-c25c-43a2-8898-b5e6fb42f284"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:15.308626988Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1284,"session_id":"b0161ead-c25c-43a2-8898-b5e6fb42f284"} {"msg":"fddb7a84-3384-425a-b8f3-465c39cab92b is now active with session: b0161ead-c25c-43a2-8898-b5e6fb42f284","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:15.30863986Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1284,"session_id":"b0161ead-c25c-43a2-8898-b5e6fb42f284"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:15.308651791Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1284,"session_id":"b0161ead-c25c-43a2-8898-b5e6fb42f284"} {"msg":"The guest has finished waiting for activation with:26","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:15.308664123Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1284} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 17 len 16384 data: 2 2 3 3 02/30 Write block 31 len 8192 data: 2 3 03/30 Read block 448 len 32768 04/30 Read block 506 len 8192 05/30 Read block 503 len 40960 06/30 Read block 105 len 20480 07/30 Write block 181 len 36864 data: 4 3 3 3 3 3 3 4 3 08/30 Write block 246 len 32768 data: 4 6 6 6 7 6 5 5 09/30 Flush 10/30 Write block 132 len 28672 data: 7 5 5 5 3 3 3 11/30 Write block 193 len 28672 data: 3 3 3 3 3 3 3 12/30 Read block 159 len 40960 13/30 Read block 399 len 12288 14/30 Write block 70 len 4096 data: 9 15/30 Read block 131 len 36864 16/30 Read block 325 len 36864 17/30 Read block 439 len 16384 18/30 Flush 19/30 Write block 333 len 20480 data: 4 4 4 3 3 20/30 Read block 18 len 40960 21/30 Write block 522 len 36864 data: 3 3 3 3 3 3 4 4 5 22/30 Read block 91 len 40960 23/30 Read block 75 len 4096 24/30 Write block 35 len 16384 data: 3 3 3 3 25/30 Write block 401 len 32768 data: 6 6 5 3 2 3 3 3 26/30 Write block 192 len 20480 data: 3 4 4 4 4 27/30 Write block 121 len 12288 data: 4 4 4 28/30 Write block 134 len 4096 data: 6 29/30 Read block 391 len 12288 30/30 Write block 385 len 32768 data: 2 2 2 2 3 2 2 2 ---------------------------------------------------------------- Crucible gen:26 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1018 Acked Write 5 Done Done Done false 1019 Acked Read 10 Done Done Done false 1020 Acked Write 9 Done Done Done false 1021 Acked Read 10 Done Done Done false 1022 Acked Read 1 Done Done Done false 1023 Acked Write 4 Done Done Done false 1024 Acked Write 8 Done Done Done false 1025 Acked Write 5 Done Done Done false 1026 Acked Write 3 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Read 3 Done Done Done false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 11 11 11 33 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1017 1017 1017 Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:51:15.331247373Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1284} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 26 26 26 39 39 39 F F F 1 020-039 26 26 26 39 39 39 T T T 2 040-059 25 25 25 38 38 38 F F F 3 060-079 26 26 26 40 40 40 F F F 4 080-099 24 24 24 36 36 36 F F F 5 100-119 24 24 24 36 36 36 F F F 6 120-139 26 26 26 40 40 40 T T T 7 140-159 18 18 18 25 25 25 F F F 8 160-179 16 16 16 21 21 21 F F F 9 180-199 26 26 26 40 40 40 T T T 10 200-219 22 22 22 34 34 34 F F F 11 220-239 15 15 15 20 20 20 F F F 12 240-259 26 26 26 39 39 39 F F F 13 260-279 25 25 25 38 38 38 F F F 14 280-299 24 24 24 35 35 35 F F F 15 300-319 24 24 24 36 36 36 F F F 16 320-339 20 20 20 31 31 31 T T T 17 340-359 24 24 24 36 36 36 F F F 18 360-379 8 8 8 12 12 12 F F F 19 380-399 22 22 22 34 34 34 T T T 20 400-419 22 22 22 34 34 34 T T T 21 420-439 24 24 24 35 35 35 F F F 22 440-459 24 24 24 36 36 36 F F F 23 460-479 24 24 24 37 37 37 F F F 24 480-499 24 24 24 36 36 36 F F F 25 500-519 24 24 24 36 36 36 F F F 26 520-539 22 22 22 34 34 34 T T T 27 540-559 19 19 19 28 28 28 F F F 28 560-579 21 21 21 33 33 33 F F F 29 580-599 22 22 22 34 34 34 F F F Max gen: 26, Max flush: 40 A Difference in extent metadata is expected here On loop 13 Verifying data now No change after verify Loop: 13 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 26 26 26 39 39 39 F F F 1 020-039 27 27 27 41 41 41 F F F 2 040-059 25 25 25 38 38 38 F F F 3 060-079 26 26 26 40 40 40 F F F 4 080-099 24 24 24 36 36 36 F F F 5 100-119 24 24 24 36 36 36 F F F 6 120-139 27 27 27 41 41 41 F F F 7 140-159 18 18 18 25 25 25 F F F 8 160-179 16 16 16 21 21 21 F F F 9 180-199 27 27 27 41 41 41 F F F 10 200-219 22 22 22 34 34 34 F F F 11 220-239 15 15 15 20 20 20 F F F 12 240-259 26 26 26 39 39 39 F F F 13 260-279 25 25 25 38 38 38 F F F 14 280-299 24 24 24 35 35 35 F F F 15 300-319 24 24 24 36 36 36 F F F 16 320-339 27 27 27 41 41 41 F F F 17 340-359 24 24 24 36 36 36 F F F 18 360-379 8 8 8 12 12 12 F F F 19 380-399 27 27 27 41 41 41 F F F 20 400-419 27 27 27 41 41 41 F F F 21 420-439 24 24 24 35 35 35 F F F 22 440-459 24 24 24 36 36 36 F F F 23 460-479 24 24 24 37 37 37 F F F 24 480-499 24 24 24 36 36 36 F F F 25 500-519 24 24 24 36 36 36 F F F 26 520-539 27 27 27 41 41 41 F F F 27 540-559 19 19 19 28 28 28 F F F 28 560-579 21 21 21 33 33 33 F F F 29 580-599 22 22 22 34 34 34 F F F Max gen: 27, Max flush: 41 Begin loop 14 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:19.664052392Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1292,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:19.707687625Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1292,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:19.708278521Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1292} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"3818f0fc76fe787c6cc412af30fd59b930203885\",\n git_commit_timestamp: \"2024-11-01T16:08:30.000000000Z\",\n git_branch: \"renovate/hyper-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:19.708327077Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1292} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:19.708344709Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1292} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:19.708359631Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1292} {"msg":"Crucible 7d79de66-256f-478a-83ed-03ecbb26e85b has session id: 66e4453d-cbb2-4e06-9c0b-1656fd65534c","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:19.708755052Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1292,"session_id":"66e4453d-cbb2-4e06-9c0b-1656fd65534c"} {"msg":"Upstairs opts: Upstairs UUID: 7d79de66-256f-478a-83ed-03ecbb26e85b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:19.708795628Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1292,"session_id":"66e4453d-cbb2-4e06-9c0b-1656fd65534c"} {"msg":"Crucible stats registered with UUID: 7d79de66-256f-478a-83ed-03ecbb26e85b","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:19.708819811Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1292,"session_id":"66e4453d-cbb2-4e06-9c0b-1656fd65534c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:19.739602517Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1292,"task":"crutest"} {"msg":"The guest has requested activation with gen:28","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:19.73963Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1292} {"msg":"7d79de66-256f-478a-83ed-03ecbb26e85b active request set","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:19.73970506Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1292,"session_id":"66e4453d-cbb2-4e06-9c0b-1656fd65534c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:19.739724022Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1292,"client":"0","":"downstairs","session_id":"66e4453d-cbb2-4e06-9c0b-1656fd65534c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:19.739748426Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1292,"client":"0","":"downstairs","session_id":"66e4453d-cbb2-4e06-9c0b-1656fd65534c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:19.739763038Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1292,"client":"1","":"downstairs","session_id":"66e4453d-cbb2-4e06-9c0b-1656fd65534c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:19.739777289Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1292,"client":"1","":"downstairs","session_id":"66e4453d-cbb2-4e06-9c0b-1656fd65534c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:19.739794062Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1292,"client":"2","":"downstairs","session_id":"66e4453d-cbb2-4e06-9c0b-1656fd65534c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:19.739808683Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1292,"client":"2","":"downstairs","session_id":"66e4453d-cbb2-4e06-9c0b-1656fd65534c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:19.739872922Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1292,"":"io task","client":"2","":"downstairs","session_id":"66e4453d-cbb2-4e06-9c0b-1656fd65534c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:19.739923588Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1292,"":"io task","client":"0","":"downstairs","session_id":"66e4453d-cbb2-4e06-9c0b-1656fd65534c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:19.739962743Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1292,"":"io task","client":"1","":"downstairs","session_id":"66e4453d-cbb2-4e06-9c0b-1656fd65534c"} {"msg":"ds_connection connected from Ok(127.0.0.1:47069)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:19.739994747Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1292,"":"io task","client":"2","":"downstairs","session_id":"66e4453d-cbb2-4e06-9c0b-1656fd65534c"} {"msg":"ds_connection connected from Ok(127.0.0.1:45615)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:19.740112652Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1292,"":"io task","client":"0","":"downstairs","session_id":"66e4453d-cbb2-4e06-9c0b-1656fd65534c"} {"msg":"ds_connection connected from Ok(127.0.0.1:50249)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:19.740160709Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1292,"":"io task","client":"1","":"downstairs","session_id":"66e4453d-cbb2-4e06-9c0b-1656fd65534c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:19.740838706Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1292,"client":"2","":"downstairs","session_id":"66e4453d-cbb2-4e06-9c0b-1656fd65534c"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:19.7408667Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1292,"client":"2","":"downstairs","session_id":"66e4453d-cbb2-4e06-9c0b-1656fd65534c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:19.740881902Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1292,"client":"1","":"downstairs","session_id":"66e4453d-cbb2-4e06-9c0b-1656fd65534c"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:19.740896213Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1292,"client":"1","":"downstairs","session_id":"66e4453d-cbb2-4e06-9c0b-1656fd65534c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:19.740909735Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1292,"client":"0","":"downstairs","session_id":"66e4453d-cbb2-4e06-9c0b-1656fd65534c"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:19.740923097Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1292,"client":"0","":"downstairs","session_id":"66e4453d-cbb2-4e06-9c0b-1656fd65534c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:19.741538086Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1292,"client":"0","":"downstairs","session_id":"66e4453d-cbb2-4e06-9c0b-1656fd65534c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:19.741587703Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1292,"client":"1","":"downstairs","session_id":"66e4453d-cbb2-4e06-9c0b-1656fd65534c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:19.741660362Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1292,"client":"2","":"downstairs","session_id":"66e4453d-cbb2-4e06-9c0b-1656fd65534c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:19.742144994Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1292,"client":"0","":"downstairs","session_id":"66e4453d-cbb2-4e06-9c0b-1656fd65534c"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:19.742171448Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1292,"client":"0","":"downstairs","session_id":"66e4453d-cbb2-4e06-9c0b-1656fd65534c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:19.74218628Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1292,"session_id":"66e4453d-cbb2-4e06-9c0b-1656fd65534c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:19.742254139Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1292,"client":"1","":"downstairs","session_id":"66e4453d-cbb2-4e06-9c0b-1656fd65534c"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:19.742273841Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1292,"client":"1","":"downstairs","session_id":"66e4453d-cbb2-4e06-9c0b-1656fd65534c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:19.742287303Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1292,"session_id":"66e4453d-cbb2-4e06-9c0b-1656fd65534c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:19.742299514Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1292,"client":"2","":"downstairs","session_id":"66e4453d-cbb2-4e06-9c0b-1656fd65534c"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:19.742312806Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1292,"client":"2","":"downstairs","session_id":"66e4453d-cbb2-4e06-9c0b-1656fd65534c"} {"msg":"[0]R flush_numbers[0..12]: [39, 41, 38, 40, 36, 36, 41, 25, 21, 41, 34, 20]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:19.742326458Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1292,"":"downstairs","":"downstairs","session_id":"66e4453d-cbb2-4e06-9c0b-1656fd65534c"} {"msg":"[0]R generation[0..12]: [26, 27, 25, 26, 24, 24, 27, 18, 16, 27, 22, 15]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:19.74234019Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1292,"":"downstairs","":"downstairs","session_id":"66e4453d-cbb2-4e06-9c0b-1656fd65534c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:19.742354171Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1292,"":"downstairs","":"downstairs","session_id":"66e4453d-cbb2-4e06-9c0b-1656fd65534c"} {"msg":"[1]R flush_numbers[0..12]: [39, 41, 38, 40, 36, 36, 41, 25, 21, 41, 34, 20]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:19.742367853Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1292,"":"downstairs","":"downstairs","session_id":"66e4453d-cbb2-4e06-9c0b-1656fd65534c"} {"msg":"[1]R generation[0..12]: [26, 27, 25, 26, 24, 24, 27, 18, 16, 27, 22, 15]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:19.742381465Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1292,"":"downstairs","":"downstairs","session_id":"66e4453d-cbb2-4e06-9c0b-1656fd65534c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:19.742398457Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1292,"":"downstairs","":"downstairs","session_id":"66e4453d-cbb2-4e06-9c0b-1656fd65534c"} {"msg":"[2]R flush_numbers[0..12]: [39, 41, 38, 40, 36, 36, 41, 25, 21, 41, 34, 20]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:19.742412539Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1292,"":"downstairs","":"downstairs","session_id":"66e4453d-cbb2-4e06-9c0b-1656fd65534c"} {"msg":"[2]R generation[0..12]: [26, 27, 25, 26, 24, 24, 27, 18, 16, 27, 22, 15]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:19.742426181Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1292,"":"downstairs","":"downstairs","session_id":"66e4453d-cbb2-4e06-9c0b-1656fd65534c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:19.742440203Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1292,"":"downstairs","":"downstairs","session_id":"66e4453d-cbb2-4e06-9c0b-1656fd65534c"} {"msg":"Max found gen is 28","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:19.742453434Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1292,"":"downstairs","":"downstairs","session_id":"66e4453d-cbb2-4e06-9c0b-1656fd65534c"} {"msg":"Generation requested: 28 >= found:28","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:19.742466636Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1292,"":"downstairs","":"downstairs","session_id":"66e4453d-cbb2-4e06-9c0b-1656fd65534c"} {"msg":"Next flush: 42","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:19.742479468Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1292,"":"downstairs","":"downstairs","session_id":"66e4453d-cbb2-4e06-9c0b-1656fd65534c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:19.742492539Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1292Wait for a query_work_queue command to finish before sending IO ,"":"downstairs","":"downstairs","session_id":"66e4453d-cbb2-4e06-9c0b-1656fd65534c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:19.742514312Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1292,"session_id":"66e4453d-cbb2-4e06-9c0b-1656fd65534c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:19.742526283Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1292,"client":"0","":"downstairs","session_id":"66e4453d-cbb2-4e06-9c0b-1656fd65534c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:19.742539465Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1292,"client":"1","":"downstairs","session_id":"66e4453d-cbb2-4e06-9c0b-1656fd65534c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:19.742552447Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1292,"client":"2","":"downstairs","session_id":"66e4453d-cbb2-4e06-9c0b-1656fd65534c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:19.742565839Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1292,"session_id":"66e4453d-cbb2-4e06-9c0b-1656fd65534c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:19.74257814Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1292,"session_id":"66e4453d-cbb2-4e06-9c0b-1656fd65534c"} {"msg":"7d79de66-256f-478a-83ed-03ecbb26e85b is now active with session: 66e4453d-cbb2-4e06-9c0b-1656fd65534c","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:19.742591062Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1292,"session_id":"66e4453d-cbb2-4e06-9c0b-1656fd65534c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:19.742604984Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1292,"session_id":"66e4453d-cbb2-4e06-9c0b-1656fd65534c"} {"msg":"The guest has finished waiting for activation with:28","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:19.742617545Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1292} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 471 len 40960 data: 6 5 4 4 5 4 4 5 5 5 02/30 Write block 428 len 20480 data: 3 3 3 3 2 03/30 Write block 305 len 12288 data: 5 5 5 04/30 Read block 62 len 28672 05/30 Read block 248 len 28672 06/30 Read block 65 len 40960 07/30 Read block 90 len 20480 08/30 Read block 11 len 40960 09/30 Read block 333 len 4096 10/30 Flush 11/30 Write block 423 len 28672 data: 5 4 4 4 3 4 4 12/30 Flush 13/30 Write block 369 len 20480 data: 3 2 2 2 3 14/30 Flush 15/30 Write block 124 len 40960 data: 3 3 3 3 3 4 5 6 8 6 16/30 Write block 389 len 20480 data: 4 3 3 3 4 17/30 Write block 89 len 20480 data: 3 5 3 3 4 18/30 Read block 340 len 32768 19/30 Write block 49 len 20480 data: 4 3 3 3 3 20/30 Read block 402 len 20480 21/30 Read block 367 len 20480 22/30 Write block 291 len 16384 data: 2 2 3 3 23/30 Write block 173 len 32768 data: 2 2 2 3 4 4 4 4 24/30 Read block 122 len 24576 25/30 Write block 273 len 40960 data: 5 6 5 4 3 2 2 2 2 2 26/30 Read block 268 len 24576 27/30 Read block 386 len 40960 28/30 Read block 317 len 32768 29/30 Read block 160 len 16384 30/30 Read block 52 len 16384 ---------------------------------------------------------------- Crucible gen:28 GIO:true work queues: Upstairs:0 downstairs:16 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1014 Acked Write 10 Done Done Done false 1015 Acked Write 5 Done Done Done false 1016 Acked Write 5 Done Done Done false 1017 Acked Read 8 Done Done Done false 1018 Acked Write 5 Done Done Done false 1019 Acked Read 5 Done Done Done false 1020 Acked Read 5 Done Done Done false 1021 Acked Write 4 Done Done Done false 1022 Acked Write 8 Done Done Done false 1023 Acked Read 6 Done Done Done false 1024 Acked Write 10 Done Done Done false 1025 Acked Read 6 Done Done Done false 1026 Acked Read 10 Done Done Done false 1027 Acked Read 8 Done Done Done false 1028 Acked Read 4 Done Done Done false 1029 Acked Read 4 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 0 1 Done 16 15 16 47 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1013 1013 1013 Downstairs last five completed: 1013 1012 1011 1010 1009 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:51:19.766823654Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1292} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 26 26 26 39 39 39 F F F 1 020-039 27 27 27 41 41 41 F F F 2 040-059 25 25 25 38 38 38 T T T 3 060-079 26 26 26 40 40 40 F F F 4 080-099 24 24 24 36 36 36 T T T 5 100-119 24 24 24 36 36 36 F F F 6 120-139 27 27 27 41 41 41 T T T 7 140-159 18 18 18 25 25 25 F F F 8 160-179 16 16 16 21 21 21 T T T 9 180-199 27 27 27 41 41 41 T T T 10 200-219 22 22 22 34 34 34 F F F 11 220-239 15 15 15 20 20 20 F F F 12 240-259 26 26 26 39 39 39 F F F 13 260-279 25 25 25 38 38 38 T T T 14 280-299 24 24 24 35 35 35 T T T 15 300-319 28 28 28 42 42 42 F F F 16 320-339 27 27 27 41 41 41 F F F 17 340-359 24 24 24 36 36 36 F F F 18 360-379 28 28 28 44 44 44 F F F 19 380-399 27 27 27 41 41 41 T T T 20 400-419 27 27 27 41 41 41 F F F 21 420-439 28 28 28 43 43 43 F F F 22 440-459 24 24 24 36 36 36 F F F 23 460-479 28 28 28 42 42 42 F F F 24 480-499 28 28 28 42 42 42 F F F 25 500-519 24 24 24 36 36 36 F F F 26 520-539 27 27 27 41 41 41 F F F 27 540-559 19 19 19 28 28 28 F F F 28 560-579 21 21 21 33 33 33 F F F 29 580-599 22 22 22 34 34 34 F F F Max gen: 28, Max flush: 44 A Difference in extent metadata is expected here On loop 14 Verifying data now No change after verify Loop: 14 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 26 26 26 39 39 39 F F F 1 020-039 27 27 27 41 41 41 F F F 2 040-059 29 29 29 45 45 45 F F F 3 060-079 26 26 26 40 40 40 F F F 4 080-099 29 29 29 45 45 45 F F F 5 100-119 24 24 24 36 36 36 F F F 6 120-139 29 29 29 45 45 45 F F F 7 140-159 18 18 18 25 25 25 F F F 8 160-179 29 29 29 45 45 45 F F F 9 180-199 29 29 29 45 45 45 F F F 10 200-219 22 22 22 34 34 34 F F F 11 220-239 15 15 15 20 20 20 F F F 12 240-259 26 26 26 39 39 39 F F F 13 260-279 29 29 29 45 45 45 F F F 14 280-299 29 29 29 45 45 45 F F F 15 300-319 28 28 28 42 42 42 F F F 16 320-339 27 27 27 41 41 41 F F F 17 340-359 24 24 24 36 36 36 F F F 18 360-379 28 28 28 44 44 44 F F F 19 380-399 29 29 29 45 45 45 F F F 20 400-419 27 27 27 41 41 41 F F F 21 420-439 28 28 28 43 43 43 F F F 22 440-459 24 24 24 36 36 36 F F F 23 460-479 28 28 28 42 42 42 F F F 24 480-499 28 28 28 42 42 42 F F F 25 500-519 24 24 24 36 36 36 F F F 26 520-539 27 27 27 41 41 41 F F F 27 540-559 19 19 19 28 28 28 F F F 28 560-579 21 21 21 33 33 33 F F F 29 580-599 22 22 22 34 34 34 F F F Max gen: 29, Max flush: 45 Begin loop 15 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:24.106499759Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1300,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:24.150328706Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1300,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:24.150973019Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1300} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"3818f0fc76fe787c6cc412af30fd59b930203885\",\n git_commit_timestamp: \"2024-11-01T16:08:30.000000000Z\",\n git_branch: \"renovate/hyper-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:24.151019065Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1300} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:24.151039918Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1300} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:24.15105534Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1300} {"msg":"Crucible 33829395-9e88-4dbb-a406-d08b51433acd has session id: 3f39bb17-70be-4bc1-a5de-2bdf2c7a7dbd","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:24.151479504Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1300,"session_id":"3f39bb17-70be-4bc1-a5de-2bdf2c7a7dbd"} {"msg":"Upstairs opts: Upstairs UUID: 33829395-9e88-4dbb-a406-d08b51433acd, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:24.151514719Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1300,"session_id":"3f39bb17-70be-4bc1-a5de-2bdf2c7a7dbd"} {"msg":"Crucible stats registered with UUID: 33829395-9e88-4dbb-a406-d08b51433acd","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:24.151537722Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1300,"session_id":"3f39bb17-70be-4bc1-a5de-2bdf2c7a7dbd"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:24.182809911Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1300,"task":"crutest"} {"msg":"The guest has requested activation with gen:30","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:24.182843105Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1300} {"msg":"33829395-9e88-4dbb-a406-d08b51433acd active request set","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:24.18287807Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1300,"session_id":"3f39bb17-70be-4bc1-a5de-2bdf2c7a7dbd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:24.182895802Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1300,"client":"0","":"downstairs","session_id":"3f39bb17-70be-4bc1-a5de-2bdf2c7a7dbd"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:24.182924096Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1300,"client":"0","":"downstairs","session_id":"3f39bb17-70be-4bc1-a5de-2bdf2c7a7dbd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:24.182941858Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1300,"client":"1","":"downstairs","session_id":"3f39bb17-70be-4bc1-a5de-2bdf2c7a7dbd"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:24.18295872Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1300,"client":"1","":"downstairs","session_id":"3f39bb17-70be-4bc1-a5de-2bdf2c7a7dbd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:24.182985204Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1300,"client":"2","":"downstairs","session_id":"3f39bb17-70be-4bc1-a5de-2bdf2c7a7dbd"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:24.183018868Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1300,"client":"2","":"downstairs","session_id":"3f39bb17-70be-4bc1-a5de-2bdf2c7a7dbd"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:24.183056013Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1300,"":"io task","client":"2","":"downstairs","session_id":"3f39bb17-70be-4bc1-a5de-2bdf2c7a7dbd"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:24.183091357Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1300,"":"io task","client":"0","":"downstairs","session_id":"3f39bb17-70be-4bc1-a5de-2bdf2c7a7dbd"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:24.183123521Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1300,"":"io task","client":"1","":"downstairs","session_id":"3f39bb17-70be-4bc1-a5de-2bdf2c7a7dbd"} {"msg":"ds_connection connected from Ok(127.0.0.1:47151)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:24.183155556Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1300,"":"io task","client":"2","":"downstairs","session_id":"3f39bb17-70be-4bc1-a5de-2bdf2c7a7dbd"} {"msg":"ds_connection connected from Ok(127.0.0.1:47894)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:24.183185629Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1300,"":"io task","client":"1","":"downstairs","session_id":"3f39bb17-70be-4bc1-a5de-2bdf2c7a7dbd"} {"msg":"ds_connection connected from Ok(127.0.0.1:53896)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:24.183429461Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1300,"":"io task","client":"0","":"downstairs","session_id":"3f39bb17-70be-4bc1-a5de-2bdf2c7a7dbd"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:24.183950828Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1300,"client":"0","":"downstairs","session_id":"3f39bb17-70be-4bc1-a5de-2bdf2c7a7dbd"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:24.183979642Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1300,"client":"0","":"downstairs","session_id":"3f39bb17-70be-4bc1-a5de-2bdf2c7a7dbd"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:24.184074914Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1300,"client":"1","":"downstairs","session_id":"3f39bb17-70be-4bc1-a5de-2bdf2c7a7dbd"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:24.184096257Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1300,"client":"1","":"downstairs","session_id":"3f39bb17-70be-4bc1-a5de-2bdf2c7a7dbd"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:24.184147724Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1300,"client":"2","":"downstairs","session_id":"3f39bb17-70be-4bc1-a5de-2bdf2c7a7dbd"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:24.184164756Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1300,"client":"2","":"downstairs","session_id":"3f39bb17-70be-4bc1-a5de-2bdf2c7a7dbd"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:24.184581289Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1300,"client":"0","":"downstairs","session_id":"3f39bb17-70be-4bc1-a5de-2bdf2c7a7dbd"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:24.184735429Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1300,"client":"1","":"downstairs","session_id":"3f39bb17-70be-4bc1-a5de-2bdf2c7a7dbd"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:24.184765233Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1300,"client":"2","":"downstairs","session_id":"3f39bb17-70be-4bc1-a5de-2bdf2c7a7dbd"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:24.185144652Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1300,"client":"0","":"downstairs","session_id":"3f39bb17-70be-4bc1-a5de-2bdf2c7a7dbd"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:24.185173736Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1300,"client":"0","":"downstairs","session_id":"3f39bb17-70be-4bc1-a5de-2bdf2c7a7dbd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:24.185190528Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1300,"session_id":"3f39bb17-70be-4bc1-a5de-2bdf2c7a7dbd"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:24.185264547Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1300,"client":"1","":"downstairs","session_id":"3f39bb17-70be-4bc1-a5de-2bdf2c7a7dbd"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:24.18528422Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1300,"client":"1","":"downstairs","session_id":"3f39bb17-70be-4bc1-a5de-2bdf2c7a7dbd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:24.185298752Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1300,"session_id":"3f39bb17-70be-4bc1-a5de-2bdf2c7a7dbd"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:24.185355109Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1300,"client":"2","":"downstairs","session_id":"3f39bb17-70be-4bc1-a5de-2bdf2c7a7dbd"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:24.185374962Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1300,"client":"2","":"downstairs","session_id":"3f39bb17-70be-4bc1-a5de-2bdf2c7a7dbd"} {"msg":"[0]R flush_numbers[0..12]: [39, 41, 45, 40, 45, 36, 45, 25, 45, 45, 34, 20]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:24.185389193Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1300,"":"downstairs","":"downstairs","session_id":"3f39bb17-70be-4bc1-a5de-2bdf2c7a7dbd"} {"msg":"[0]R generation[0..12]: [26, 27, 29, 26, 29, 24, 29, 18, 29, 29, 22, 15]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:24.185403145Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1300,"":"downstairs","":"downstairs","session_id":"3f39bb17-70be-4bc1-a5de-2bdf2c7a7dbd"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:24.185417417Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1300,"":"downstairs","":"downstairs","session_id":"3f39bb17-70be-4bc1-a5de-2bdf2c7a7dbd"} {"msg":"[1]R flush_numbers[0..12]: [39, 41, 45, 40, 45, 36, 45, 25, 45, 45, 34, 20]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:24.185431259Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1300,"":"downstairs","":"downstairs","session_id":"3f39bb17-70be-4bc1-a5de-2bdf2c7a7dbd"} {"msg":"[1]R generation[0..12]: [26, 27, 29, 26, 29, 24, 29, 18, 29, 29, 22, 15]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:24.185444821Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1300,"":"downstairs","":"downstairs","session_id":"3f39bb17-70be-4bc1-a5de-2bdf2c7a7dbd"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:24.185458922Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1300,"":"downstairs","":"downstairs","session_id":"3f39bb17-70be-4bc1-a5de-2bdf2c7a7dbd"} {"msg":"[2]R flush_numbers[0..12]: [39, 41, 45, 40, 45, 36, 45, 25, 45, 45, 34, 20]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:24.185473414Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1300,"":"downstairs","":"downstairs","session_id":"3f39bb17-70be-4bc1-a5de-2bdf2c7a7dbd"} {"msg":"[2]R generation[0..12]: [26, 27, 29, 26, 29, 24, 29, 18, 29, 29, 22, 15]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:24.185491137Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1300,"":"downstairs","":"downstairs","session_id":"3f39bb17-70be-4bc1-a5de-2bdf2c7a7dbd"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:24.185505539Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1300,"":"downstairs","":"downstairs","session_id":"3f39bb17-70be-4bc1-a5de-2bdf2c7a7dbd"} {"msg":"Max found gen is 30","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:24.18551875Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1300,"":"downstairs","":"downstairs","session_id":"3f39bb17-70be-4bc1-a5de-2bdf2c7a7dbd"} {"msg":"Generation requested: 30 >= found:30","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:24.185532732Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1300,"":"downstairs","":"downstairs","session_id":"3f39bb17-70be-4bc1-a5de-2bdf2c7a7dbd"} {"msg":"Next flush: 46","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:24.185545884Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1300,"":"downstairs","":"downstairs","Wait for a query_work_queue command to finish before sending IO session_id":"3f39bb17-70be-4bc1-a5de-2bdf2c7a7dbd"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:24.185566896Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1300,"":"downstairs","":"downstairs","session_id":"3f39bb17-70be-4bc1-a5de-2bdf2c7a7dbd"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:24.185580798Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1300,"session_id":"3f39bb17-70be-4bc1-a5de-2bdf2c7a7dbd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:24.18559302Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1300,"client":"0","":"downstairs","session_id":"3f39bb17-70be-4bc1-a5de-2bdf2c7a7dbd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:24.185606201Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1300,"client":"1","":"downstairs","session_id":"3f39bb17-70be-4bc1-a5de-2bdf2c7a7dbd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:24.185619353Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1300,"client":"2","":"downstairs","session_id":"3f39bb17-70be-4bc1-a5de-2bdf2c7a7dbd"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:24.185632535Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1300,"session_id":"3f39bb17-70be-4bc1-a5de-2bdf2c7a7dbd"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:24.185644736Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1300,"session_id":"3f39bb17-70be-4bc1-a5de-2bdf2c7a7dbd"} {"msg":"33829395-9e88-4dbb-a406-d08b51433acd is now active with session: 3f39bb17-70be-4bc1-a5de-2bdf2c7a7dbd","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:24.185657478Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1300,"session_id":"3f39bb17-70be-4bc1-a5de-2bdf2c7a7dbd"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:24.18566948Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1300,"session_id":"3f39bb17-70be-4bc1-a5de-2bdf2c7a7dbd"} {"msg":"The guest has finished waiting for activation with:30","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:24.185682201Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1300} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 337 len 36864 data: 4 3 2 2 2 2 2 3 3 02/30 Read block 402 len 4096 03/30 Read block 473 len 36864 04/30 Read block 321 len 28672 05/30 Write block 365 len 8192 data: 2 2 06/30 Flush 07/30 Write block 258 len 16384 data: 3 4 4 5 08/30 Write block 11 len 24576 data: 3 2 2 2 2 2 09/30 Write block 559 len 24576 data: 3 2 2 2 2 2 10/30 Read block 428 len 24576 11/30 Write block 238 len 32768 data: 3 2 2 2 2 2 3 3 12/30 Read block 418 len 4096 13/30 Flush 14/30 Write block 534 len 20480 data: 4 3 2 2 2 15/30 Read block 4 len 12288 16/30 Read block 422 len 24576 17/30 Read block 507 len 20480 18/30 Read block 208 len 28672 19/30 Write block 381 len 28672 data: 3 3 2 2 3 3 3 20/30 Read block 150 len 8192 21/30 Flush 22/30 Write block 39 len 24576 data: 3 2 3 5 6 6 23/30 Write block 543 len 28672 data: 2 2 2 3 3 3 3 24/30 Read block 469 len 24576 25/30 Write block 211 len 40960 data: 5 5 4 2 2 4 4 5 3 3 26/30 Read block 579 len 20480 27/30 Read block 445 len 4096 28/30 Read block 184 len 16384 29/30 Write block 424 len 32768 data: 5 5 5 4 5 5 4 4 30/30 Read block 464 len 12288 ---------------------------------------------------------------- Crucible gen:30 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 6 Done Done Done false 1022 Acked Write 7 Done Done Done false 1023 Acked Read 6 Done Done Done false 1024 Acked Write 10 Done Done Done false 1025 Acked Read 5 Done Done Done false 1026 Acked Read 1 Done Done Done false 1027 Acked Read 4 Done Done Done false 1028 Acked Write 8 Sent Sent Done false 1029 Acked Read 3 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 0 4 Done 7 7 9 23 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1020 1020 1020 Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:51:24.210831252Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1300} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 30 30 30 47 47 47 F F F 1 020-039 27 27 27 41 41 41 T T T 2 040-059 29 29 29 45 45 45 T T T 3 060-079 26 26 26 40 40 40 F F F 4 080-099 29 29 29 45 45 45 F F F 5 100-119 24 24 24 36 36 36 F F F 6 120-139 29 29 29 45 45 45 F F F 7 140-159 18 18 18 25 25 25 F F F 8 160-179 29 29 29 45 45 45 F F F 9 180-199 29 29 29 45 45 45 F F F 10 200-219 22 22 22 34 34 34 T T T 11 220-239 30 30 30 47 47 47 T T T 12 240-259 30 30 30 47 47 47 F F F 13 260-279 30 30 30 47 47 47 F F F 14 280-299 29 29 29 45 45 45 F F F 15 300-319 28 28 28 42 42 42 F F F 16 320-339 30 30 30 46 46 46 F F F 17 340-359 30 30 30 46 46 46 F F F 18 360-379 30 30 30 46 46 46 F F F 19 380-399 30 30 30 48 48 48 F F F 20 400-419 27 27 27 41 41 41 F F F 21 420-439 28 28 28 43 43 43 T T T 22 440-459 24 24 24 36 36 36 F F F 23 460-479 28 28 28 42 42 42 F F F 24 480-499 28 28 28 42 42 42 F F F 25 500-519 24 24 24 36 36 36 F F F 26 520-539 30 30 30 48 48 48 F F F 27 540-559 30 30 30 47 47 47 T T T 28 560-579 30 30 30 47 47 47 F F F 29 580-599 22 22 22 34 34 34 F F F Max gen: 30, Max flush: 48 A Difference in extent metadata is expected here On loop 15 Verifying data now No change after verify Loop: 15 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 30 30 30 47 47 47 F F F 1 020-039 31 31 31 49 49 49 F F F 2 040-059 31 31 31 49 49 49 F F F 3 060-079 26 26 26 40 40 40 F F F 4 080-099 29 29 29 45 45 45 F F F 5 100-119 24 24 24 36 36 36 F F F 6 120-139 29 29 29 45 45 45 F F F 7 140-159 18 18 18 25 25 25 F F F 8 160-179 29 29 29 45 45 45 F F F 9 180-199 29 29 29 45 45 45 F F F 10 200-219 31 31 31 49 49 49 F F F 11 220-239 31 31 31 49 49 49 F F F 12 240-259 30 30 30 47 47 47 F F F 13 260-279 30 30 30 47 47 47 F F F 14 280-299 29 29 29 45 45 45 F F F 15 300-319 28 28 28 42 42 42 F F F 16 320-339 30 30 30 46 46 46 F F F 17 340-359 30 30 30 46 46 46 F F F 18 360-379 30 30 30 46 46 46 F F F 19 380-399 30 30 30 48 48 48 F F F 20 400-419 27 27 27 41 41 41 F F F 21 420-439 31 31 31 49 49 49 F F F 22 440-459 24 24 24 36 36 36 F F F 23 460-479 28 28 28 42 42 42 F F F 24 480-499 28 28 28 42 42 42 F F F 25 500-519 24 24 24 36 36 36 F F F 26 520-539 30 30 30 48 48 48 F F F 27 540-559 31 31 31 49 49 49 F F F 28 560-579 30 30 30 47 47 47 F F F 29 580-599 22 22 22 34 34 34 F F F Max gen: 31, Max flush: 49 Begin loop 16 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:28.528247019Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1308,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:51:28.570845548Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1308,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:28.571204614Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1308,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:28.603201167Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1308,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:28.603704472Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1308} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"3818f0fc76fe787c6cc412af30fd59b930203885\",\n git_commit_timestamp: \"2024-11-01T16:08:30.000000000Z\",\n git_branch: \"renovate/hyper-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:28.603734855Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1308} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:28.603748247Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1308} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:28.603762429Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1308} {"msg":"Crucible a9af12b3-2c41-41c0-909a-b4c4b3953360 has session id: 2b849fa5-eca3-4fb5-a8d8-8b5bcdbae512","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:28.60423709Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1308,"session_id":"2b849fa5-eca3-4fb5-a8d8-8b5bcdbae512"} {"msg":"Upstairs opts: Upstairs UUID: a9af12b3-2c41-41c0-909a-b4c4b3953360, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:28.604279146Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1308,"session_id":"2b849fa5-eca3-4fb5-a8d8-8b5bcdbae512"} {"msg":"Crucible stats registered with UUID: a9af12b3-2c41-41c0-909a-b4c4b3953360","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:28.604303319Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1308,"session_id":"2b849fa5-eca3-4fb5-a8d8-8b5bcdbae512"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:28.635598121Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1308,"task":"crutest"} {"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:28.635629935Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1308} {"msg":"a9af12b3-2c41-41c0-909a-b4c4b3953360 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:28.635679081Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1308,"session_id":"2b849fa5-eca3-4fb5-a8d8-8b5bcdbae512"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:28.635698014Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1308,"client":"0","":"downstairs","session_id":"2b849fa5-eca3-4fb5-a8d8-8b5bcdbae512"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:28.635727187Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1308,"client":"0","":"downstairs","session_id":"2b849fa5-eca3-4fb5-a8d8-8b5bcdbae512"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:28.63574503Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1308,"client":"1","":"downstairs","session_id":"2b849fa5-eca3-4fb5-a8d8-8b5bcdbae512"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:28.635765583Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1308,"client":"1","":"downstairs","session_id":"2b849fa5-eca3-4fb5-a8d8-8b5bcdbae512"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:28.635796076Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1308,"client":"2","":"downstairs","session_id":"2b849fa5-eca3-4fb5-a8d8-8b5bcdbae512"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:28.635828631Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1308,"client":"2","":"downstairs","session_id":"2b849fa5-eca3-4fb5-a8d8-8b5bcdbae512"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:28.635861515Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1308,"":"io task","client":"2","":"downstairs","session_id":"2b849fa5-eca3-4fb5-a8d8-8b5bcdbae512"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:28.635894929Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1308,"":"io task","client":"0","":"downstairs","session_id":"2b849fa5-eca3-4fb5-a8d8-8b5bcdbae512"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:28.635931314Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1308,"":"io task","client":"1","":"downstairs","session_id":"2b849fa5-eca3-4fb5-a8d8-8b5bcdbae512"} {"msg":"ds_connection connected from Ok(127.0.0.1:47334)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:28.635964838Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1308,"":"io task","client":"2","":"downstairs","session_id":"2b849fa5-eca3-4fb5-a8d8-8b5bcdbae512"} {"msg":"ds_connection connected from Ok(127.0.0.1:37153)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:28.636091785Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1308,"":"io task","client":"0","":"downstairs","session_id":"2b849fa5-eca3-4fb5-a8d8-8b5bcdbae512"} {"msg":"ds_connection connected from Ok(127.0.0.1:45936)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:28.636138991Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1308,"":"io task","client":"1","":"downstairs","session_id":"2b849fa5-eca3-4fb5-a8d8-8b5bcdbae512"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:28.636710284Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1308,"client":"1","":"downstairs","session_id":"2b849fa5-eca3-4fb5-a8d8-8b5bcdbae512"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:28.636738318Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1308,"client":"1","":"downstairs","session_id":"2b849fa5-eca3-4fb5-a8d8-8b5bcdbae512"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:28.63675304Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1308,"client":"2","":"downstairs","session_id":"2b849fa5-eca3-4fb5-a8d8-8b5bcdbae512"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:28.636767032Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1308,"client":"2","":"downstairs","session_id":"2b849fa5-eca3-4fb5-a8d8-8b5bcdbae512"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:28.636780623Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1308,"client":"0","":"downstairs","session_id":"2b849fa5-eca3-4fb5-a8d8-8b5bcdbae512"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:28.636804176Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1308,"client":"0","":"downstairs","session_id":"2b849fa5-eca3-4fb5-a8d8-8b5bcdbae512"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:28.637406294Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1308,"client":"1","":"downstairs","session_id":"2b849fa5-eca3-4fb5-a8d8-8b5bcdbae512"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:28.637577686Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1308,"client":"0","":"downstairs","session_id":"2b849fa5-eca3-4fb5-a8d8-8b5bcdbae512"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:28.63761092Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1308,"client":"2","":"downstairs","session_id":"2b849fa5-eca3-4fb5-a8d8-8b5bcdbae512"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:28.638018793Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1308,"client":"1","":"downstairs","session_id":"2b849fa5-eca3-4fb5-a8d8-8b5bcdbae512"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:28.638051157Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1308,"client":"1","":"downstairs","session_id":"2b849fa5-eca3-4fb5-a8d8-8b5bcdbae512"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:28.638071Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1308,"session_id":"2b849fa5-eca3-4fb5-a8d8-8b5bcdbae512"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:28.638139608Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1308,"client":"0","":"downstairs","session_id":"2b849fa5-eca3-4fb5-a8d8-8b5bcdbae512"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:28.638159781Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1308,"client":"0","":"downstairs","session_id":"2b849fa5-eca3-4fb5-a8d8-8b5bcdbae512"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:28.638174243Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1308,"session_id":"2b849fa5-eca3-4fb5-a8d8-8b5bcdbae512"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:28.638211368Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1308,"client":"2","":"downstairs","session_id":"2b849fa5-eca3-4fb5-a8d8-8b5bcdbae512"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:28.63822858Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1308,"client":"2","":"downstairs","session_id":"2b849fa5-eca3-4fb5-a8d8-8b5bcdbae512"} {"msg":"[0]R flush_numbers[0..12]: [47, 49, 49, 40, 45, 36, 45, 25, 45, 45, 49, 49]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:28.638242542Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1308,"":"downstairs","":"downstairs","session_id":"2b849fa5-eca3-4fb5-a8d8-8b5bcdbae512"} {"msg":"[0]R generation[0..12]: [30, 31, 31, 26, 29, 24, 29, 18, 29, 29, 31, 31]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:28.638256433Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1308,"":"downstairs","":"downstairs","session_id":"2b849fa5-eca3-4fb5-a8d8-8b5bcdbae512"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:28.638270875Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1308,"":"downstairs","":"downstairs","session_id":"2b849fa5-eca3-4fb5-a8d8-8b5bcdbae512"} {"msg":"[1]R flush_numbers[0..12]: [47, 49, 49, 40, 45, 36, 45, 25, 45, 45, 49, 49]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:28.638284617Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1308,"":"downstairs","":"downstairs","session_id":"2b849fa5-eca3-4fb5-a8d8-8b5bcdbae512"} {"msg":"[1]R generation[0..12]: [30, 31, 31, 26, 29, 24, 29, 18, 29, 29, 31, 31]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:28.63831022Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1308,"":"downstairs","":"downstairs","session_id":"2b849fa5-eca3-4fb5-a8d8-8b5bcdbae512"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:28.638324782Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1308,"":"downstairs","":"downstairs","session_id":"2b849fa5-eca3-4fb5-a8d8-8b5bcdbae512"} {"msg":"[2]R flush_numbers[0..12]: [47, 49, 49, 40, 45, 36, 45, 25, 45, 45, 49, 49]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:28.638338394Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1308,"":"downstairs","":"downstairs","session_id":"2b849fa5-eca3-4fb5-a8d8-8b5bcdbae512"} {"msg":"[2]R generation[0..12]: [30, 31, 31, 26, 29, 24, 29, 18, 29, 29, 31, 31]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:28.638352046Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1308,"":"downstairs","":"downstairs","session_id":"2b849fa5-eca3-4fb5-a8d8-8b5bcdbae512"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:28.638370008Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1308,"":"downstairs","":"downstairs","session_id":"2b849fa5-eca3-4fb5-a8d8-8b5bcdbae512"} {"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:28.638399332Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1308,"":"downstairs","":"downstairs","session_id":"2b849fa5-eca3-4fb5-a8d8-8b5bcdbae512"} {"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:28.638426725Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1308,"":"downstairs","":"downstairs","session_id":"2b849fa5-eca3-4fb5-a8d8-8b5bcdbae512"} {"msg":"Next flush: 50","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:28.638444898Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1308,"":"downstairs","":"downstairs","session_id":"2b849fa5-eca3-4fb5-a8d8-8b5bcdbae512"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-01T16:51:28.638468651Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1308,"":"downstairs","":"downstairs","session_id":"2b849fa5-eca3-4fb5-a8d8-8b5bcdbae512"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:28.638495894Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1308,"session_id":"2b849fa5-eca3-4fb5-a8d8-8b5bcdbae512"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:28.638520518Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1308,"client":"0","":"downstairs","session_id":"2b849fa5-eca3-4fb5-a8d8-8b5bcdbae512"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:28.638536809Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1308,"client":"1","":"downstairs","session_id":"2b849fa5-eca3-4fb5-a8d8-8b5bcdbae512"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:28.638550071Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1308,"client":"2","":"downstairs","session_id":"2b849fa5-eca3-4fb5-a8d8-8b5bcdbae512"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:28.638570514Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1308,"session_id":"2b849fa5-eca3-4fb5-a8d8-8b5bcdbae512"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:28.638594277Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1308,"session_id":"2b849fa5-eca3-4fb5-a8d8-8b5bcdbae512"} {"msg":"a9af12b3-2c41-41c0-909a-b4c4b3953360 is now active with session: 2b849fa5-eca3-4fb5-a8d8-8b5bcdbae512","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:28.63861996Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1308,"session_id":"2b849fa5-eca3-4fb5-a8d8-8b5bcdbae512"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:28.638639183Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1308,"session_id":"2b849fa5-eca3-4fb5-a8d8-8b5bcdbae512"} {"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:28.638652024Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1308} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 496 len 20480 data: 2 3 3 4 4 02/30 Flush 03/30 Write block 78 len 24576 data: 2 4 4 3 3 3 04/30 Write block 551 len 36864 data: 5 4 4 3 3 3 3 3 4 05/30 Read block 217 len 36864 06/30 Write block 557 len 32768 data: 4 4 5 3 3 3 3 3 07/30 Write block 279 len 20480 data: 3 3 3 3 2 08/30 Flush 09/30 Flush 10/30 Write block 199 len 4096 data: 4 11/30 Write block 302 len 28672 data: 3 4 5 6 6 6 6 12/30 Write block 353 len 28672 data: 5 4 4 4 4 3 3 13/30 Write block 478 len 20480 data: 6 6 6 4 3 14/30 Write block 133 len 40960 data: 7 7 6 4 4 4 3 3 3 3 15/30 Read block 368 len 4096 16/30 Write block 364 len 28672 data: 2 3 3 2 2 4 3 17/30 Read block 268 len 24576 18/30 Write block 343 len 16384 data: 3 4 4 4 19/30 Flush 20/30 Write block 554 len 24576 data: 4 4 4 5 5 6 21/30 Read block 453 len 32768 22/30 Read block 50 len 8192 23/30 Read block 406 len 20480 24/30 Read block 589 len 32768 25/30 Flush 26/30 Write block 24 len 32768 data: 3 3 3 3 3 2 2 3 27/30 Read block 367 len 28672 28/30 Write block 340 len 20480 data: 3 3 3 4 5 29/30 Write block 82 len 40960 data: 4 4 3 4 3 3 3 4 6 4 30/30 Write block 38 len 24576 data: 4 4 3 4 6 7 ---------------------------------------------------------------- Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 8 Done Done Done false 1026 Acked Read 7 Done Done Done false 1027 Acked Write 5 Done Done Done false 1028 Acked Write 10 Sent Sent Sent false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:51:28.666500383Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1308} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 30 30 30 47 47 47 F F F 1 020-039 31 31 31 49 49 49 T T T 2 040-059 31 31 31 49 49 49 T T T 3 060-079 32 32 32 51 51 51 F F F 4 080-099 32 32 32 51 51 51 T T T 5 100-119 24 24 24 36 36 36 F F F 6 120-139 32 32 32 53 53 53 F F F 7 140-159 32 32 32 53 53 53 F F F 8 160-179 29 29 29 45 45 45 F F F 9 180-199 32 32 32 53 53 53 F F F 10 200-219 31 31 31 49 49 49 F F F 11 220-239 31 31 31 49 49 49 F F F 12 240-259 30 30 30 47 47 47 F F F 13 260-279 32 32 32 51 51 51 F F F 14 280-299 32 32 32 51 51 51 F F F 15 300-319 32 32 32 53 53 53 F F F 16 320-339 30 30 30 46 46 46 F F F 17 340-359 32 32 32 53 53 53 T T T 18 360-379 32 32 32 53 53 53 F F F 19 380-399 30 30 30 48 48 48 F F F 20 400-419 27 27 27 41 41 41 F F F 21 420-439 31 31 31 49 49 49 F F F 22 440-459 24 24 24 36 36 36 F F F 23 460-479 32 32 32 53 53 53 F F F 24 480-499 32 32 32 53 53 53 F F F 25 500-519 32 32 32 50 50 50 F F F 26 520-539 30 30 30 48 48 48 F F F 27 540-559 32 32 32 54 54 54 F F F 28 560-579 32 32 32 51 51 51 F F F 29 580-599 22 22 22 34 34 34 F F F Max gen: 32, Max flush: 54 A Difference in extent metadata is expected here On loop 16 Verifying data now No change after verify Loop: 16 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 30 30 30 47 47 47 F F F 1 020-039 33 33 33 55 55 55 F F F 2 040-059 33 33 33 55 55 55 F F F 3 060-079 32 32 32 51 51 51 F F F 4 080-099 33 33 33 55 55 55 F F F 5 100-119 24 24 24 36 36 36 F F F 6 120-139 32 32 32 53 53 53 F F F 7 140-159 32 32 32 53 53 53 F F F 8 160-179 29 29 29 45 45 45 F F F 9 180-199 32 32 32 53 53 53 F F F 10 200-219 31 31 31 49 49 49 F F F 11 220-239 31 31 31 49 49 49 F F F 12 240-259 30 30 30 47 47 47 F F F 13 260-279 32 32 32 51 51 51 F F F 14 280-299 32 32 32 51 51 51 F F F 15 300-319 32 32 32 53 53 53 F F F 16 320-339 30 30 30 46 46 46 F F F 17 340-359 33 33 33 55 55 55 F F F 18 360-379 32 32 32 53 53 53 F F F 19 380-399 30 30 30 48 48 48 F F F 20 400-419 27 27 27 41 41 41 F F F 21 420-439 31 31 31 49 49 49 F F F 22 440-459 24 24 24 36 36 36 F F F 23 460-479 32 32 32 53 53 53 F F F 24 480-499 32 32 32 53 53 53 F F F 25 500-519 32 32 32 50 50 50 F F F 26 520-539 30 30 30 48 48 48 F F F 27 540-559 32 32 32 54 54 54 F F F 28 560-579 32 32 32 51 51 51 F F F 29 580-599 22 22 22 34 34 34 F F F Max gen: 33, Max flush: 55 Begin loop 17 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:32.989733278Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1316,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:51:33.032239695Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1316,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:33.032625654Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1316,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:33.064436053Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1316,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:33.06495834Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1316} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"3818f0fc76fe787c6cc412af30fd59b930203885\",\n git_commit_timestamp: \"2024-11-01T16:08:30.000000000Z\",\n git_branch: \"renovate/hyper-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:33.064994305Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1316} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:33.065008467Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1316} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:33.065022779Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1316} {"msg":"Crucible 9daaf0a2-3f75-4bf1-8231-70792ad1271e has session id: ad471b52-741e-46d4-aeb5-f42991e26edb","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:33.065433352Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1316,"session_id":"ad471b52-741e-46d4-aeb5-f42991e26edb"} {"msg":"Upstairs opts: Upstairs UUID: 9daaf0a2-3f75-4bf1-8231-70792ad1271e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:33.065476107Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1316,"session_id":"ad471b52-741e-46d4-aeb5-f42991e26edb"} {"msg":"Crucible stats registered with UUID: 9daaf0a2-3f75-4bf1-8231-70792ad1271e","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:33.065507931Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1316,"session_id":"ad471b52-741e-46d4-aeb5-f42991e26edb"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:33.096244782Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1316,"task":"crutest"} {"msg":"The guest has requested activation with gen:34","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:33.096287787Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1316} {"msg":"9daaf0a2-3f75-4bf1-8231-70792ad1271e active request set","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:33.096354206Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1316,"session_id":"ad471b52-741e-46d4-aeb5-f42991e26edb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:33.09638855Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1316,"client":"0","":"downstairs","session_id":"ad471b52-741e-46d4-aeb5-f42991e26edb"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:33.096419764Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1316,"client":"0","":"downstairs","session_id":"ad471b52-741e-46d4-aeb5-f42991e26edb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:33.096438167Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1316,"client":"1","":"downstairs","session_id":"ad471b52-741e-46d4-aeb5-f42991e26edb"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:33.096455619Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1316,"client":"1","":"downstairs","session_id":"ad471b52-741e-46d4-aeb5-f42991e26edb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:33.096475441Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1316,"client":"2","":"downstairs","session_id":"ad471b52-741e-46d4-aeb5-f42991e26edb"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:33.096505085Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1316,"client":"2","":"downstairs","session_id":"ad471b52-741e-46d4-aeb5-f42991e26edb"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:33.096535809Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1316,"":"io task","client":"2","":"downstairs","session_id":"ad471b52-741e-46d4-aeb5-f42991e26edb"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:33.096567343Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1316,"":"io task","client":"0","":"downstairs","session_id":"ad471b52-741e-46d4-aeb5-f42991e26edb"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:33.096600457Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1316,"":"io task","client":"1","":"downstairs","session_id":"ad471b52-741e-46d4-aeb5-f42991e26edb"} {"msg":"ds_connection connected from Ok(127.0.0.1:44100)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:33.096632512Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1316,"":"io task","client":"2","":"downstairs","session_id":"ad471b52-741e-46d4-aeb5-f42991e26edb"} {"msg":"ds_connection connected from Ok(127.0.0.1:58691)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:33.096666676Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1316,"":"io task","client":"1","":"downstairs","session_id":"ad471b52-741e-46d4-aeb5-f42991e26edb"} {"msg":"ds_connection connected from Ok(127.0.0.1:41516)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:33.096838558Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1316,"":"io task","client":"0","":"downstairs","session_id":"ad471b52-741e-46d4-aeb5-f42991e26edb"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:33.097348704Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1316,"client":"1","":"downstairs","session_id":"ad471b52-741e-46d4-aeb5-f42991e26edb"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:33.097375167Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1316,"client":"1","":"downstairs","session_id":"ad471b52-741e-46d4-aeb5-f42991e26edb"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:33.097508624Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1316,"client":"0","":"downstairs","session_id":"ad471b52-741e-46d4-aeb5-f42991e26edb"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:33.097531127Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1316,"client":"0","":"downstairs","session_id":"ad471b52-741e-46d4-aeb5-f42991e26edb"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:33.097638321Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1316,"client":"2","":"downstairs","session_id":"ad471b52-741e-46d4-aeb5-f42991e26edb"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:33.097671995Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1316,"client":"2","":"downstairs","session_id":"ad471b52-741e-46d4-aeb5-f42991e26edb"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:33.098126094Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1316,"client":"1","":"downstairs","session_id":"ad471b52-741e-46d4-aeb5-f42991e26edb"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:33.098229497Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1316,"client":"0","":"downstairs","session_id":"ad471b52-741e-46d4-aeb5-f42991e26edb"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:33.098259091Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1316,"client":"2","":"downstairs","session_id":"ad471b52-741e-46d4-aeb5-f42991e26edb"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:33.098692137Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1316,"client":"1","":"downstairs","session_id":"ad471b52-741e-46d4-aeb5-f42991e26edb"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:33.09872077Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1316,"client":"1","":"downstairs","session_id":"ad471b52-741e-46d4-aeb5-f42991e26edb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:33.098735563Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1316,"session_id":"ad471b52-741e-46d4-aeb5-f42991e26edb"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:33.098801851Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1316,"client":"0","":"downstairs","session_id":"ad471b52-741e-46d4-aeb5-f42991e26edb"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:33.098822564Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1316,"client":"0","":"downstairs","session_id":"ad471b52-741e-46d4-aeb5-f42991e26edb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:33.098839566Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1316,"session_id":"ad471b52-741e-46d4-aeb5-f42991e26edb"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:33.098852427Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1316,"client":"2","":"downstairs","session_id":"ad471b52-741e-46d4-aeb5-f42991e26edb"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:33.098865909Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1316,"client":"2","":"downstairs","session_id":"ad471b52-741e-46d4-aeb5-f42991e26edb"} {"msg":"[0]R flush_numbers[0..12]: [47, 55, 55, 51, 55, 36, 53, 53, 45, 53, 49, 49]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:33.098879801Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1316,"":"downstairs","":"downstairs","session_id":"ad471b52-741e-46d4-aeb5-f42991e26edb"} {"msg":"[0]R generation[0..12]: [30, 33, 33, 32, 33, 24, 32, 32, 29, 32, 31, 31]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:33.098893823Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1316,"":"downstairs","":"downstairs","session_id":"ad471b52-741e-46d4-aeb5-f42991e26edb"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:33.098907915Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1316,"":"downstairs","":"downstairs","session_id":"ad471b52-741e-46d4-aeb5-f42991e26edb"} {"msg":"[1]R flush_numbers[0..12]: [47, 55, 55, 51, 55, 36, 53, 53, 45, 53, 49, 49]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:33.098921736Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1316,"":"downstairs","":"downstairs","session_id":"ad471b52-741e-46d4-aeb5-f42991e26edb"} {"msg":"[1]R generation[0..12]: [30, 33, 33, 32, 33, 24, 32, 32, 29, 32, 31, 31]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:33.098935628Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1316,"":"downstairs","":"downstairs","session_id":"ad471b52-741e-46d4-aeb5-f42991e26edb"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:33.09894975Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1316,"":"downstairs","":"downstairs","session_id":"ad471b52-741e-46d4-aeb5-f42991e26edb"} {"msg":"[2]R flush_numbers[0..12]: [47, 55, 55, 51, 55, 36, 53, 53, 45, 53, 49, 49]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:33.098963592Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1316,"":"downstairs","":"downstairs","session_id":"ad471b52-741e-46d4-aeb5-f42991e26edb"} {"msg":"[2]R generation[0..12]: [30, 33, 33, 32, 33, 24, 32, 32, 29, 32, 31, 31]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:33.098977244Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1316,"":"downstairs","":"downstairs","session_id":"ad471b52-741e-46d4-aeb5-f42991e26edb"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:33.098991165Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1316,"":"downstairs","":"downstairs","session_id":"ad471b52-741e-46d4-aeb5-f42991e26edb"} {"msg":"Max found gen is 34","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:33.099004607Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1316,"":"downstairs","":"downstairs","session_id":"ad471b52-741e-46d4-aeb5-f42991e26edb"} {"msg":"Generation requested: 34 >= found:34","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:33.099017959Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1316,"":"downstairs","":"downstairs","session_id":"ad471b52-741e-46d4-aeb5-f42991e26edb"} {"msg":"Next flush: 56","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-01T16:51:33.099033321Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1316,"":"downstairs","":"downstairs","session_id":"ad471b52-741e-46d4-aeb5-f42991e26edb"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:33.099052043Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1316,"":"downstairs","":"downstairs","session_id":"ad471b52-741e-46d4-aeb5-f42991e26edb"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:33.099065505Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1316,"session_id":"ad471b52-741e-46d4-aeb5-f42991e26edb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:33.099077607Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1316,"client":"0","":"downstairs","session_id":"ad471b52-741e-46d4-aeb5-f42991e26edb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:33.099090868Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1316,"client":"1","":"downstairs","session_id":"ad471b52-741e-46d4-aeb5-f42991e26edb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:33.09910415Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1316,"client":"2","":"downstairs","session_id":"ad471b52-741e-46d4-aeb5-f42991e26edb"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:33.099117522Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1316,"session_id":"ad471b52-741e-46d4-aeb5-f42991e26edb"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:33.099129963Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1316,"session_id":"ad471b52-741e-46d4-aeb5-f42991e26edb"} {"msg":"9daaf0a2-3f75-4bf1-8231-70792ad1271e is now active with session: ad471b52-741e-46d4-aeb5-f42991e26edb","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:33.099142885Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1316,"session_id":"ad471b52-741e-46d4-aeb5-f42991e26edb"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:33.099154977Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1316,"session_id":"ad471b52-741e-46d4-aeb5-f42991e26edb"} {"msg":"The guest has finished waiting for activation with:34","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:33.099167198Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1316} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 304 len 28672 data: 6 7 7 7 7 3 3 02/30 Read block 334 len 4096 03/30 Read block 319 len 8192 04/30 Write block 269 len 28672 data: 4 3 5 4 6 7 6 05/30 Write block 66 len 40960 data: 9 8 8 8 10 8 7 6 5 4 06/30 Read block 141 len 28672 07/30 Write block 82 len 40960 data: 5 5 4 5 4 4 4 5 7 5 08/30 Read block 369 len 20480 09/30 Write block 435 len 16384 data: 2 3 3 4 10/30 Read block 57 len 12288 11/30 Write block 23 len 12288 data: 4 4 4 12/30 Flush 13/30 Flush 14/30 Write block 274 len 24576 data: 8 7 5 4 3 4 15/30 Write block 218 len 4096 data: 6 16/30 Read block 233 len 36864 17/30 Read block 156 len 20480 18/30 Read block 386 len 36864 19/30 Read block 438 len 40960 20/30 Read block 109 len 28672 21/30 Read block 555 len 24576 22/30 Write block 577 len 28672 data: 4 4 5 5 4 4 3 23/30 Read block 26 len 40960 24/30 Read block 424 len 20480 25/30 Write block 16 len 36864 data: 3 3 3 4 4 2 3 5 5 26/30 Write block 46 len 4096 data: 5 27/30 Read block 193 len 16384 28/30 Write block 222 len 24576 data: 4 5 5 5 4 3 29/30 Read block 387 len 8192 30/30 Read block 232 len 8192 ---------------------------------------------------------------- Crucible gen:34 GIO:true work queues: Upstairs:0 downstairs:17 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1013 Acked Write 6 Done Done Done false 1014 Acked Write 1 Done Done Done false 1015 Acked Read 9 Done Done Done false 1016 Acked Read 5 Done Done Done false 1017 Acked Read 9 Done Done Done false 1018 Acked Read 10 Done Done Done false 1019 Acked Read 7 Done Done Done false 1020 Acked Read 6 Done Done Done false 1021 Acked Write 7 Done Done Done false 1022 Acked Read 10 Done Done Done false 1023 Acked Read 5 Done Done Done false 1024 Acked Write 9 Done Done Done false 1025 Acked Write 1 Done Done Done false 1026 Acked Read 4 Done Done Done false 1027 Acked Write 6 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Read 2 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 17 16 16 49 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1012 1012 1012 Downstairs last five completed: 1012 1011 1010 1009 1008 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:51:33.12409796Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1316} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 30 30 30 47 47 47 T T T 1 020-039 34 34 34 56 56 56 T T T 2 040-059 33 33 33 55 55 55 T T T 3 060-079 34 34 34 56 56 56 F F F 4 080-099 34 34 34 56 56 56 F F F 5 100-119 24 24 24 36 36 36 F F F 6 120-139 32 32 32 53 53 53 F F F 7 140-159 32 32 32 53 53 53 F F F 8 160-179 29 29 29 45 45 45 F F F 9 180-199 32 32 32 53 53 53 F F F 10 200-219 31 31 31 49 49 49 T T T 11 220-239 31 31 31 49 49 49 T T T 12 240-259 30 30 30 47 47 47 F F F 13 260-279 34 34 34 56 56 56 T T T 14 280-299 32 32 32 51 51 51 F F F 15 300-319 34 34 34 56 56 56 F F F 16 320-339 30 30 30 46 46 46 F F F 17 340-359 33 33 33 55 55 55 F F F 18 360-379 32 32 32 53 53 53 F F F 19 380-399 30 30 30 48 48 48 F F F 20 400-419 27 27 27 41 41 41 F F F 21 420-439 34 34 34 56 56 56 F F F 22 440-459 24 24 24 36 36 36 F F F 23 460-479 32 32 32 53 53 53 F F F 24 480-499 32 32 32 53 53 53 F F F 25 500-519 32 32 32 50 50 50 F F F 26 520-539 30 30 30 48 48 48 F F F 27 540-559 32 32 32 54 54 54 F F F 28 560-579 32 32 32 51 51 51 T T T 29 580-599 22 22 22 34 34 34 T T T Max gen: 34, Max flush: 56 A Difference in extent metadata is expected here On loop 17 Verifying data now No change after verify Loop: 17 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 35 35 35 57 57 57 F F F 1 020-039 35 35 35 57 57 57 F F F 2 040-059 35 35 35 57 57 57 F F F 3 060-079 34 34 34 56 56 56 F F F 4 080-099 34 34 34 56 56 56 F F F 5 100-119 24 24 24 36 36 36 F F F 6 120-139 32 32 32 53 53 53 F F F 7 140-159 32 32 32 53 53 53 F F F 8 160-179 29 29 29 45 45 45 F F F 9 180-199 32 32 32 53 53 53 F F F 10 200-219 35 35 35 57 57 57 F F F 11 220-239 35 35 35 57 57 57 F F F 12 240-259 30 30 30 47 47 47 F F F 13 260-279 35 35 35 57 57 57 F F F 14 280-299 32 32 32 51 51 51 F F F 15 300-319 34 34 34 56 56 56 F F F 16 320-339 30 30 30 46 46 46 F F F 17 340-359 33 33 33 55 55 55 F F F 18 360-379 32 32 32 53 53 53 F F F 19 380-399 30 30 30 48 48 48 F F F 20 400-419 27 27 27 41 41 41 F F F 21 420-439 34 34 34 56 56 56 F F F 22 440-459 24 24 24 36 36 36 F F F 23 460-479 32 32 32 53 53 53 F F F 24 480-499 32 32 32 53 53 53 F F F 25 500-519 32 32 32 50 50 50 F F F 26 520-539 30 30 30 48 48 48 F F F 27 540-559 32 32 32 54 54 54 F F F 28 560-579 35 35 35 57 57 57 F F F 29 580-599 35 35 35 57 57 57 F F F Max gen: 35, Max flush: 57 Begin loop 18 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:37.488732592Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1324,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:37.533526964Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1324,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:37.534159816Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1324} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"3818f0fc76fe787c6cc412af30fd59b930203885\",\n git_commit_timestamp: \"2024-11-01T16:08:30.000000000Z\",\n git_branch: \"renovate/hyper-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:37.5341944Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1324} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:37.534208392Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1324} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:37.534222704Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1324} {"msg":"Crucible f1f95477-a826-47c3-bf3a-33603b799fda has session id: 1fb2d9bc-69c0-4eb3-bc1b-7d1e66e0e633","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:37.534703036Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1324,"session_id":"1fb2d9bc-69c0-4eb3-bc1b-7d1e66e0e633"} {"msg":"Upstairs opts: Upstairs UUID: f1f95477-a826-47c3-bf3a-33603b799fda, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:37.534750842Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1324,"session_id":"1fb2d9bc-69c0-4eb3-bc1b-7d1e66e0e633"} {"msg":"Crucible stats registered with UUID: f1f95477-a826-47c3-bf3a-33603b799fda","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:37.534780316Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1324,"session_id":"1fb2d9bc-69c0-4eb3-bc1b-7d1e66e0e633"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:37.565532488Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1324,"task":"crutest"} {"msg":"The guest has requested activation with gen:36","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:37.565563352Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1324} {"msg":"f1f95477-a826-47c3-bf3a-33603b799fda active request set","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:37.56563024Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1324,"session_id":"1fb2d9bc-69c0-4eb3-bc1b-7d1e66e0e633"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:37.565652963Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1324,"client":"0","":"downstairs","session_id":"1fb2d9bc-69c0-4eb3-bc1b-7d1e66e0e633"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:37.565681347Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1324,"client":"0","":"downstairs","session_id":"1fb2d9bc-69c0-4eb3-bc1b-7d1e66e0e633"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:37.565698469Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1324,"client":"1","":"downstairs","session_id":"1fb2d9bc-69c0-4eb3-bc1b-7d1e66e0e633"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:37.565712521Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1324,"client":"1","":"downstairs","session_id":"1fb2d9bc-69c0-4eb3-bc1b-7d1e66e0e633"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:37.565726003Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1324,"client":"2","":"downstairs","session_id":"1fb2d9bc-69c0-4eb3-bc1b-7d1e66e0e633"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:37.565833557Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1324,"client":"2","":"downstairs","session_id":"1fb2d9bc-69c0-4eb3-bc1b-7d1e66e0e633"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:37.56586132Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1324,"":"io task","client":"2","":"downstairs","session_id":"1fb2d9bc-69c0-4eb3-bc1b-7d1e66e0e633"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:37.565891294Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1324,"":"io task","client":"0","":"downstairs","session_id":"1fb2d9bc-69c0-4eb3-bc1b-7d1e66e0e633"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:37.565920188Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1324,"":"io task","client":"1","":"downstairs","session_id":"1fb2d9bc-69c0-4eb3-bc1b-7d1e66e0e633"} {"msg":"ds_connection connected from Ok(127.0.0.1:39751)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:37.566024831Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1324,"":"io task","client":"1","":"downstairs","session_id":"1fb2d9bc-69c0-4eb3-bc1b-7d1e66e0e633"} {"msg":"ds_connection connected from Ok(127.0.0.1:56907)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:37.566051825Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1324,"":"io task","client":"2","":"downstairs","session_id":"1fb2d9bc-69c0-4eb3-bc1b-7d1e66e0e633"} {"msg":"ds_connection connected from Ok(127.0.0.1:38434)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:37.566077238Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1324,"":"io task","client":"0","":"downstairs","session_id":"1fb2d9bc-69c0-4eb3-bc1b-7d1e66e0e633"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:37.566656813Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1324,"client":"1","":"downstairs","session_id":"1fb2d9bc-69c0-4eb3-bc1b-7d1e66e0e633"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:37.566702299Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1324,"client":"1","":"downstairs","session_id":"1fb2d9bc-69c0-4eb3-bc1b-7d1e66e0e633"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:37.566734963Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1324,"client":"2","":"downstairs","session_id":"1fb2d9bc-69c0-4eb3-bc1b-7d1e66e0e633"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:37.566765797Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1324,"client":"2","":"downstairs","session_id":"1fb2d9bc-69c0-4eb3-bc1b-7d1e66e0e633"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:37.566797011Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1324,"client":"0","":"downstairs","session_id":"1fb2d9bc-69c0-4eb3-bc1b-7d1e66e0e633"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:37.566823624Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1324,"client":"0","":"downstairs","session_id":"1fb2d9bc-69c0-4eb3-bc1b-7d1e66e0e633"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:37.567328329Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1324,"client":"1","":"downstairs","session_id":"1fb2d9bc-69c0-4eb3-bc1b-7d1e66e0e633"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:37.56741322Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1324,"client":"0","":"downstairs","session_id":"1fb2d9bc-69c0-4eb3-bc1b-7d1e66e0e633"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:37.567434763Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1324,"client":"2","":"downstairs","session_id":"1fb2d9bc-69c0-4eb3-bc1b-7d1e66e0e633"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:37.567991755Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1324,"client":"0","":"downstairs","session_id":"1fb2d9bc-69c0-4eb3-bc1b-7d1e66e0e633"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:37.568020648Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1324,"client":"0","":"downstairs","session_id":"1fb2d9bc-69c0-4eb3-bc1b-7d1e66e0e633"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:37.56803575Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1324,"session_id":"1fb2d9bc-69c0-4eb3-bc1b-7d1e66e0e633"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:37.568048732Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1324,"client":"1","":"downstairs","session_id":"1fb2d9bc-69c0-4eb3-bc1b-7d1e66e0e633"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:37.568062074Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1324,"client":"1","":"downstairs","session_id":"1fb2d9bc-69c0-4eb3-bc1b-7d1e66e0e633"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:37.568075235Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1324,"session_id":"1fb2d9bc-69c0-4eb3-bc1b-7d1e66e0e633"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:37.568087467Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1324,"client":"2","":"downstairs","session_id":"1fb2d9bc-69c0-4eb3-bc1b-7d1e66e0e633"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:37.568100979Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1324,"client":"2","":"downstairs","session_id":"1fb2d9bc-69c0-4eb3-bc1b-7d1e66e0e633"} {"msg":"[0]R flush_numbers[0..12]: [57, 57, 57, 56, 56, 36, 53, 53, 45, 53, 57, 57]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:37.568118001Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1324,"":"downstairs","":"downstairs","session_id":"1fb2d9bc-69c0-4eb3-bc1b-7d1e66e0e633"} {"msg":"[0]R generation[0..12]: [35, 35, 35, 34, 34, 24, 32, 32, 29, 32, 35, 35]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:37.568132363Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1324,"":"downstairs","":"downstairs","session_id":"1fb2d9bc-69c0-4eb3-bc1b-7d1e66e0e633"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:37.568146425Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1324,"":"downstairs","":"downstairs","session_id":"1fb2d9bc-69c0-4eb3-bc1b-7d1e66e0e633"} {"msg":"[1]R flush_numbers[0..12]: [57, 57, 57, 56, 56, 36, 53, 53, 45, 53, 57, 57]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:37.568160197Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1324,"":"downstairs","":"downstairs","session_id":"1fb2d9bc-69c0-4eb3-bc1b-7d1e66e0e633"} {"msg":"[1]R generation[0..12]: [35, 35, 35, 34, 34, 24, 32, 32, 29, 32, 35, 35]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:37.568173638Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1324,"":"downstairs","":"downstairs","session_id":"1fb2d9bc-69c0-4eb3-bc1b-7d1e66e0e633"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:37.5681875Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1324,"":"downstairs","":"downstairs","session_id":"1fb2d9bc-69c0-4eb3-bc1b-7d1e66e0e633"} {"msg":"[2]R flush_numbers[0..12]: [57, 57, 57, 56, 56, 36, 53, 53, 45, 53, 57, 57]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:37.568201572Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1324,"":"downstairs","":"downstairs","session_id":"1fb2d9bc-69c0-4eb3-bc1b-7d1e66e0e633"} {"msg":"[2]R generation[0..12]: [35, 35, 35, 34, 34, 24, 32, 32, 29, 32, 35, 35]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:37.568215424Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1324,"":"downstairs","":"downstairs","session_id":"1fb2d9bc-69c0-4eb3-bc1b-7d1e66e0e633"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:37.568229435Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1324,"":"downstairs","":"downstairs","session_id":"1fb2d9bc-69c0-4eb3-bc1b-7d1e66e0e633"} {"msg":"Max found gen is 36","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:37.568242557Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1324,"":"downstairs","":"downstairs","session_id":"1fb2d9bc-69c0-4eb3-bc1b-7d1e66e0e633"} {"msg":"Generation requested: 36 >= found:36","v":Wait for a query_work_queue command to finish before sending IO 0,"name":"crucible","level":30,"time":"2024-11-01T16:51:37.5682641Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1324,"":"downstairs","":"downstairs","session_id":"1fb2d9bc-69c0-4eb3-bc1b-7d1e66e0e633"} {"msg":"Next flush: 58","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:37.568277342Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1324,"":"downstairs","":"downstairs","session_id":"1fb2d9bc-69c0-4eb3-bc1b-7d1e66e0e633"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:37.568290363Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1324,"":"downstairs","":"downstairs","session_id":"1fb2d9bc-69c0-4eb3-bc1b-7d1e66e0e633"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:37.568303345Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1324,"session_id":"1fb2d9bc-69c0-4eb3-bc1b-7d1e66e0e633"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:37.568317007Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1324,"client":"0","":"downstairs","session_id":"1fb2d9bc-69c0-4eb3-bc1b-7d1e66e0e633"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:37.568330419Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1324,"client":"1","":"downstairs","session_id":"1fb2d9bc-69c0-4eb3-bc1b-7d1e66e0e633"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:37.56834342Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1324,"client":"2","":"downstairs","session_id":"1fb2d9bc-69c0-4eb3-bc1b-7d1e66e0e633"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:37.568356792Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1324,"session_id":"1fb2d9bc-69c0-4eb3-bc1b-7d1e66e0e633"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:37.568369213Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1324,"session_id":"1fb2d9bc-69c0-4eb3-bc1b-7d1e66e0e633"} {"msg":"f1f95477-a826-47c3-bf3a-33603b799fda is now active with session: 1fb2d9bc-69c0-4eb3-bc1b-7d1e66e0e633","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:37.568382125Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1324,"session_id":"1fb2d9bc-69c0-4eb3-bc1b-7d1e66e0e633"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:37.568394157Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1324,"session_id":"1fb2d9bc-69c0-4eb3-bc1b-7d1e66e0e633"} {"msg":"The guest has finished waiting for activation with:36","v":0,"name":"crucible","level":30Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2024-11-01T16:51:37.568406398Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1324} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 160 len 36864 data: 3 3 3 3 2 2 2 2 3 02/30 Read block 520 len 16384 03/30 Read block 74 len 16384 04/30 Read block 349 len 28672 05/30 Write block 477 len 36864 data: 5 7 7 7 5 4 3 3 3 06/30 Write block 245 len 40960 data: 4 5 7 7 7 8 7 6 6 4 07/30 Write block 543 len 36864 data: 3 3 3 4 4 4 4 3 6 08/30 Read block 385 len 32768 09/30 Read block 282 len 36864 10/30 Read block 137 len 36864 11/30 Write block 194 len 36864 data: 5 5 5 4 4 5 3 4 4 12/30 Read block 95 len 16384 13/30 Read block 446 len 28672 14/30 Write block 43 len 40960 data: 8 7 5 6 3 2 5 4 4 4 15/30 Read block 300 len 32768 16/30 Write block 462 len 20480 data: 3 3 3 2 3 17/30 Write block 289 len 4096 data: 2 18/30 Read block 345 len 40960 19/30 Read block 539 len 8192 20/30 Flush 21/30 Write block 96 len 16384 data: 4 4 4 5 22/30 Write block 160 len 32768 data: 4 4 4 4 3 3 3 3 23/30 Write block 234 len 20480 data: 3 3 4 4 4 24/30 Read block 418 len 24576 25/30 Write block 82 len 8192 data: 6 6 26/30 Flush 27/30 Write block 216 len 36864 data: 5 5 7 4 4 4 5 6 6 28/30 Write block 70 len 12288 data: 11 9 8 29/30 Write block 137 len 36864 data: 5 5 4 4 4 4 2 3 3 30/30 Write block 275 len 8192 data: 8 6 ---------------------------------------------------------------- Crucible gen:36 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 9 Done Sent Sent false 1027 Acked Write 3 Sent Sent Sent false 1028 Acked Write 9 Sent Sent Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 4 4 11 Done 1 0 0 1 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:51:37.595996743Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1324} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 35 35 35 57 57 57 F F F 1 020-039 35 35 35 57 57 57 F F F 2 040-059 36 36 36 58 58 58 F F F 3 060-079 34 34 34 56 56 56 T T T 4 080-099 36 36 36 59 59 59 F F F 5 100-119 24 24 24 36 36 36 F F F 6 120-139 32 32 32 53 53 53 T T T 7 140-159 32 32 32 53 53 53 T T T 8 160-179 36 36 36 59 59 59 F F F 9 180-199 36 36 36 58 58 58 F F F 10 200-219 36 36 36 58 58 58 T T T 11 220-239 36 36 36 59 59 59 T T T 12 240-259 36 36 36 58 58 58 F F F 13 260-279 35 35 35 57 57 57 T T T 14 280-299 36 36 36 58 58 58 F F F 15 300-319 34 34 34 56 56 56 F F F 16 320-339 30 30 30 46 46 46 F F F 17 340-359 33 33 33 55 55 55 F F F 18 360-379 32 32 32 53 53 53 F F F 19 380-399 30 30 30 48 48 48 F F F 20 400-419 27 27 27 41 41 41 F F F 21 420-439 34 34 34 56 56 56 F F F 22 440-459 24 24 24 36 36 36 F F F 23 460-479 36 36 36 58 58 58 F F F 24 480-499 36 36 36 58 58 58 F F F 25 500-519 32 32 32 50 50 50 F F F 26 520-539 30 30 30 48 48 48 F F F 27 540-559 36 36 36 58 58 58 F F F 28 560-579 35 35 35 57 57 57 F F F 29 580-599 35 35 35 57 57 57 F F F Max gen: 36, Max flush: 59 A Difference in extent metadata is expected here On loop 18 Verifying data now No change after verify Loop: 18 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 35 35 35 57 57 57 F F F 1 020-039 35 35 35 57 57 57 F F F 2 040-059 36 36 36 58 58 58 F F F 3 060-079 37 37 37 60 60 60 F F F 4 080-099 36 36 36 59 59 59 F F F 5 100-119 24 24 24 36 36 36 F F F 6 120-139 37 37 37 60 60 60 F F F 7 140-159 37 37 37 60 60 60 F F F 8 160-179 36 36 36 59 59 59 F F F 9 180-199 36 36 36 58 58 58 F F F 10 200-219 37 37 37 60 60 60 F F F 11 220-239 37 37 37 60 60 60 F F F 12 240-259 36 36 36 58 58 58 F F F 13 260-279 37 37 37 60 60 60 F F F 14 280-299 36 36 36 58 58 58 F F F 15 300-319 34 34 34 56 56 56 F F F 16 320-339 30 30 30 46 46 46 F F F 17 340-359 33 33 33 55 55 55 F F F 18 360-379 32 32 32 53 53 53 F F F 19 380-399 30 30 30 48 48 48 F F F 20 400-419 27 27 27 41 41 41 F F F 21 420-439 34 34 34 56 56 56 F F F 22 440-459 24 24 24 36 36 36 F F F 23 460-479 36 36 36 58 58 58 F F F 24 480-499 36 36 36 58 58 58 F F F 25 500-519 32 32 32 50 50 50 F F F 26 520-539 30 30 30 48 48 48 F F F 27 540-559 36 36 36 58 58 58 F F F 28 560-579 35 35 35 57 57 57 F F F 29 580-599 35 35 35 57 57 57 F F F Max gen: 37, Max flush: 60 Begin loop 19 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:41.910601089Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1332,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:41.955000639Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1332,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:41.955622079Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1332} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"3818f0fc76fe787c6cc412af30fd59b930203885\",\n git_commit_timestamp: \"2024-11-01T16:08:30.000000000Z\",\n git_branch: \"renovate/hyper-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:41.955671896Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1332} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:41.95569952Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1332} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:41.955717932Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1332} {"msg":"Crucible 4af065e7-0f0f-4d61-bdad-df8d040b4731 has session id: a698d3f2-6eed-4876-bdf2-4a3a1e4159d0","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:41.956118083Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1332,"session_id":"a698d3f2-6eed-4876-bdf2-4a3a1e4159d0"} {"msg":"Upstairs opts: Upstairs UUID: 4af065e7-0f0f-4d61-bdad-df8d040b4731, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:41.956149938Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1332,"session_id":"a698d3f2-6eed-4876-bdf2-4a3a1e4159d0"} {"msg":"Crucible stats registered with UUID: 4af065e7-0f0f-4d61-bdad-df8d040b4731","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:41.95616659Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1332,"session_id":"a698d3f2-6eed-4876-bdf2-4a3a1e4159d0"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:41.98706175Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1332,"task":"crutest"} {"msg":"The guest has requested activation with gen:38","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:41.987093665Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1332} {"msg":"4af065e7-0f0f-4d61-bdad-df8d040b4731 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:41.987125269Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1332,"session_id":"a698d3f2-6eed-4876-bdf2-4a3a1e4159d0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:41.987143361Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1332,"client":"0","":"downstairs","session_id":"a698d3f2-6eed-4876-bdf2-4a3a1e4159d0"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:41.987172555Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1332,"client":"0","":"downstairs","session_id":"a698d3f2-6eed-4876-bdf2-4a3a1e4159d0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:41.987195668Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1332,"client":"1","":"downstairs","session_id":"a698d3f2-6eed-4876-bdf2-4a3a1e4159d0"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:41.98721516Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1332,"client":"1","":"downstairs","session_id":"a698d3f2-6eed-4876-bdf2-4a3a1e4159d0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:41.987245034Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1332,"client":"2","":"downstairs","session_id":"a698d3f2-6eed-4876-bdf2-4a3a1e4159d0"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:41.987278318Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1332,"client":"2","":"downstairs","session_id":"a698d3f2-6eed-4876-bdf2-4a3a1e4159d0"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:41.987307672Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1332,"":"io task","client":"2","":"downstairs","session_id":"a698d3f2-6eed-4876-bdf2-4a3a1e4159d0"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:41.987334916Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1332,"":"io task","client":"0","":"downstairs","session_id":"a698d3f2-6eed-4876-bdf2-4a3a1e4159d0"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:41.987361459Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1332,"":"io task","client":"1","":"downstairs","session_id":"a698d3f2-6eed-4876-bdf2-4a3a1e4159d0"} {"msg":"ds_connection connected from Ok(127.0.0.1:56295)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:41.987390653Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1332,"":"io task","client":"2","":"downstairs","session_id":"a698d3f2-6eed-4876-bdf2-4a3a1e4159d0"} {"msg":"ds_connection connected from Ok(127.0.0.1:62972)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:41.987418246Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1332,"":"io task","client":"0","":"downstairs","session_id":"a698d3f2-6eed-4876-bdf2-4a3a1e4159d0"} {"msg":"ds_connection connected from Ok(127.0.0.1:61306)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:41.98744462Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1332,"":"io task","client":"1","":"downstairs","session_id":"a698d3f2-6eed-4876-bdf2-4a3a1e4159d0"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:41.988198807Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1332,"client":"1","":"downstairs","session_id":"a698d3f2-6eed-4876-bdf2-4a3a1e4159d0"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:41.988245783Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1332,"client":"1","":"downstairs","session_id":"a698d3f2-6eed-4876-bdf2-4a3a1e4159d0"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:41.988282928Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1332,"client":"2","":"downstairs","session_id":"a698d3f2-6eed-4876-bdf2-4a3a1e4159d0"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:41.988316812Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1332,"client":"2","":"downstairs","session_id":"a698d3f2-6eed-4876-bdf2-4a3a1e4159d0"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:41.988345866Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1332,"client":"0","":"downstairs","session_id":"a698d3f2-6eed-4876-bdf2-4a3a1e4159d0"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:41.988366529Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1332,"client":"0","":"downstairs","session_id":"a698d3f2-6eed-4876-bdf2-4a3a1e4159d0"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:41.988786853Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1332,"client":"0","":"downstairs","session_id":"a698d3f2-6eed-4876-bdf2-4a3a1e4159d0"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:41.988874074Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1332,"client":"1","":"downstairs","session_id":"a698d3f2-6eed-4876-bdf2-4a3a1e4159d0"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:41.988935472Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1332,"client":"2","":"downstairs","session_id":"a698d3f2-6eed-4876-bdf2-4a3a1e4159d0"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:41.989274995Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1332,"client":"0","":"downstairs","session_id":"a698d3f2-6eed-4876-bdf2-4a3a1e4159d0"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:41.989317121Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1332,"client":"0","":"downstairs","session_id":"a698d3f2-6eed-4876-bdf2-4a3a1e4159d0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:41.989345985Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1332,"session_id":"a698d3f2-6eed-4876-bdf2-4a3a1e4159d0"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:41.989381579Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1332,"client":"1","":"downstairs","session_id":"a698d3f2-6eed-4876-bdf2-4a3a1e4159d0"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:41.989413073Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1332,"client":"1","":"downstairs","session_id":"a698d3f2-6eed-4876-bdf2-4a3a1e4159d0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:41.989440977Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1332,"session_id":"a698d3f2-6eed-4876-bdf2-4a3a1e4159d0"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:41.989499855Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1332,"client":"2","":"downstairs","session_id":"a698d3f2-6eed-4876-bdf2-4a3a1e4159d0"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:41.989528528Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1332,"client":"2","":"downstairs","session_id":"a698d3f2-6eed-4876-bdf2-4a3a1e4159d0"} {"msg":"[0]R flush_numbers[0..12]: [57, 57, 58, 60, 59, 36, 60, 60, 59, 58, 60, 60]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:41.98954478Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1332,"":"downstairs","":"downstairs","session_id":"a698d3f2-6eed-4876-bdf2-4a3a1e4159d0"} {"msg":"[0]R generation[0..12]: [35, 35, 36, 37, 36, 24, 37, 37, 36, 36, 37, 37]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:41.989559322Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1332,"":"downstairs","":"downstairs","session_id":"a698d3f2-6eed-4876-bdf2-4a3a1e4159d0"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:41.989573924Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1332,"":"downstairs","":"downstairs","session_id":"a698d3f2-6eed-4876-bdf2-4a3a1e4159d0"} {"msg":"[1]R flush_numbers[0..12]: [57, 57, 58, 60, 59, 36, 60, 60, 59, 58, 60, 60]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:41.989588216Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1332,"":"downstairs","":"downstairs","session_id":"a698d3f2-6eed-4876-bdf2-4a3a1e4159d0"} {"msg":"[1]R generation[0..12]: [35, 35, 36, 37, 36, 24, 37, 37, 36, 36, 37, 37]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:41.989605238Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1332,"":"downstairs","":"downstairs","session_id":"a698d3f2-6eed-4876-bdf2-4a3a1e4159d0"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:41.98961954Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1332,"":"downstairs","":"downstairs","session_id":"a698d3f2-6eed-4876-bdf2-4a3a1e4159d0"} {"msg":"[2]R flush_numbers[0..12]: [57, 57, 58, 60, 59, 36, 60, 60, 59, 58, 60, 60]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:41.989633552Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1332,"":"downstairs","":"downstairs","session_id":"a698d3f2-6eed-4876-bdf2-4a3a1e4159d0"} {"msg":"[2]R generation[0..12]: [35, 35, 36, 37, 36, 24, 37, 37, 36, 36, 37, 37]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:41.989647243Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1332,"":"downstairs","":"downstairs","session_id":"a698d3f2-6eed-4876-bdf2-4a3a1e4159d0"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:41.989661305Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1332,"":"downstairs","":"downstairs","session_id":"a698d3f2-6eed-4876-bdf2-4a3a1e4159d0"} {"msg":"Max found gen is 38","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:41.989674297Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1332,"":"downstairs","":"downstairs","session_id":"a698d3f2-6eed-4876-bdf2-4a3a1e4159d0"} {"msg":"Generation requested: 38 >= found:38","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:41.989687389Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1332,"":"downstairs","":"downstairs","session_id":"a698d3f2-6eed-4876-bdf2-4a3a1e4159d0"} {"msg":"Next flush: 61","v":0,"name":"crucible","level":30,"Wait for a query_work_queue command to finish before sending IO time":"2024-11-01T16:51:41.98970042Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1332,"":"downstairs","":"downstairs","session_id":"a698d3f2-6eed-4876-bdf2-4a3a1e4159d0"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:41.989721403Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1332,"":"downstairs","":"downstairs","session_id":"a698d3f2-6eed-4876-bdf2-4a3a1e4159d0"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:41.989734965Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1332,"session_id":"a698d3f2-6eed-4876-bdf2-4a3a1e4159d0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:41.989747246Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1332,"client":"0","":"downstairs","session_id":"a698d3f2-6eed-4876-bdf2-4a3a1e4159d0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:41.989760688Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1332,"client":"1","":"downstairs","session_id":"a698d3f2-6eed-4876-bdf2-4a3a1e4159d0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:41.98977387Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1332,"client":"2","":"downstairs","session_id":"a698d3f2-6eed-4876-bdf2-4a3a1e4159d0"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:41.989787161Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1332,"session_id":"a698d3f2-6eed-4876-bdf2-4a3a1e4159d0"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:41.989800873Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1332,"session_id":"a698d3f2-6eed-4876-bdf2-4a3a1e4159d0"} {"msg":"4af065e7-0f0f-4d61-bdad-df8d040b4731 is now active with session: a698d3f2-6eed-4876-bdf2-4a3a1e4159d0","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:41.989813855Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1332,"session_id":"a698d3f2-6eed-4876-bdf2-4a3a1e4159d0"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:41.989825666Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1332,"session_id":"a698d3f2-6eed-4876-bdf2-4a3a1e4159d0"} {"msg":"The guest has finished waiting for activation with:38","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:41.989837928Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1332} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 516 len 20480 data: 2 2 2 3 3 02/30 Read block 537 len 40960 03/30 Read block 166 len 36864 04/30 Read block 111 len 32768 05/30 Read block 269 len 12288 06/30 Read block 489 len 4096 07/30 Read block 276 len 16384 08/30 Write block 504 len 32768 data: 3 3 3 3 4 3 3 3 09/30 Flush 10/30 Write block 79 len 8192 data: 5 5 11/30 Read block 219 len 12288 12/30 Write block 117 len 4096 data: 2 13/30 Write block 93 len 32768 data: 5 4 4 5 5 5 6 5 14/30 Write block 89 len 24576 data: 6 8 6 4 6 5 15/30 Flush 16/30 Write block 53 len 24576 data: 4 3 3 3 3 3 17/30 Read block 501 len 40960 18/30 Write block 289 len 24576 data: 3 2 3 3 4 4 19/30 Write block 472 len 16384 data: 6 5 5 6 20/30 Read block 32 len 20480 21/30 Read block 76 len 32768 22/30 Write block 541 len 36864 data: 2 2 4 4 4 5 5 5 5 23/30 Read block 184 len 24576 24/30 Write block 553 len 16384 data: 5 5 5 5 25/30 Write block 292 len 40960 data: 4 5 5 3 4 4 3 3 3 2 26/30 Write block 230 len 28672 data: 3 3 3 3 4 4 5 27/30 Read block 269 len 24576 28/30 Write block 318 len 36864 data: 6 6 6 6 6 5 4 3 3 29/30 Write block 324 len 12288 data: 5 4 4 30/30 Write block 175 len 4096 data: 3 ---------------------------------------------------------------- Crucible gen:38 GIO:true work queues: Upstairs:0 downstairs:15 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1015 Acked Write 6 Done Done Done false 1016 Acked Read 10 Done Done Done false 1017 Acked Write 6 Done Done Done false 1018 Acked Write 4 Done Done Done false 1019 Acked Read 5 Done Done Done false 1020 Acked Read 8 Done Done Done false 1021 Acked Write 9 Done Done Done false 1022 Acked Read 6 Done Done Done false 1023 Acked Write 4 Done Done Done false 1024 Acked Write 10 Done Done Done false 1025 Acked Write 7 Done Done Done false 1026 Acked Read 6 Sent Sent Done false 1027 Acked Write 9 Sent Sent Sent false 1028 Acked Write 3 Sent Sent Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 4 4 3 11 Done 11 11 12 34 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1014 1014 1014 Downstairs last five completed: 1014 1013 1012 1011 1010 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:51:42.011468435Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1332} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 35 35 35 57 57 57 F F F 1 020-039 35 35 35 57 57 57 F F F 2 040-059 36 36 36 58 58 58 T T T 3 060-079 38 38 38 62 62 62 F F F 4 080-099 38 38 38 62 62 62 F F F 5 100-119 38 38 38 62 62 62 F F F 6 120-139 37 37 37 60 60 60 F F F 7 140-159 37 37 37 60 60 60 F F F 8 160-179 36 36 36 59 59 59 T F T <--- 9 180-199 36 36 36 58 58 58 F F F 10 200-219 37 37 37 60 60 60 F F F 11 220-239 37 37 37 60 60 60 T T T 12 240-259 36 36 36 58 58 58 F F F 13 260-279 37 37 37 60 60 60 F F F 14 280-299 36 36 36 58 58 58 T T T 15 300-319 34 34 34 56 56 56 T T T 16 320-339 30 30 30 46 46 46 T T T 17 340-359 33 33 33 55 55 55 F F F 18 360-379 32 32 32 53 53 53 F F F 19 380-399 30 30 30 48 48 48 F F F 20 400-419 27 27 27 41 41 41 F F F 21 420-439 34 34 34 56 56 56 F F F 22 440-459 24 24 24 36 36 36 F F F 23 460-479 36 36 36 58 58 58 T T T 24 480-499 36 36 36 58 58 58 F F F 25 500-519 38 38 38 61 61 61 F F F 26 520-539 38 38 38 61 61 61 F F F 27 540-559 36 36 36 58 58 58 T T T 28 560-579 35 35 35 57 57 57 F F F 29 580-599 35 35 35 57 57 57 F F F Max gen: 38, Max flush: 62 Error: Difference in extent metadata found! A Difference in extent metadata is expected here On loop 19 Verifying data now No change after verify Loop: 19 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 35 35 35 57 57 57 F F F 1 020-039 35 35 35 57 57 57 F F F 2 040-059 39 39 39 63 63 63 F F F 3 060-079 38 38 38 62 62 62 F F F 4 080-099 38 38 38 62 62 62 F F F 5 100-119 38 38 38 62 62 62 F F F 6 120-139 37 37 37 60 60 60 F F F 7 140-159 37 37 37 60 60 60 F F F 8 160-179 39 39 39 63 63 63 F F F 9 180-199 36 36 36 58 58 58 F F F 10 200-219 37 37 37 60 60 60 F F F 11 220-239 39 39 39 63 63 63 F F F 12 240-259 36 36 36 58 58 58 F F F 13 260-279 37 37 37 60 60 60 F F F 14 280-299 39 39 39 63 63 63 F F F 15 300-319 39 39 39 63 63 63 F F F 16 320-339 39 39 39 63 63 63 F F F 17 340-359 33 33 33 55 55 55 F F F 18 360-379 32 32 32 53 53 53 F F F 19 380-399 30 30 30 48 48 48 F F F 20 400-419 27 27 27 41 41 41 F F F 21 420-439 34 34 34 56 56 56 F F F 22 440-459 24 24 24 36 36 36 F F F 23 460-479 39 39 39 63 63 63 F F F 24 480-499 36 36 36 58 58 58 F F F 25 500-519 38 38 38 61 61 61 F F F 26 520-539 38 38 38 61 61 61 F F F 27 540-559 39 39 39 63 63 63 F F F 28 560-579 35 35 35 57 57 57 F F F 29 580-599 35 35 35 57 57 57 F F F Max gen: 39, Max flush: 63 Begin loop 20 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:46.349417777Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1340,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:51:46.391809279Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1340,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:46.392188208Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1340,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:46.423803192Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1340,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:46.424327119Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1340} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"3818f0fc76fe787c6cc412af30fd59b930203885\",\n git_commit_timestamp: \"2024-11-01T16:08:30.000000000Z\",\n git_branch: \"renovate/hyper-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:46.424371655Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1340} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:46.424389037Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1340} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:46.424407089Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1340} {"msg":"Crucible 70afe430-3802-4562-8d45-6cd5b0cd79f8 has session id: 231aa9df-bd93-43b5-b3bc-e28e37d2a964","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:46.424871359Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1340,"session_id":"231aa9df-bd93-43b5-b3bc-e28e37d2a964"} {"msg":"Upstairs opts: Upstairs UUID: 70afe430-3802-4562-8d45-6cd5b0cd79f8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:46.424912975Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1340,"session_id":"231aa9df-bd93-43b5-b3bc-e28e37d2a964"} {"msg":"Crucible stats registered with UUID: 70afe430-3802-4562-8d45-6cd5b0cd79f8","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:46.424930467Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1340,"session_id":"231aa9df-bd93-43b5-b3bc-e28e37d2a964"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:46.455537531Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1340,"task":"crutest"} {"msg":"The guest has requested activation with gen:40","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:46.455568095Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1340} {"msg":"70afe430-3802-4562-8d45-6cd5b0cd79f8 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:46.455651165Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1340,"session_id":"231aa9df-bd93-43b5-b3bc-e28e37d2a964"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:46.455674148Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1340,"client":"0","":"downstairs","session_id":"231aa9df-bd93-43b5-b3bc-e28e37d2a964"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:46.455702202Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1340,"client":"0","":"downstairs","session_id":"231aa9df-bd93-43b5-b3bc-e28e37d2a964"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:46.455720984Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1340,"client":"1","":"downstairs","session_id":"231aa9df-bd93-43b5-b3bc-e28e37d2a964"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:46.455741917Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1340,"client":"1","":"downstairs","session_id":"231aa9df-bd93-43b5-b3bc-e28e37d2a964"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:46.455758719Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1340,"client":"2","":"downstairs","session_id":"231aa9df-bd93-43b5-b3bc-e28e37d2a964"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:46.455777922Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1340,"client":"2","":"downstairs","session_id":"231aa9df-bd93-43b5-b3bc-e28e37d2a964"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:46.455805095Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1340,"":"io task","client":"2","":"downstairs","session_id":"231aa9df-bd93-43b5-b3bc-e28e37d2a964"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:46.455824768Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1340,"":"io task","client":"0","":"downstairs","session_id":"231aa9df-bd93-43b5-b3bc-e28e37d2a964"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:46.455855361Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1340,"":"io task","client":"1","":"downstairs","session_id":"231aa9df-bd93-43b5-b3bc-e28e37d2a964"} {"msg":"ds_connection connected from Ok(127.0.0.1:44499)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:46.455889146Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1340,"":"io task","client":"2","":"downstairs","session_id":"231aa9df-bd93-43b5-b3bc-e28e37d2a964"} {"msg":"ds_connection connected from Ok(127.0.0.1:55376)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:46.45592417Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1340,"":"io task","client":"0","":"downstairs","session_id":"231aa9df-bd93-43b5-b3bc-e28e37d2a964"} {"msg":"ds_connection connected from Ok(127.0.0.1:44482)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:46.455956245Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1340,"":"io task","client":"1","":"downstairs","session_id":"231aa9df-bd93-43b5-b3bc-e28e37d2a964"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:46.456809264Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1340,"client":"2","":"downstairs","session_id":"231aa9df-bd93-43b5-b3bc-e28e37d2a964"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:46.456837718Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1340,"client":"2","":"downstairs","session_id":"231aa9df-bd93-43b5-b3bc-e28e37d2a964"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:46.456898886Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1340,"client":"1","":"downstairs","session_id":"231aa9df-bd93-43b5-b3bc-e28e37d2a964"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:46.456974596Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1340,"client":"1","":"downstairs","session_id":"231aa9df-bd93-43b5-b3bc-e28e37d2a964"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:46.457022952Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1340,"client":"0","":"downstairs","session_id":"231aa9df-bd93-43b5-b3bc-e28e37d2a964"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:46.457056606Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1340,"client":"0","":"downstairs","session_id":"231aa9df-bd93-43b5-b3bc-e28e37d2a964"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:46.457412402Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1340,"client":"2","":"downstairs","session_id":"231aa9df-bd93-43b5-b3bc-e28e37d2a964"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:46.457512915Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1340,"client":"1","":"downstairs","session_id":"231aa9df-bd93-43b5-b3bc-e28e37d2a964"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:46.457720982Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1340,"client":"0","":"downstairs","session_id":"231aa9df-bd93-43b5-b3bc-e28e37d2a964"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:46.457888823Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1340,"client":"2","":"downstairs","session_id":"231aa9df-bd93-43b5-b3bc-e28e37d2a964"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:46.457928729Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1340,"client":"2","":"downstairs","session_id":"231aa9df-bd93-43b5-b3bc-e28e37d2a964"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:46.457958363Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1340,"session_id":"231aa9df-bd93-43b5-b3bc-e28e37d2a964"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:46.458021721Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1340,"client":"1","":"downstairs","session_id":"231aa9df-bd93-43b5-b3bc-e28e37d2a964"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:46.458050814Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1340,"client":"1","":"downstairs","session_id":"231aa9df-bd93-43b5-b3bc-e28e37d2a964"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:46.458065506Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1340,"session_id":"231aa9df-bd93-43b5-b3bc-e28e37d2a964"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:46.45817254Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1340,"client":"0","":"downstairs","session_id":"231aa9df-bd93-43b5-b3bc-e28e37d2a964"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:46.458192243Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1340,"client":"0","":"downstairs","session_id":"231aa9df-bd93-43b5-b3bc-e28e37d2a964"} {"msg":"[0]R flush_numbers[0..12]: [57, 57, 63, 62, 62, 62, 60, 60, 63, 58, 60, 63]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:46.458206624Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1340,"":"downstairs","":"downstairs","session_id":"231aa9df-bd93-43b5-b3bc-e28e37d2a964"} {"msg":"[0]R generation[0..12]: [35, 35, 39, 38, 38, 38, 37, 37, 39, 36, 37, 39]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:46.458220516Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1340,"":"downstairs","":"downstairs","session_id":"231aa9df-bd93-43b5-b3bc-e28e37d2a964"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:46.458234608Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1340,"":"downstairs","":"downstairs","session_id":"231aa9df-bd93-43b5-b3bc-e28e37d2a964"} {"msg":"[1]R flush_numbers[0..12]: [57, 57, 63, 62, 62, 62, 60, 60, 63, 58, 60, 63]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:46.45824845Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1340,"":"downstairs","":"downstairs","session_id":"231aa9df-bd93-43b5-b3bc-e28e37d2a964"} {"msg":"[1]R generation[0..12]: [35, 35, 39, 38, 38, 38, 37, 37, 39, 36, 37, 39]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:46.458265022Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1340,"":"downstairs","":"downstairs","session_id":"231aa9df-bd93-43b5-b3bc-e28e37d2a964"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:46.458279284Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1340,"":"downstairs","":"downstairs","session_id":"231aa9df-bd93-43b5-b3bc-e28e37d2a964"} {"msg":"[2]R flush_numbers[0..12]: [57, 57, 63, 62, 62, 62, 60, 60, 63, 58, 60, 63]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:46.458293196Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1340,"":"downstairs","":"downstairs","session_id":"231aa9df-bd93-43b5-b3bc-e28e37d2a964"} {"msg":"[2]R generation[0..12]: [35, 35, 39, 38, 38, 38, 37, 37, 39, 36, 37, 39]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:46.458306977Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1340,"":"downstairs","":"downstairs","session_id":"231aa9df-bd93-43b5-b3bc-e28e37d2a964"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:46.458321179Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1340,"":"downstairs","":"downstairs","session_id":"231aa9df-bd93-43b5-b3bc-e28e37d2a964"} {"msg":"Max found gen is 40","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:46.458334511Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1340,"":"downstairs","":"downstairs","session_id":"231aa9df-bd93-43b5-b3bc-e28e37d2a964"} {"msg":"Generation requested: 40 >= found:40","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:46.458347833Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1340,"":"downstairs","":"downstairs","session_id":"231aa9df-bd93-43b5-b3bc-e28e37d2a964"} {"msg":"Next flush: 64","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:46.458361024Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1340,"":"downstairs","":"downstairs","session_id":"231aa9df-bd93-43b5-b3bc-e28e37d2a964"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:46.458374306Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1340,"":"downstairs","":"downstairs","session_id":"231aa9df-bd93-43b5-b3bc-e28e37d2a964"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:46.458387528Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1340,"session_id":"231aa9df-bd93-43b5-b3bc-e28e37d2a964"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:46.458399749Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1340,"client":"0","":"downstairs","Wait for a query_work_queue command to finish before sending IO session_id":"231aa9df-bd93-43b5-b3bc-e28e37d2a964"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:46.458421262Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1340,"client":"1","":"downstairs","session_id":"231aa9df-bd93-43b5-b3bc-e28e37d2a964"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:46.458434294Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1340,"client":"2","":"downstairs","session_id":"231aa9df-bd93-43b5-b3bc-e28e37d2a964"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:46.458447755Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1340,"session_id":"231aa9df-bd93-43b5-b3bc-e28e37d2a964"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:46.458460047Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1340,"session_id":"231aa9df-bd93-43b5-b3bc-e28e37d2a964"} {"msg":"70afe430-3802-4562-8d45-6cd5b0cd79f8 is now active with session: 231aa9df-bd93-43b5-b3bc-e28e37d2a964","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:46.458474539Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1340,"session_id":"231aa9df-bd93-43b5-b3bc-e28e37d2a964"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:46.45848672Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1340,"session_id":"231aa9df-bd93-43b5-b3bc-e28e37d2a964"} {"msg":"The guest has finished waiting for activation with:40","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:46.458499292Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1340} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 167 len 24576 data: 4 4 2 2 2 2 02/30 Flush 03/30 Write block 589 len 20480 data: 3 3 3 3 4 04/30 Read block 372 len 8192 05/30 Read block 143 len 28672 06/30 Write block 364 len 8192 data: 3 4 07/30 Read block 373 len 32768 08/30 Read block 118 len 32768 09/30 Write block 114 len 36864 data: 3 2 2 3 2 2 4 5 5 10/30 Flush 11/30 Write block 239 len 36864 data: 3 3 3 3 3 4 5 6 8 12/30 Write block 176 len 20480 data: 4 5 5 5 5 13/30 Write block 584 len 16384 data: 3 4 4 4 14/30 Write block 381 len 12288 data: 4 4 3 15/30 Flush 16/30 Write block 521 len 12288 data: 3 4 4 17/30 Read block 587 len 20480 18/30 Flush 19/30 Write block 14 len 20480 data: 3 3 4 4 4 20/30 Read block 379 len 36864 21/30 Read block 150 len 28672 22/30 Write block 85 len 16384 data: 6 5 5 5 23/30 Read block 159 len 40960 24/30 Write block 203 len 36864 data: 3 4 6 6 6 5 4 5 6 25/30 Read block 573 len 4096 26/30 Write block 38 len 12288 data: 5 5 4 27/30 Write block 106 len 20480 data: 5 5 4 4 3 28/30 Write block 454 len 4096 data: 4 29/30 Write block 510 len 12288 data: 4 4 4 30/30 Read block 389 len 8192 ---------------------------------------------------------------- Crucible gen:40 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1018 Acked Write 5 Done Done Done false 1019 Acked Read 9 Done Done Done false 1020 Acked Read 7 Done Done Done false 1021 Acked Write 4 Done Done Done false 1022 Acked Read 10 Done Done Done false 1023 Acked Write 9 Done Done Done false 1024 Acked Read 1 Done Done Done false 1025 Acked Write 3 Done Done Done false 1026 Acked Write 5 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Write 3 Done Done Done false 1029 Acked Read 2 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 12 12 12 36 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1017 1017 1017 Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:51:46.484788009Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1340} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 35 35 35 57 57 57 T T T 1 020-039 35 35 35 57 57 57 T T T 2 040-059 39 39 39 63 63 63 T T T 3 060-079 38 38 38 62 62 62 F F F 4 080-099 38 38 38 62 62 62 T T T 5 100-119 40 40 40 65 65 65 T T T 6 120-139 40 40 40 65 65 65 F F F 7 140-159 37 37 37 60 60 60 F F F 8 160-179 40 40 40 66 66 66 F F F 9 180-199 40 40 40 66 66 66 F F F 10 200-219 37 37 37 60 60 60 T T T 11 220-239 40 40 40 66 66 66 F F F 12 240-259 40 40 40 66 66 66 F F F 13 260-279 37 37 37 60 60 60 F F F 14 280-299 39 39 39 63 63 63 F F F 15 300-319 39 39 39 63 63 63 F F F 16 320-339 39 39 39 63 63 63 F F F 17 340-359 33 33 33 55 55 55 F F F 18 360-379 40 40 40 65 65 65 F F F 19 380-399 40 40 40 66 66 66 F F F 20 400-419 27 27 27 41 41 41 F F F 21 420-439 34 34 34 56 56 56 F F F 22 440-459 24 24 24 36 36 36 T T T 23 460-479 39 39 39 63 63 63 F F F 24 480-499 36 36 36 58 58 58 F F F 25 500-519 38 38 38 61 61 61 T T T 26 520-539 40 40 40 67 67 67 F F F 27 540-559 39 39 39 63 63 63 F F F 28 560-579 35 35 35 57 57 57 F F F 29 580-599 40 40 40 66 66 66 F F F Max gen: 40, Max flush: 67 A Difference in extent metadata is expected here On loop 20 Verifying data now No change after verify Loop: 20 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 41 41 41 68 68 68 F F F 1 020-039 41 41 41 68 68 68 F F F 2 040-059 41 41 41 68 68 68 F F F 3 060-079 38 38 38 62 62 62 F F F 4 080-099 41 41 41 68 68 68 F F F 5 100-119 41 41 41 68 68 68 F F F 6 120-139 40 40 40 65 65 65 F F F 7 140-159 37 37 37 60 60 60 F F F 8 160-179 40 40 40 66 66 66 F F F 9 180-199 40 40 40 66 66 66 F F F 10 200-219 41 41 41 68 68 68 F F F 11 220-239 40 40 40 66 66 66 F F F 12 240-259 40 40 40 66 66 66 F F F 13 260-279 37 37 37 60 60 60 F F F 14 280-299 39 39 39 63 63 63 F F F 15 300-319 39 39 39 63 63 63 F F F 16 320-339 39 39 39 63 63 63 F F F 17 340-359 33 33 33 55 55 55 F F F 18 360-379 40 40 40 65 65 65 F F F 19 380-399 40 40 40 66 66 66 F F F 20 400-419 27 27 27 41 41 41 F F F 21 420-439 34 34 34 56 56 56 F F F 22 440-459 41 41 41 68 68 68 F F F 23 460-479 39 39 39 63 63 63 F F F 24 480-499 36 36 36 58 58 58 F F F 25 500-519 41 41 41 68 68 68 F F F 26 520-539 40 40 40 67 67 67 F F F 27 540-559 39 39 39 63 63 63 F F F 28 560-579 35 35 35 57 57 57 F F F 29 580-599 40 40 40 66 66 66 F F F Max gen: 41, Max flush: 68 Begin loop 21 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:50.850474485Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1348,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:50.894326706Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1348,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:50.894950906Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1348} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"3818f0fc76fe787c6cc412af30fd59b930203885\",\n git_commit_timestamp: \"2024-11-01T16:08:30.000000000Z\",\n git_branch: \"renovate/hyper-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:50.894992742Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1348} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:50.895010684Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1348} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:50.895025626Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1348} {"msg":"Crucible dcd502fc-36f0-4cef-b321-b7f2ebe6f147 has session id: d6dca4ff-ed18-47cc-bf5b-c8cd1f84036f","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:50.895427327Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1348,"session_id":"d6dca4ff-ed18-47cc-bf5b-c8cd1f84036f"} {"msg":"Upstairs opts: Upstairs UUID: dcd502fc-36f0-4cef-b321-b7f2ebe6f147, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:50.895478814Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1348,"session_id":"d6dca4ff-ed18-47cc-bf5b-c8cd1f84036f"} {"msg":"Crucible stats registered with UUID: dcd502fc-36f0-4cef-b321-b7f2ebe6f147","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:50.895509768Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1348,"session_id":"d6dca4ff-ed18-47cc-bf5b-c8cd1f84036f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:50.926349512Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1348,"task":"crutest"} {"msg":"The guest has requested activation with gen:42","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:50.926393617Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1348} {"msg":"dcd502fc-36f0-4cef-b321-b7f2ebe6f147 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:50.926428992Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1348,"session_id":"d6dca4ff-ed18-47cc-bf5b-c8cd1f84036f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:50.926459486Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1348,"client":"0","":"downstairs","session_id":"d6dca4ff-ed18-47cc-bf5b-c8cd1f84036f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:50.926508532Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1348,"client":"0","":"downstairs","session_id":"d6dca4ff-ed18-47cc-bf5b-c8cd1f84036f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:50.926544727Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1348,"client":"1","":"downstairs","session_id":"d6dca4ff-ed18-47cc-bf5b-c8cd1f84036f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:50.92657228Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1348,"client":"1","":"downstairs","session_id":"d6dca4ff-ed18-47cc-bf5b-c8cd1f84036f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:50.926599314Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1348,"client":"2","":"downstairs","session_id":"d6dca4ff-ed18-47cc-bf5b-c8cd1f84036f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:50.926624857Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1348,"client":"2","":"downstairs","session_id":"d6dca4ff-ed18-47cc-bf5b-c8cd1f84036f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:50.92664784Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1348,"":"io task","client":"2","":"downstairs","session_id":"d6dca4ff-ed18-47cc-bf5b-c8cd1f84036f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:50.926673123Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1348,"":"io task","client":"0","":"downstairs","session_id":"d6dca4ff-ed18-47cc-bf5b-c8cd1f84036f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:50.926697727Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1348,"":"io task","client":"1","":"downstairs","session_id":"d6dca4ff-ed18-47cc-bf5b-c8cd1f84036f"} {"msg":"ds_connection connected from Ok(127.0.0.1:47291)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:50.92672227Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1348,"":"io task","client":"1","":"downstairs","session_id":"d6dca4ff-ed18-47cc-bf5b-c8cd1f84036f"} {"msg":"ds_connection connected from Ok(127.0.0.1:37827)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:50.926748043Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1348,"":"io task","client":"2","":"downstairs","session_id":"d6dca4ff-ed18-47cc-bf5b-c8cd1f84036f"} {"msg":"ds_connection connected from Ok(127.0.0.1:61983)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:50.926942648Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1348,"":"io task","client":"0","":"downstairs","session_id":"d6dca4ff-ed18-47cc-bf5b-c8cd1f84036f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:50.927522493Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1348,"client":"1","":"downstairs","session_id":"d6dca4ff-ed18-47cc-bf5b-c8cd1f84036f"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:50.927550966Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1348,"client":"1","":"downstairs","session_id":"d6dca4ff-ed18-47cc-bf5b-c8cd1f84036f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:50.927681203Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1348,"client":"0","":"downstairs","session_id":"d6dca4ff-ed18-47cc-bf5b-c8cd1f84036f"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:50.927709817Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1348,"client":"0","":"downstairs","session_id":"d6dca4ff-ed18-47cc-bf5b-c8cd1f84036f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:50.92773625Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1348,"client":"2","":"downstairs","session_id":"d6dca4ff-ed18-47cc-bf5b-c8cd1f84036f"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:50.927763544Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1348,"client":"2","":"downstairs","session_id":"d6dca4ff-ed18-47cc-bf5b-c8cd1f84036f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:50.928156484Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1348,"client":"1","":"downstairs","session_id":"d6dca4ff-ed18-47cc-bf5b-c8cd1f84036f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:50.928264729Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1348,"client":"0","":"downstairs","session_id":"d6dca4ff-ed18-47cc-bf5b-c8cd1f84036f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:50.92843307Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1348,"client":"2","":"downstairs","session_id":"d6dca4ff-ed18-47cc-bf5b-c8cd1f84036f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:50.928781505Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1348,"client":"1","":"downstairs","session_id":"d6dca4ff-ed18-47cc-bf5b-c8cd1f84036f"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:50.928810119Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1348,"client":"1","":"downstairs","session_id":"d6dca4ff-ed18-47cc-bf5b-c8cd1f84036f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:50.928825191Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1348,"session_id":"d6dca4ff-ed18-47cc-bf5b-c8cd1f84036f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:50.928837562Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1348,"client":"0","":"downstairs","session_id":"d6dca4ff-ed18-47cc-bf5b-c8cd1f84036f"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:50.928851094Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1348,"client":"0","":"downstairs","session_id":"d6dca4ff-ed18-47cc-bf5b-c8cd1f84036f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:50.928864266Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1348,"session_id":"d6dca4ff-ed18-47cc-bf5b-c8cd1f84036f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:50.928991462Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1348,"client":"2","":"downstairs","session_id":"d6dca4ff-ed18-47cc-bf5b-c8cd1f84036f"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:50.929010725Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1348,"client":"2","":"downstairs","session_id":"d6dca4ff-ed18-47cc-bf5b-c8cd1f84036f"} {"msg":"[0]R flush_numbers[0..12]: [68, 68, 68, 62, 68, 68, 65, 60, 66, 66, 68, 66]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:50.929027207Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1348,"":"downstairs","":"downstairs","session_id":"d6dca4ff-ed18-47cc-bf5b-c8cd1f84036f"} {"msg":"[0]R generation[0..12]: [41, 41, 41, 38, 41, 41, 40, 37, 40, 40, 41, 40]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:50.929041309Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1348,"":"downstairs","":"downstairs","session_id":"d6dca4ff-ed18-47cc-bf5b-c8cd1f84036f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:50.92905558Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1348,"":"downstairs","":"downstairs","session_id":"d6dca4ff-ed18-47cc-bf5b-c8cd1f84036f"} {"msg":"[1]R flush_numbers[0..12]: [68, 68, 68, 62, 68, 68, 65, 60, 66, 66, 68, 66]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:50.929069442Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1348,"":"downstairs","":"downstairs","session_id":"d6dca4ff-ed18-47cc-bf5b-c8cd1f84036f"} {"msg":"[1]R generation[0..12]: [41, 41, 41, 38, 41, 41, 40, 37, 40, 40, 41, 40]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:50.929083024Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1348,"":"downstairs","":"downstairs","session_id":"d6dca4ff-ed18-47cc-bf5b-c8cd1f84036f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:50.929096886Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1348,"":"downstairs","":"downstairs","session_id":"d6dca4ff-ed18-47cc-bf5b-c8cd1f84036f"} {"msg":"[2]R flush_numbers[0..12]: [68, 68, 68, 62, 68, 68, 65, 60, 66, 66, 68, 66]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:50.929110487Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1348,"":"downstairs","":"downstairs","session_id":"d6dca4ff-ed18-47cc-bf5b-c8cd1f84036f"} {"msg":"[2]R generation[0..12]: [41, 41, 41, 38, 41, 41, 40, 37, 40, 40, 41, 40]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:50.92912748Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1348,"":"downstairs","":"downstairs","session_id":"d6dca4ff-ed18-47cc-bf5b-c8cd1f84036f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:50.929141871Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1348,"":"downstairs","":"downstairs","session_id":"d6dca4ff-ed18-47cc-bf5b-c8cd1f84036f"} {"msg":"Max found gen is 42","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:50.929154913Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1348,"":"downstairs","":"downstairs","session_id":"d6dca4ff-ed18-47cc-bf5b-c8cd1f84036f"} {"msg":"Generation requested: 42 >= found:42","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:50.929168115Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1348,"":"downstairs","":"downstairs","session_id":"d6dca4ff-ed18-47cc-bf5b-c8cd1f84036f"} {"msg":"Next flush: 69","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:50.929181256Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1348,"":"downstairs","":"downstairs","session_id":"d6dca4ff-ed18-47cc-bf5b-c8cd1f84036f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:50.929194188Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1348,"":"downstairs","":"downstairs","session_id":"d6dca4ff-ed18-47cc-bf5b-c8cd1f84036f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-01T16:51:50.92920767Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1348,"session_id":"d6dca4ff-ed18-47cc-bf5b-c8cd1f84036f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:50.929225882Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1348,"client":"0","":"downstairs","session_id":"d6dca4ff-ed18-47cc-bf5b-c8cd1f84036f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:50.929239204Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1348,"client":"1","":"downstairs","session_id":"d6dca4ff-ed18-47cc-bf5b-c8cd1f84036f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:50.929252336Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1348,"client":"2","":"downstairs","session_id":"d6dca4ff-ed18-47cc-bf5b-c8cd1f84036f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:50.929265678Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1348,"session_id":"d6dca4ff-ed18-47cc-bf5b-c8cd1f84036f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:50.929277979Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1348,"session_id":"d6dca4ff-ed18-47cc-bf5b-c8cd1f84036f"} {"msg":"dcd502fc-36f0-4cef-b321-b7f2ebe6f147 is now active with session: d6dca4ff-ed18-47cc-bf5b-c8cd1f84036f","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:50.929290731Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1348,"session_id":"d6dca4ff-ed18-47cc-bf5b-c8cd1f84036f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:50.929302642Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1348,"session_id":"d6dca4ff-ed18-47cc-bf5b-c8cd1f84036f"} {"msg":"The guest has finished waiting for activation with:42","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:50.929314824Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1348} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 0 len 28672 data: 2 3 3 3 2 3 4 02/30 Write block 37 len 8192 data: 4 6 03/30 Write block 216 len 12288 data: 6 6 8 04/30 Read block 212 len 32768 05/30 Read block 209 len 36864 06/30 Write block 486 len 28672 data: 3 3 4 4 4 5 4 07/30 Write block 41 len 12288 data: 5 7 9 08/30 Write block 109 len 16384 data: 5 4 3 3 09/30 Write block 382 len 20480 data: 5 4 3 4 4 10/30 Read block 166 len 32768 11/30 Write block 4 len 20480 data: 3 4 5 3 3 12/30 Read block 386 len 36864 13/30 Read block 70 len 40960 14/30 Read block 384 len 24576 15/30 Write block 383 len 32768 data: 5 4 5 5 4 3 5 4 16/30 Write block 84 len 8192 data: 5 7 17/30 Read block 411 len 4096 18/30 Read block 133 len 28672 19/30 Write block 83 len 28672 data: 7 6 8 6 6 6 7 20/30 Write block 583 len 28672 data: 4 4 5 5 5 3 4 21/30 Read block 320 len 8192 22/30 Write block 261 len 12288 data: 6 5 5 23/30 Read block 371 len 40960 24/30 Read block 353 len 16384 25/30 Write block 333 len 24576 data: 5 5 5 4 5 4 26/30 Read block 232 len 4096 27/30 Read block 366 len 8192 28/30 Write block 530 len 32768 data: 6 5 5 5 5 4 3 3 29/30 Write block 465 len 12288 data: 3 4 3 30/30 Write block 205 len 40960 data: 7 7 7 6 5 6 7 6 5 3 ---------------------------------------------------------------- Crucible gen:42 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 7 Done Done Done false 1001 Acked Write 2 Done Done Done false 1002 Acked Write 3 Done Done Done false 1003 Acked Read 8 Done Done Done false 1004 Acked Read 9 Done Done Done false 1005 Acked Write 7 Done Done Done false 1006 Acked Write 3 Done Done Done false 1007 Acked Write 4 Done Done Done false 1008 Acked Write 5 Done Done Done false 1009 Acked Read 8 Done Done Done false 1010 Acked Write 5 Done Done Done false 1011 Acked Read 9 Done Done Done false 1012 Acked Read 10 Done Done Done false 1013 Acked Read 6 Done Done Done false 1014 Acked Write 8 Done Done Done false 1015 Acked Write 2 Done Done Done false 1016 Acked Read 1 Done Done Done false 1017 Acked Read 7 Done Done Done false 1018 Acked Write 7 Done Done Done false 1019 Acked Write 7 Done Done Done false 1020 Acked Read 2 Done Done Done false 1021 Acked Write 3 Done Done Done false 1022 Acked Read 10 Done Done Done false 1023 Acked Read 4 Done Done Done false 1024 Acked Write 6 Done Done Done false 1025 Acked Read 1 Done Done Done false 1026 Acked Read 2 Done Done Done false 1027 Acked Write 8 Done Sent Done false 1028 Acked Write 3 Sent Sent Sent false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 3 2 7 Done 28 27 28 83 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 0 0 0 Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:51:50.947710694Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1348} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 41 41 41 68 68 68 T T T 1 020-039 41 41 41 68 68 68 T T T 2 040-059 41 41 41 68 68 68 T T T 3 060-079 38 38 38 62 62 62 F F F 4 080-099 41 41 41 68 68 68 T T T 5 100-119 41 41 41 68 68 68 T T T 6 120-139 40 40 40 65 65 65 F F F 7 140-159 37 37 37 60 60 60 F F F 8 160-179 40 40 40 66 66 66 F F F 9 180-199 40 40 40 66 66 66 F F F 10 200-219 41 41 41 68 68 68 T T T 11 220-239 40 40 40 66 66 66 F F F 12 240-259 40 40 40 66 66 66 F F F 13 260-279 37 37 37 60 60 60 T T T 14 280-299 39 39 39 63 63 63 F F F 15 300-319 39 39 39 63 63 63 F F F 16 320-339 39 39 39 63 63 63 T T T 17 340-359 33 33 33 55 55 55 F F F 18 360-379 40 40 40 65 65 65 F F F 19 380-399 40 40 40 66 66 66 T T T 20 400-419 27 27 27 41 41 41 F F F 21 420-439 34 34 34 56 56 56 F F F 22 440-459 41 41 41 68 68 68 F F F 23 460-479 39 39 39 63 63 63 T T T 24 480-499 36 36 36 58 58 58 T T T 25 500-519 41 41 41 68 68 68 F F F 26 520-539 40 40 40 67 67 67 T T T 27 540-559 39 39 39 63 63 63 F F F 28 560-579 35 35 35 57 57 57 F F F 29 580-599 40 40 40 66 66 66 T T T Max gen: 41, Max flush: 68 A Difference in extent metadata is expected here On loop 21 Verifying data now No change after verify Loop: 21 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 42 42 42 69 69 69 F F F 1 020-039 42 42 42 69 69 69 F F F 2 040-059 42 42 42 69 69 69 F F F 3 060-079 38 38 38 62 62 62 F F F 4 080-099 42 42 42 69 69 69 F F F 5 100-119 42 42 42 69 69 69 F F F 6 120-139 40 40 40 65 65 65 F F F 7 140-159 37 37 37 60 60 60 F F F 8 160-179 40 40 40 66 66 66 F F F 9 180-199 40 40 40 66 66 66 F F F 10 200-219 42 42 42 69 69 69 F F F 11 220-239 40 40 40 66 66 66 F F F 12 240-259 40 40 40 66 66 66 F F F 13 260-279 42 42 42 69 69 69 F F F 14 280-299 39 39 39 63 63 63 F F F 15 300-319 39 39 39 63 63 63 F F F 16 320-339 42 42 42 69 69 69 F F F 17 340-359 33 33 33 55 55 55 F F F 18 360-379 40 40 40 65 65 65 F F F 19 380-399 42 42 42 69 69 69 F F F 20 400-419 27 27 27 41 41 41 F F F 21 420-439 34 34 34 56 56 56 F F F 22 440-459 41 41 41 68 68 68 F F F 23 460-479 42 42 42 69 69 69 F F F 24 480-499 42 42 42 69 69 69 F F F 25 500-519 41 41 41 68 68 68 F F F 26 520-539 42 42 42 69 69 69 F F F 27 540-559 39 39 39 63 63 63 F F F 28 560-579 35 35 35 57 57 57 F F F 29 580-599 42 42 42 69 69 69 F F F Max gen: 42, Max flush: 69 Begin loop 22 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:55.346335746Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1356,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:55.390152512Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1356,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:55.390736897Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1356} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"3818f0fc76fe787c6cc412af30fd59b930203885\",\n git_commit_timestamp: \"2024-11-01T16:08:30.000000000Z\",\n git_branch: \"renovate/hyper-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:55.390784713Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1356} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:55.390802605Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1356} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:55.390820228Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1356} {"msg":"Crucible 103e01f4-336d-4604-b8db-d52a9ae71ee8 has session id: b39a6c92-df4f-4962-a63e-754d3c7b10ae","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:55.391198016Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1356,"session_id":"b39a6c92-df4f-4962-a63e-754d3c7b10ae"} {"msg":"Upstairs opts: Upstairs UUID: 103e01f4-336d-4604-b8db-d52a9ae71ee8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:55.391239812Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1356,"session_id":"b39a6c92-df4f-4962-a63e-754d3c7b10ae"} {"msg":"Crucible stats registered with UUID: 103e01f4-336d-4604-b8db-d52a9ae71ee8","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:55.391264435Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1356,"session_id":"b39a6c92-df4f-4962-a63e-754d3c7b10ae"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:55.422069254Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1356,"task":"crutest"} {"msg":"The guest has requested activation with gen:44","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:55.422101868Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1356} {"msg":"103e01f4-336d-4604-b8db-d52a9ae71ee8 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:55.422151815Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1356,"session_id":"b39a6c92-df4f-4962-a63e-754d3c7b10ae"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:55.422171917Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1356,"client":"0","":"downstairs","session_id":"b39a6c92-df4f-4962-a63e-754d3c7b10ae"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:55.422201701Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1356,"client":"0","":"downstairs","session_id":"b39a6c92-df4f-4962-a63e-754d3c7b10ae"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:55.422219513Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1356,"client":"1","":"downstairs","session_id":"b39a6c92-df4f-4962-a63e-754d3c7b10ae"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:55.422236736Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1356,"client":"1","":"downstairs","session_id":"b39a6c92-df4f-4962-a63e-754d3c7b10ae"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:55.422254758Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1356,"client":"2","":"downstairs","session_id":"b39a6c92-df4f-4962-a63e-754d3c7b10ae"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:55.422274941Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1356,"client":"2","":"downstairs","session_id":"b39a6c92-df4f-4962-a63e-754d3c7b10ae"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:55.422291753Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1356,"":"io task","client":"2","":"downstairs","session_id":"b39a6c92-df4f-4962-a63e-754d3c7b10ae"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:55.422309065Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1356,"":"io task","client":"0","":"downstairs","session_id":"b39a6c92-df4f-4962-a63e-754d3c7b10ae"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:55.422331738Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1356,"":"io task","client":"1","":"downstairs","session_id":"b39a6c92-df4f-4962-a63e-754d3c7b10ae"} {"msg":"ds_connection connected from Ok(127.0.0.1:55270)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:55.422363602Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1356,"":"io task","client":"2","":"downstairs","session_id":"b39a6c92-df4f-4962-a63e-754d3c7b10ae"} {"msg":"ds_connection connected from Ok(127.0.0.1:36799)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:55.422397296Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1356,"":"io task","client":"1","":"downstairs","session_id":"b39a6c92-df4f-4962-a63e-754d3c7b10ae"} {"msg":"ds_connection connected from Ok(127.0.0.1:48606)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:55.422432451Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1356,"":"io task","client":"0","":"downstairs","session_id":"b39a6c92-df4f-4962-a63e-754d3c7b10ae"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:55.423150324Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1356,"client":"1","":"downstairs","session_id":"b39a6c92-df4f-4962-a63e-754d3c7b10ae"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:55.423179807Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1356,"client":"1","":"downstairs","session_id":"b39a6c92-df4f-4962-a63e-754d3c7b10ae"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:55.423341648Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1356,"client":"0","":"downstairs","session_id":"b39a6c92-df4f-4962-a63e-754d3c7b10ae"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:55.423362911Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1356,"client":"0","":"downstairs","session_id":"b39a6c92-df4f-4962-a63e-754d3c7b10ae"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:55.423424229Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1356,"client":"2","":"downstairs","session_id":"b39a6c92-df4f-4962-a63e-754d3c7b10ae"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:55.423456323Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1356,"client":"2","":"downstairs","session_id":"b39a6c92-df4f-4962-a63e-754d3c7b10ae"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:55.423889649Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1356,"client":"1","":"downstairs","session_id":"b39a6c92-df4f-4962-a63e-754d3c7b10ae"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:55.423931654Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1356,"client":"0","":"downstairs","session_id":"b39a6c92-df4f-4962-a63e-754d3c7b10ae"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:55.424224352Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1356,"client":"2","":"downstairs","session_id":"b39a6c92-df4f-4962-a63e-754d3c7b10ae"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:55.424497417Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1356,"client":"1","":"downstairs","session_id":"b39a6c92-df4f-4962-a63e-754d3c7b10ae"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:55.4245223Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1356,"client":"1","":"downstairs","session_id":"b39a6c92-df4f-4962-a63e-754d3c7b10ae"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:55.424538002Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1356,"session_id":"b39a6c92-df4f-4962-a63e-754d3c7b10ae"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:55.4245985Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1356,"client":"0","":"downstairs","session_id":"b39a6c92-df4f-4962-a63e-754d3c7b10ae"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:55.424618553Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1356,"client":"0","":"downstairs","session_id":"b39a6c92-df4f-4962-a63e-754d3c7b10ae"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:55.424633215Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1356,"session_id":"b39a6c92-df4f-4962-a63e-754d3c7b10ae"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:55.424671909Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1356,"client":"2","":"downstairs","session_id":"b39a6c92-df4f-4962-a63e-754d3c7b10ae"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:55.424689672Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1356,"client":"2","":"downstairs","session_id":"b39a6c92-df4f-4962-a63e-754d3c7b10ae"} {"msg":"[0]R flush_numbers[0..12]: [69, 69, 69, 62, 69, 69, 65, 60, 66, 66, 69, 66]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:55.424703624Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1356,"":"downstairs","":"downstairs","session_id":"b39a6c92-df4f-4962-a63e-754d3c7b10ae"} {"msg":"[0]R generation[0..12]: [42, 42, 42, 38, 42, 42, 40, 37, 40, 40, 42, 40]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:55.424717605Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1356,"":"downstairs","":"downstairs","session_id":"b39a6c92-df4f-4962-a63e-754d3c7b10ae"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:55.424731477Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1356,"":"downstairs","":"downstairs","session_id":"b39a6c92-df4f-4962-a63e-754d3c7b10ae"} {"msg":"[1]R flush_numbers[0..12]: [69, 69, 69, 62, 69, 69, 65, 60, 66, 66, 69, 66]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:55.424745069Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1356,"":"downstairs","":"downstairs","session_id":"b39a6c92-df4f-4962-a63e-754d3c7b10ae"} {"msg":"[1]R generation[0..12]: [42, 42, 42, 38, 42, 42, 40, 37, 40, 40, 42, 40]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:55.424758511Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1356,"":"downstairs","":"downstairs","session_id":"b39a6c92-df4f-4962-a63e-754d3c7b10ae"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:55.424772273Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1356,"":"downstairs","":"downstairs","session_id":"b39a6c92-df4f-4962-a63e-754d3c7b10ae"} {"msg":"[2]R flush_numbers[0..12]: [69, 69, 69, 62, 69, 69, 65, 60, 66, 66, 69, 66]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:55.424788935Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1356,"":"downstairs","":"downstairs","session_id":"b39a6c92-df4f-4962-a63e-754d3c7b10ae"} {"msg":"[2]R generation[0..12]: [42, 42, 42, 38, 42, 42, 40, 37, 40, 40, 42, 40]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:55.424803016Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1356,"":"downstairs","":"downstairs","session_id":"b39a6c92-df4f-4962-a63e-754d3c7b10ae"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:55.424817158Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1356,"":"downstairs","":"downstairs","session_id":"b39a6c92-df4f-4962-a63e-754d3c7b10ae"} {"msg":"Max found gen is 43","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:55.42483013Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1356,"":"downstairs","":"downstairs","session_id":"b39a6c92-df4f-4962-a63e-754d3c7b10ae"} {"msg":"Generation requested: 44 >= found:43","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:55.424843502Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1356,"":"downstairs","":"downstairs","session_id":"b39a6c92-df4f-4962-a63e-754d3c7b10ae"} {"msg":"Next flush: 70","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:55.424856543Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1356,"":"downstairs","":"downstairs","session_id":"b39a6c92-df4f-4962-a63e-754d3c7b10ae"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:55.424869585Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1356,"":"downstairs","":"downstairs","session_id":"b39a6c92-df4f-4962-a63e-754d3c7b10ae"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:55.424882737Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1356,"session_id":"b39a6c92-df4f-4962-a63e-754d3c7b10ae"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-01T16:51:55.424894708Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1356,"client":"0","":"downstairs","session_id":"b39a6c92-df4f-4962-a63e-754d3c7b10ae"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:55.424912091Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1356,"client":"1","":"downstairs","session_id":"b39a6c92-df4f-4962-a63e-754d3c7b10ae"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:55.424925292Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1356,"client":"2","":"downstairs","session_id":"b39a6c92-df4f-4962-a63e-754d3c7b10ae"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:55.424938724Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1356,"session_id":"b39a6c92-df4f-4962-a63e-754d3c7b10ae"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:55.424951056Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1356,"session_id":"b39a6c92-df4f-4962-a63e-754d3c7b10ae"} {"msg":"103e01f4-336d-4604-b8db-d52a9ae71ee8 is now active with session: b39a6c92-df4f-4962-a63e-754d3c7b10ae","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:55.424963807Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1356,"session_id":"b39a6c92-df4f-4962-a63e-754d3c7b10ae"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:55.424975889Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1356,"session_id":"b39a6c92-df4f-4962-a63e-754d3c7b10ae"} {"msg":"The guest has finished waiting for activation with:44","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:55.424990081Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1356} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 390 len 4096 data: 5 03/30 Write block 87 len 16384 data: 7 7 8 9 04/30 Read block 69 len 32768 05/30 Write block 152 len 20480 data: 4 4 6 6 6 06/30 Flush 07/30 Write block 546 len 36864 data: 6 6 6 6 4 7 5 6 6 08/30 Read block 417 len 40960 09/30 Write block 186 len 20480 data: 4 4 5 4 4 10/30 Read block 145 len 8192 11/30 Write block 555 len 24576 data: 6 6 6 6 7 4 12/30 Write block 536 len 24576 data: 4 4 3 2 2 3 13/30 Read block 457 len 16384 14/30 Read block 141 len 16384 15/30 Write block 122 len 4096 data: 6 16/30 Read block 156 len 36864 17/30 Read block 452 len 32768 18/30 Write block 487 len 20480 data: 4 5 5 5 6 19/30 Read block 94 len 12288 20/30 Flush 21/30 Write block 558 len 40960 data: 7 8 5 4 4 4 4 2 2 3 22/30 Read block 116 len 20480 23/30 Write block 382 len 20480 data: 6 6 5 6 6 24/30 Read block 327 len 20480 25/30 Write block 245 len 32768 data: 6 7 9 8 8 9 8 7 26/30 Write block 424 len 32768 data: 6 6 6 5 6 6 5 5 27/30 Read block 23 len 28672 28/30 Write block 294 len 36864 data: 6 4 5 5 4 4 4 3 4 29/30 Read block 535 len 24576 30/30 Write block 286 len 16384 data: 3 3 3 4 ---------------------------------------------------------------- Crucible gen:44 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1020 Acked Write 10 Done Done Done false 1021 Acked Read 5 Done Done Done false 1022 Acked Write 5 Done Done Done false 1023 Acked Read 5 Done Done Done false 1024 Acked Write 8 Done Done Done false 1025 Acked Write 8 Done Done Done false 1026 Acked Read 7 Done Done Done false 1027 Acked Write 9 Done Done Done false 1028 Acked Read 6 Done Done Done false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 9 9 9 27 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1019 1019 1019 Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:51:55.448904822Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1356} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 42 42 42 69 69 69 F F F 1 020-039 42 42 42 69 69 69 F F F 2 040-059 42 42 42 69 69 69 F F F 3 060-079 38 38 38 62 62 62 F F F 4 080-099 44 44 44 71 71 71 F F F 5 100-119 42 42 42 69 69 69 F F F 6 120-139 44 44 44 72 72 72 F F F 7 140-159 44 44 44 71 71 71 F F F 8 160-179 40 40 40 66 66 66 F F F 9 180-199 44 44 44 72 72 72 F F F 10 200-219 42 42 42 69 69 69 F F F 11 220-239 40 40 40 66 66 66 F F F 12 240-259 40 40 40 66 66 66 T T T 13 260-279 42 42 42 69 69 69 F F F 14 280-299 39 39 39 63 63 63 T T T 15 300-319 39 39 39 63 63 63 T T T 16 320-339 42 42 42 69 69 69 F F F 17 340-359 33 33 33 55 55 55 F F F 18 360-379 40 40 40 65 65 65 F F F 19 380-399 44 44 44 71 71 71 T T T 20 400-419 27 27 27 41 41 41 F F F 21 420-439 34 34 34 56 56 56 T T T 22 440-459 41 41 41 68 68 68 F F F 23 460-479 42 42 42 69 69 69 F F F 24 480-499 44 44 44 72 72 72 F F F 25 500-519 41 41 41 68 68 68 F F F 26 520-539 44 44 44 72 72 72 F F F 27 540-559 44 44 44 72 72 72 T T T 28 560-579 44 44 44 72 72 72 T T T 29 580-599 42 42 42 69 69 69 F F F Max gen: 44, Max flush: 72 A Difference in extent metadata is expected here On loop 22 Verifying data now No change after verify Loop: 22 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 42 42 42 69 69 69 F F F 1 020-039 42 42 42 69 69 69 F F F 2 040-059 42 42 42 69 69 69 F F F 3 060-079 38 38 38 62 62 62 F F F 4 080-099 44 44 44 71 71 71 F F F 5 100-119 42 42 42 69 69 69 F F F 6 120-139 44 44 44 72 72 72 F F F 7 140-159 44 44 44 71 71 71 F F F 8 160-179 40 40 40 66 66 66 F F F 9 180-199 44 44 44 72 72 72 F F F 10 200-219 42 42 42 69 69 69 F F F 11 220-239 40 40 40 66 66 66 F F F 12 240-259 45 45 45 73 73 73 F F F 13 260-279 42 42 42 69 69 69 F F F 14 280-299 45 45 45 73 73 73 F F F 15 300-319 45 45 45 73 73 73 F F F 16 320-339 42 42 42 69 69 69 F F F 17 340-359 33 33 33 55 55 55 F F F 18 360-379 40 40 40 65 65 65 F F F 19 380-399 45 45 45 73 73 73 F F F 20 400-419 27 27 27 41 41 41 F F F 21 420-439 45 45 45 73 73 73 F F F 22 440-459 41 41 41 68 68 68 F F F 23 460-479 42 42 42 69 69 69 F F F 24 480-499 44 44 44 72 72 72 F F F 25 500-519 41 41 41 68 68 68 F F F 26 520-539 44 44 44 72 72 72 F F F 27 540-559 45 45 45 73 73 73 F F F 28 560-579 45 45 45 73 73 73 F F F 29 580-599 42 42 42 69 69 69 F F F Max gen: 45, Max flush: 73 Begin loop 23 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:59.775261301Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1364,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:51:59.819354572Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1364,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:59.819727731Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1364,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:59.851459819Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1364,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:59.851963144Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1364} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"3818f0fc76fe787c6cc412af30fd59b930203885\",\n git_commit_timestamp: \"2024-11-01T16:08:30.000000000Z\",\n git_branch: \"renovate/hyper-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:59.851994338Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1364} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:59.85201153Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1364} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:59.852026292Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1364} {"msg":"Crucible 3025a0d0-b96a-4c04-8c2c-3852098b5ef7 has session id: 62ab835c-f5a6-4c18-a585-cba9bcee3702","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:59.85239814Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1364,"session_id":"62ab835c-f5a6-4c18-a585-cba9bcee3702"} {"msg":"Upstairs opts: Upstairs UUID: 3025a0d0-b96a-4c04-8c2c-3852098b5ef7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:59.852527887Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1364,"session_id":"62ab835c-f5a6-4c18-a585-cba9bcee3702"} {"msg":"Crucible stats registered with UUID: 3025a0d0-b96a-4c04-8c2c-3852098b5ef7","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:59.852559101Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1364,"session_id":"62ab835c-f5a6-4c18-a585-cba9bcee3702"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:59.883701753Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1364,"task":"crutest"} {"msg":"The guest has requested activation with gen:46","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:59.883729817Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1364} {"msg":"3025a0d0-b96a-4c04-8c2c-3852098b5ef7 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:59.883794005Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1364,"session_id":"62ab835c-f5a6-4c18-a585-cba9bcee3702"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:59.883821659Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1364,"client":"0","":"downstairs","session_id":"62ab835c-f5a6-4c18-a585-cba9bcee3702"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:59.883846572Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1364,"client":"0","":"downstairs","session_id":"62ab835c-f5a6-4c18-a585-cba9bcee3702"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:59.883861634Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1364,"client":"1","":"downstairs","session_id":"62ab835c-f5a6-4c18-a585-cba9bcee3702"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:59.883875576Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1364,"client":"1","":"downstairs","session_id":"62ab835c-f5a6-4c18-a585-cba9bcee3702"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:59.883889347Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1364,"client":"2","":"downstairs","session_id":"62ab835c-f5a6-4c18-a585-cba9bcee3702"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:59.883903119Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1364,"client":"2","":"downstairs","session_id":"62ab835c-f5a6-4c18-a585-cba9bcee3702"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:59.883916571Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1364,"":"io task","client":"2","":"downstairs","session_id":"62ab835c-f5a6-4c18-a585-cba9bcee3702"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:59.883934613Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1364,"":"io task","client":"0","":"downstairs","session_id":"62ab835c-f5a6-4c18-a585-cba9bcee3702"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:59.883951106Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1364,"":"io task","client":"1","":"downstairs","session_id":"62ab835c-f5a6-4c18-a585-cba9bcee3702"} {"msg":"ds_connection connected from Ok(127.0.0.1:44817)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:59.8840664Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1364,"":"io task","client":"2","":"downstairs","session_id":"62ab835c-f5a6-4c18-a585-cba9bcee3702"} {"msg":"ds_connection connected from Ok(127.0.0.1:53878)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:59.884240363Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1364,"":"io task","client":"0","":"downstairs","session_id":"62ab835c-f5a6-4c18-a585-cba9bcee3702"} {"msg":"ds_connection connected from Ok(127.0.0.1:48591)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:59.884276297Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1364,"":"io task","client":"1","":"downstairs","session_id":"62ab835c-f5a6-4c18-a585-cba9bcee3702"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:59.884964686Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1364,"client":"1","":"downstairs","session_id":"62ab835c-f5a6-4c18-a585-cba9bcee3702"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:59.885001311Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1364,"client":"1","":"downstairs","session_id":"62ab835c-f5a6-4c18-a585-cba9bcee3702"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:59.885028804Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1364,"client":"0","":"downstairs","session_id":"62ab835c-f5a6-4c18-a585-cba9bcee3702"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:59.885057898Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1364,"client":"0","":"downstairs","session_id":"62ab835c-f5a6-4c18-a585-cba9bcee3702"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:59.885088452Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1364,"client":"2","":"downstairs","session_id":"62ab835c-f5a6-4c18-a585-cba9bcee3702"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:59.885108324Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1364,"client":"2","":"downstairs","session_id":"62ab835c-f5a6-4c18-a585-cba9bcee3702"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:59.885596618Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1364,"client":"2","":"downstairs","session_id":"62ab835c-f5a6-4c18-a585-cba9bcee3702"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:59.885638033Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1364,"client":"1","":"downstairs","session_id":"62ab835c-f5a6-4c18-a585-cba9bcee3702"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:59.88576919Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1364,"client":"0","":"downstairs","session_id":"62ab835c-f5a6-4c18-a585-cba9bcee3702"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:59.88615692Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1364,"client":"1","":"downstairs","session_id":"62ab835c-f5a6-4c18-a585-cba9bcee3702"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:59.886193404Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1364,"client":"1","":"downstairs","session_id":"62ab835c-f5a6-4c18-a585-cba9bcee3702"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:59.886220688Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1364,"session_id":"62ab835c-f5a6-4c18-a585-cba9bcee3702"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:59.886246401Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1364,"client":"2","":"downstairs","session_id":"62ab835c-f5a6-4c18-a585-cba9bcee3702"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:59.886271745Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1364,"client":"2","":"downstairs","session_id":"62ab835c-f5a6-4c18-a585-cba9bcee3702"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:59.886300838Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1364,"session_id":"62ab835c-f5a6-4c18-a585-cba9bcee3702"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:59.886323641Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1364,"client":"0","":"downstairs","session_id":"62ab835c-f5a6-4c18-a585-cba9bcee3702"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:59.886347044Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1364,"client":"0","":"downstairs","session_id":"62ab835c-f5a6-4c18-a585-cba9bcee3702"} {"msg":"[0]R flush_numbers[0..12]: [69, 69, 69, 62, 71, 69, 72, 71, 66, 72, 69, 66]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:59.886371137Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1364,"":"downstairs","":"downstairs","session_id":"62ab835c-f5a6-4c18-a585-cba9bcee3702"} {"msg":"[0]R generation[0..12]: [42, 42, 42, 38, 44, 42, 44, 44, 40, 44, 42, 40]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:59.886396831Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1364,"":"downstairs","":"downstairs","session_id":"62ab835c-f5a6-4c18-a585-cba9bcee3702"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:59.886422084Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1364,"":"downstairs","":"downstairs","session_id":"62ab835c-f5a6-4c18-a585-cba9bcee3702"} {"msg":"[1]R flush_numbers[0..12]: [69, 69, 69, 62, 71, 69, 72, 71, 66, 72, 69, 66]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:59.886444877Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1364,"":"downstairs","":"downstairs","session_id":"62ab835c-f5a6-4c18-a585-cba9bcee3702"} {"msg":"[1]R generation[0..12]: [42, 42, 42, 38, 44, 42, 44, 44, 40, 44, 42, 40]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:59.88646758Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1364,"":"downstairs","":"downstairs","session_id":"62ab835c-f5a6-4c18-a585-cba9bcee3702"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:59.886491463Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1364,"":"downstairs","":"downstairs","session_id":"62ab835c-f5a6-4c18-a585-cba9bcee3702"} {"msg":"[2]R flush_numbers[0..12]: [69, 69, 69, 62, 71, 69, 72, 71, 66, 72, 69, 66]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:59.886517376Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1364,"":"downstairs","":"downstairs","session_id":"62ab835c-f5a6-4c18-a585-cba9bcee3702"} {"msg":"[2]R generation[0..12]: [42, 42, 42, 38, 44, 42, 44, 44, 40, 44, 42, 40]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:59.88654685Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1364,"":"downstairs","":"downstairs","session_id":"62ab835c-f5a6-4c18-a585-cba9bcee3702"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:59.886574813Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1364,"":"downstairs","":"downstairs","session_id":"62ab835c-f5a6-4c18-a585-cba9bcee3702"} {"msg":"Max found gen is 46","v":0,"name":"crucible","level":30,"Wait for a query_work_queue command to finish before sending IO time":"2024-11-01T16:51:59.886602217Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1364,"":"downstairs","":"downstairs","session_id":"62ab835c-f5a6-4c18-a585-cba9bcee3702"} {"msg":"Generation requested: 46 >= found:46","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:59.886639282Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1364,"":"downstairs","":"downstairs","session_id":"62ab835c-f5a6-4c18-a585-cba9bcee3702"} {"msg":"Next flush: 74","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:59.886668726Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1364,"":"downstairs","":"downstairs","session_id":"62ab835c-f5a6-4c18-a585-cba9bcee3702"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:59.886695349Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1364,"":"downstairs","":"downstairs","session_id":"62ab835c-f5a6-4c18-a585-cba9bcee3702"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:59.886722612Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1364,"session_id":"62ab835c-f5a6-4c18-a585-cba9bcee3702"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:59.886747496Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1364,"client":"0","":"downstairs","session_id":"62ab835c-f5a6-4c18-a585-cba9bcee3702"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:59.886774529Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1364,"client":"1","":"downstairs","session_id":"62ab835c-f5a6-4c18-a585-cba9bcee3702"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:59.886802773Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1364,"client":"2","":"downstairs","session_id":"62ab835c-f5a6-4c18-a585-cba9bcee3702"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:59.886824256Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1364,"session_id":"62ab835c-f5a6-4c18-a585-cba9bcee3702"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:59.886837087Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1364,"session_id":"62ab835c-f5a6-4c18-a585-cba9bcee3702"} {"msg":"3025a0d0-b96a-4c04-8c2c-3852098b5ef7 is now active with session: 62ab835c-f5a6-4c18-a585-cba9bcee3702","v":0,"name":"crucible","level":30Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2024-11-01T16:51:59.886850219Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1364,"session_id":"62ab835c-f5a6-4c18-a585-cba9bcee3702"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:59.886866421Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1364,"session_id":"62ab835c-f5a6-4c18-a585-cba9bcee3702"} {"msg":"The guest has finished waiting for activation with:46","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:51:59.886879063Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1364} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 387 len 12288 data: 5 4 6 02/30 Read block 364 len 4096 03/30 Flush 04/30 Write block 203 len 20480 data: 4 5 8 8 8 05/30 Write block 90 len 4096 data: 10 06/30 Flush 07/30 Write block 197 len 32768 data: 5 5 6 4 5 5 5 6 08/30 Read block 260 len 12288 09/30 Read block 137 len 8192 10/30 Read block 369 len 28672 11/30 Read block 38 len 8192 12/30 Write block 189 len 36864 data: 5 5 3 4 5 6 6 6 6 13/30 Flush 14/30 Write block 272 len 40960 data: 5 7 9 9 7 5 4 5 4 4 15/30 Read block 418 len 40960 16/30 Read block 352 len 8192 17/30 Read block 106 len 8192 18/30 Read block 143 len 16384 19/30 Read block 433 len 36864 20/30 Read block 418 len 8192 21/30 Write block 496 len 4096 data: 3 22/30 Write block 471 len 32768 data: 7 7 6 6 7 5 6 8 23/30 Read block 462 len 16384 24/30 Write block 258 len 40960 data: 4 5 5 7 6 6 5 5 5 4 25/30 Write block 482 len 36864 data: 5 4 4 4 4 5 6 6 6 26/30 Read block 28 len 32768 27/30 Read block 181 len 24576 28/30 Write block 206 len 36864 data: 9 9 7 6 7 8 7 6 4 29/30 Write block 227 len 4096 data: 4 30/30 Read block 194 len 40960 ---------------------------------------------------------------- Crucible gen:46 GIO:true work queues: Upstairs:0 downstairs:17 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1013 Acked Write 10 Done Done Done false 1014 Acked Read 10 Done Done Done false 1015 Acked Read 2 Done Done Done false 1016 Acked Read 2 Done Done Done false 1017 Acked Read 4 Done Done Done false 1018 Acked Read 9 Done Done Done false 1019 Acked Read 2 Done Done Done false 1020 Acked Write 1 Done Done Done false 1021 Acked Write 8 Done Done Done false 1022 Acked Read 4 Done Done Done false 1023 Acked Write 10 Done Done Done false 1024 Acked Write 9 Done Done Done false 1025 Acked Read 8 Done Done Done false 1026 Acked Read 6 Done Done Done false 1027 Acked Write 9 Done Done Sent false 1028 Acked Write 1 Done Done Sent false 1029 Acked Read 10 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 3 4 Done 16 17 14 47 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1012 1012 1012 Downstairs last five completed: 1012 1011 1010 1009 1008 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:51:59.911315431Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1364} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 42 42 42 69 69 69 F F F 1 020-039 42 42 42 69 69 69 F F F 2 040-059 42 42 42 69 69 69 F F F 3 060-079 38 38 38 62 62 62 F F F 4 080-099 46 46 46 75 75 75 F F F 5 100-119 42 42 42 69 69 69 F F F 6 120-139 44 44 44 72 72 72 F F F 7 140-159 44 44 44 71 71 71 F F F 8 160-179 40 40 40 66 66 66 F F F 9 180-199 46 46 46 76 76 76 F F F 10 200-219 46 46 46 76 76 76 T T T 11 220-239 40 40 40 66 66 66 T T T 12 240-259 45 45 45 73 73 73 T T T 13 260-279 42 42 42 69 69 69 T T T 14 280-299 45 45 45 73 73 73 T T T 15 300-319 45 45 45 73 73 73 F F F 16 320-339 42 42 42 69 69 69 F F F 17 340-359 33 33 33 55 55 55 F F F 18 360-379 40 40 40 65 65 65 F F F 19 380-399 46 46 46 74 74 74 F F F 20 400-419 27 27 27 41 41 41 F F F 21 420-439 45 45 45 73 73 73 F F F 22 440-459 41 41 41 68 68 68 F F F 23 460-479 42 42 42 69 69 69 T T T 24 480-499 44 44 44 72 72 72 T T T 25 500-519 41 41 41 68 68 68 F F F 26 520-539 44 44 44 72 72 72 F F F 27 540-559 45 45 45 73 73 73 F F F 28 560-579 45 45 45 73 73 73 F F F 29 580-599 42 42 42 69 69 69 F F F Max gen: 46, Max flush: 76 A Difference in extent metadata is expected here On loop 23 Verifying data now No change after verify Loop: 23 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 42 42 42 69 69 69 F F F 1 020-039 42 42 42 69 69 69 F F F 2 040-059 42 42 42 69 69 69 F F F 3 060-079 38 38 38 62 62 62 F F F 4 080-099 46 46 46 75 75 75 F F F 5 100-119 42 42 42 69 69 69 F F F 6 120-139 44 44 44 72 72 72 F F F 7 140-159 44 44 44 71 71 71 F F F 8 160-179 40 40 40 66 66 66 F F F 9 180-199 46 46 46 76 76 76 F F F 10 200-219 47 47 47 77 77 77 F F F 11 220-239 47 47 47 77 77 77 F F F 12 240-259 47 47 47 77 77 77 F F F 13 260-279 47 47 47 77 77 77 F F F 14 280-299 47 47 47 77 77 77 F F F 15 300-319 45 45 45 73 73 73 F F F 16 320-339 42 42 42 69 69 69 F F F 17 340-359 33 33 33 55 55 55 F F F 18 360-379 40 40 40 65 65 65 F F F 19 380-399 46 46 46 74 74 74 F F F 20 400-419 27 27 27 41 41 41 F F F 21 420-439 45 45 45 73 73 73 F F F 22 440-459 41 41 41 68 68 68 F F F 23 460-479 47 47 47 77 77 77 F F F 24 480-499 47 47 47 77 77 77 F F F 25 500-519 41 41 41 68 68 68 F F F 26 520-539 44 44 44 72 72 72 F F F 27 540-559 45 45 45 73 73 73 F F F 28 560-579 45 45 45 73 73 73 F F F 29 580-599 42 42 42 69 69 69 F F F Max gen: 47, Max flush: 77 Begin loop 24 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:04.266342944Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1372,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:04.310122145Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1372,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:04.310729613Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1372} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"3818f0fc76fe787c6cc412af30fd59b930203885\",\n git_commit_timestamp: \"2024-11-01T16:08:30.000000000Z\",\n git_branch: \"renovate/hyper-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:04.31078349Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1372} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:04.310805723Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1372} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:04.310829136Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1372} {"msg":"Crucible 2291bb98-5e41-4244-8bbb-e04b1ed2acb8 has session id: 2253bbc4-4524-4931-b871-1177081e78ba","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:04.311218556Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1372,"session_id":"2253bbc4-4524-4931-b871-1177081e78ba"} {"msg":"Upstairs opts: Upstairs UUID: 2291bb98-5e41-4244-8bbb-e04b1ed2acb8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:04.311259441Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1372,"session_id":"2253bbc4-4524-4931-b871-1177081e78ba"} {"msg":"Crucible stats registered with UUID: 2291bb98-5e41-4244-8bbb-e04b1ed2acb8","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:04.311282935Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1372,"session_id":"2253bbc4-4524-4931-b871-1177081e78ba"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:04.342161323Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1372,"task":"crutest"} {"msg":"The guest has requested activation with gen:48","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:04.342195988Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1372} {"msg":"2291bb98-5e41-4244-8bbb-e04b1ed2acb8 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:04.342233492Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1372,"session_id":"2253bbc4-4524-4931-b871-1177081e78ba"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:04.342250365Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1372,"client":"0","":"downstairs","session_id":"2253bbc4-4524-4931-b871-1177081e78ba"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:04.342279809Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1372,"client":"0","":"downstairs","session_id":"2253bbc4-4524-4931-b871-1177081e78ba"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:04.342297651Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1372,"client":"1","":"downstairs","session_id":"2253bbc4-4524-4931-b871-1177081e78ba"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:04.342314753Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1372,"client":"1","":"downstairs","session_id":"2253bbc4-4524-4931-b871-1177081e78ba"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:04.342331385Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1372,"client":"2","":"downstairs","session_id":"2253bbc4-4524-4931-b871-1177081e78ba"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:04.342358799Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1372,"client":"2","":"downstairs","session_id":"2253bbc4-4524-4931-b871-1177081e78ba"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:04.342391913Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1372,"":"io task","client":"2","":"downstairs","session_id":"2253bbc4-4524-4931-b871-1177081e78ba"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:04.342421987Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1372,"":"io task","client":"0","":"downstairs","session_id":"2253bbc4-4524-4931-b871-1177081e78ba"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:04.34245092Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1372,"":"io task","client":"1","":"downstairs","session_id":"2253bbc4-4524-4931-b871-1177081e78ba"} {"msg":"ds_connection connected from Ok(127.0.0.1:54586)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:04.342482275Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1372,"":"io task","client":"2","":"downstairs","session_id":"2253bbc4-4524-4931-b871-1177081e78ba"} {"msg":"ds_connection connected from Ok(127.0.0.1:65021)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:04.342509488Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1372,"":"io task","client":"1","":"downstairs","session_id":"2253bbc4-4524-4931-b871-1177081e78ba"} {"msg":"ds_connection connected from Ok(127.0.0.1:57709)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:04.342708694Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1372,"":"io task","client":"0","":"downstairs","session_id":"2253bbc4-4524-4931-b871-1177081e78ba"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:04.34322162Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1372,"client":"2","":"downstairs","session_id":"2253bbc4-4524-4931-b871-1177081e78ba"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:04.343251984Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1372,"client":"2","":"downstairs","session_id":"2253bbc4-4524-4931-b871-1177081e78ba"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:04.343268086Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1372,"client":"1","":"downstairs","session_id":"2253bbc4-4524-4931-b871-1177081e78ba"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:04.343282538Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1372,"client":"1","":"downstairs","session_id":"2253bbc4-4524-4931-b871-1177081e78ba"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:04.343310961Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1372,"client":"0","":"downstairs","session_id":"2253bbc4-4524-4931-b871-1177081e78ba"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:04.343325223Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1372,"client":"0","":"downstairs","session_id":"2253bbc4-4524-4931-b871-1177081e78ba"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:04.343915739Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1372,"client":"0","":"downstairs","session_id":"2253bbc4-4524-4931-b871-1177081e78ba"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:04.343944473Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1372,"client":"2","":"downstairs","session_id":"2253bbc4-4524-4931-b871-1177081e78ba"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:04.344022143Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1372,"client":"1","":"downstairs","session_id":"2253bbc4-4524-4931-b871-1177081e78ba"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:04.34454512Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1372,"client":"2","":"downstairs","session_id":"2253bbc4-4524-4931-b871-1177081e78ba"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:04.344573544Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1372,"client":"2","":"downstairs","session_id":"2253bbc4-4524-4931-b871-1177081e78ba"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:04.344589206Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1372,"session_id":"2253bbc4-4524-4931-b871-1177081e78ba"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:04.344605288Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1372,"client":"0","":"downstairs","session_id":"2253bbc4-4524-4931-b871-1177081e78ba"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:04.34461931Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1372,"client":"0","":"downstairs","session_id":"2253bbc4-4524-4931-b871-1177081e78ba"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:04.344632872Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1372,"session_id":"2253bbc4-4524-4931-b871-1177081e78ba"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:04.344645193Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1372,"client":"1","":"downstairs","session_id":"2253bbc4-4524-4931-b871-1177081e78ba"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:04.344658755Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1372,"client":"1","":"downstairs","session_id":"2253bbc4-4524-4931-b871-1177081e78ba"} {"msg":"[0]R flush_numbers[0..12]: [69, 69, 69, 62, 75, 69, 72, 71, 66, 76, 77, 77]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:04.344672777Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1372,"":"downstairs","":"downstairs","session_id":"2253bbc4-4524-4931-b871-1177081e78ba"} {"msg":"[0]R generation[0..12]: [42, 42, 42, 38, 46, 42, 44, 44, 40, 46, 47, 47]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:04.344686659Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1372,"":"downstairs","":"downstairs","session_id":"2253bbc4-4524-4931-b871-1177081e78ba"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:04.34470083Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1372,"":"downstairs","":"downstairs","session_id":"2253bbc4-4524-4931-b871-1177081e78ba"} {"msg":"[1]R flush_numbers[0..12]: [69, 69, 69, 62, 75, 69, 72, 71, 66, 76, 77, 77]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:04.344714602Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1372,"":"downstairs","":"downstairs","session_id":"2253bbc4-4524-4931-b871-1177081e78ba"} {"msg":"[1]R generation[0..12]: [42, 42, 42, 38, 46, 42, 44, 44, 40, 46, 47, 47]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:04.344728454Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1372,"":"downstairs","":"downstairs","session_id":"2253bbc4-4524-4931-b871-1177081e78ba"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:04.344742306Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1372,"":"downstairs","":"downstairs","session_id":"2253bbc4-4524-4931-b871-1177081e78ba"} {"msg":"[2]R flush_numbers[0..12]: [69, 69, 69, 62, 75, 69, 72, 71, 66, 76, 77, 77]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:04.344756168Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1372,"":"downstairs","":"downstairs","session_id":"2253bbc4-4524-4931-b871-1177081e78ba"} {"msg":"[2]R generation[0..12]: [42, 42, 42, 38, 46, 42, 44, 44, 40, 46, 47, 47]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:04.3447768Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1372,"":"downstairs","":"downstairs","session_id":"2253bbc4-4524-4931-b871-1177081e78baWait for a query_work_queue command to finish before sending IO "} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:04.344860391Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1372,"":"downstairs","":"downstairs","session_id":"2253bbc4-4524-4931-b871-1177081e78ba"} {"msg":"Max found gen is 48","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:04.344897576Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1372,"":"downstairs","":"downstairs","session_id":"2253bbc4-4524-4931-b871-1177081e78ba"} {"msg":"Generation requested: 48 >= found:48","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:04.344926739Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1372,"":"downstairs","":"downstairs","session_id":"2253bbc4-4524-4931-b871-1177081e78ba"} {"msg":"Next flush: 78","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:04.344954003Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1372,"":"downstairs","":"downstairs","session_id":"2253bbc4-4524-4931-b871-1177081e78ba"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:04.344981877Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1372,"":"downstairs","":"downstairs","session_id":"2253bbc4-4524-4931-b871-1177081e78ba"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:04.34500899Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1372,"session_id":"2253bbc4-4524-4931-b871-1177081e78ba"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:04.345034553Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1372,"client":"0","":"downstairs","session_id":"2253bbc4-4524-4931-b871-1177081e78ba"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:04.345061397Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1372,"client":"1","":"downstairs","session_id":"2253bbc4-4524-4931-b871-1177081e78ba"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2024-11-01T16:52:04.34508338Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1372,"client":"2","":"downstairs","session_id":"2253bbc4-4524-4931-b871-1177081e78ba"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:04.345107273Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1372,"session_id":"2253bbc4-4524-4931-b871-1177081e78ba"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:04.345131486Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1372,"session_id":"2253bbc4-4524-4931-b871-1177081e78ba"} {"msg":"2291bb98-5e41-4244-8bbb-e04b1ed2acb8 is now active with session: 2253bbc4-4524-4931-b871-1177081e78ba","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:04.345150458Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1372,"session_id":"2253bbc4-4524-4931-b871-1177081e78ba"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:04.34516308Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1372,"session_id":"2253bbc4-4524-4931-b871-1177081e78ba"} {"msg":"The guest has finished waiting for activation with:48","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:04.345175491Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1372} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 282 len 20480 data: 4 3 2 2 4 02/30 Write block 245 len 24576 data: 7 8 10 9 9 10 03/30 Write block 508 len 20480 data: 5 4 5 5 5 04/30 Write block 388 len 4096 data: 5 05/30 Write block 214 len 24576 data: 5 3 7 7 9 5 06/30 Write block 365 len 40960 data: 5 4 3 3 5 4 3 3 4 3 07/30 Write block 449 len 20480 data: 5 6 6 4 5 08/30 Write block 447 len 32768 data: 6 5 6 7 7 5 6 5 09/30 Write block 569 len 8192 data: 4 3 10/30 Read block 44 len 20480 11/30 Write block 72 len 16384 data: 9 7 6 5 12/30 Write block 462 len 4096 data: 4 13/30 Read block 60 len 8192 14/30 Read block 271 len 28672 15/30 Read block 482 len 8192 16/30 Write block 78 len 28672 data: 3 6 6 4 7 8 7 17/30 Read block 330 len 12288 18/30 Write block 90 len 24576 data: 11 7 5 7 6 5 19/30 Flush 20/30 Write block 398 len 4096 data: 8 21/30 Write block 585 len 40960 data: 6 6 6 4 5 4 4 4 5 4 22/30 Read block 163 len 32768 23/30 Write block 92 len 12288 data: 6 8 7 24/30 Write block 44 len 24576 data: 8 6 7 4 3 6 25/30 Read block 530 len 24576 26/30 Write block 239 len 12288 data: 4 4 4 27/30 Read block 135 len 20480 28/30 Read block 357 len 36864 29/30 Read block 203 len 16384 30/30 Read block 504 len 24576 ---------------------------------------------------------------- Crucible gen:48 GIO:true work queues: Upstairs:0 downstairs:11 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1019 Acked Write 1 Done Done Done false 1020 Acked Write 10 Done Done Done false 1021 Acked Read 8 Done Done Done false 1022 Acked Write 3 Done Done Done false 1023 Acked Write 6 Done Done Done false 1024 Acked Read 6 Done Done Done false 1025 Acked Write 3 Done Done Done false 1026 Acked Read 5 Done Done Done false 1027 Acked Read 9 Done Done Done false 1028 Acked Read 4 Done Done Done false 1029 Acked Read 6 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 0 1 Done 11 10 11 32 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1018 1018 1018 Downstairs last five completed: 1018 1017 1016 1015 1014 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:52:04.365876229Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1372} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 42 42 42 69 69 69 F F F 1 020-039 42 42 42 69 69 69 F F F 2 040-059 42 42 42 69 69 69 T T T 3 060-079 48 48 48 78 78 78 F F F 4 080-099 48 48 48 78 78 78 T T T 5 100-119 42 42 42 69 69 69 F F F 6 120-139 44 44 44 72 72 72 F F F 7 140-159 44 44 44 71 71 71 F F F 8 160-179 40 40 40 66 66 66 F F F 9 180-199 46 46 46 76 76 76 F F F 10 200-219 48 48 48 78 78 78 F F F 11 220-239 47 47 47 77 77 77 T T T 12 240-259 48 48 48 78 78 78 T T T 13 260-279 47 47 47 77 77 77 F F F 14 280-299 48 48 48 78 78 78 F F F 15 300-319 45 45 45 73 73 73 F F F 16 320-339 42 42 42 69 69 69 F F F 17 340-359 33 33 33 55 55 55 F F F 18 360-379 48 48 48 78 78 78 F F F 19 380-399 48 48 48 78 78 78 T T T 20 400-419 27 27 27 41 41 41 F F F 21 420-439 45 45 45 73 73 73 F F F 22 440-459 48 48 48 78 78 78 F F F 23 460-479 48 48 48 78 78 78 F F F 24 480-499 47 47 47 77 77 77 F F F 25 500-519 48 48 48 78 78 78 F F F 26 520-539 44 44 44 72 72 72 F F F 27 540-559 45 45 45 73 73 73 F F F 28 560-579 48 48 48 78 78 78 F F F 29 580-599 42 42 42 69 69 69 T T T Max gen: 48, Max flush: 78 A Difference in extent metadata is expected here On loop 24 Verifying data now No change after verify Loop: 24 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 42 42 42 69 69 69 F F F 1 020-039 42 42 42 69 69 69 F F F 2 040-059 49 49 49 79 79 79 F F F 3 060-079 48 48 48 78 78 78 F F F 4 080-099 49 49 49 79 79 79 F F F 5 100-119 42 42 42 69 69 69 F F F 6 120-139 44 44 44 72 72 72 F F F 7 140-159 44 44 44 71 71 71 F F F 8 160-179 40 40 40 66 66 66 F F F 9 180-199 46 46 46 76 76 76 F F F 10 200-219 48 48 48 78 78 78 F F F 11 220-239 49 49 49 79 79 79 F F F 12 240-259 49 49 49 79 79 79 F F F 13 260-279 47 47 47 77 77 77 F F F 14 280-299 48 48 48 78 78 78 F F F 15 300-319 45 45 45 73 73 73 F F F 16 320-339 42 42 42 69 69 69 F F F 17 340-359 33 33 33 55 55 55 F F F 18 360-379 48 48 48 78 78 78 F F F 19 380-399 49 49 49 79 79 79 F F F 20 400-419 27 27 27 41 41 41 F F F 21 420-439 45 45 45 73 73 73 F F F 22 440-459 48 48 48 78 78 78 F F F 23 460-479 48 48 48 78 78 78 F F F 24 480-499 47 47 47 77 77 77 F F F 25 500-519 48 48 48 78 78 78 F F F 26 520-539 44 44 44 72 72 72 F F F 27 540-559 45 45 45 73 73 73 F F F 28 560-579 48 48 48 78 78 78 F F F 29 580-599 49 49 49 79 79 79 F F F Max gen: 49, Max flush: 79 Begin loop 25 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:08.68503146Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1380,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:52:08.727500492Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1380,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:08.727897093Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1380,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:08.760065328Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1380,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:08.760595727Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1380} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"3818f0fc76fe787c6cc412af30fd59b930203885\",\n git_commit_timestamp: \"2024-11-01T16:08:30.000000000Z\",\n git_branch: \"renovate/hyper-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:08.760630171Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1380} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:08.760644083Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1380} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:08.760658215Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1380} {"msg":"Crucible b18f66b7-f2f6-4b61-9b36-382989688e41 has session id: c142cce0-db5a-4edc-a766-df3054eeb60d","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:08.761073298Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1380,"session_id":"c142cce0-db5a-4edc-a766-df3054eeb60d"} {"msg":"Upstairs opts: Upstairs UUID: b18f66b7-f2f6-4b61-9b36-382989688e41, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:08.761126505Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1380,"session_id":"c142cce0-db5a-4edc-a766-df3054eeb60d"} {"msg":"Crucible stats registered with UUID: b18f66b7-f2f6-4b61-9b36-382989688e41","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:08.761159549Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1380,"session_id":"c142cce0-db5a-4edc-a766-df3054eeb60d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:08.792412116Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1380,"task":"crutest"} {"msg":"The guest has requested activation with gen:50","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:08.792457091Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1380} {"msg":"b18f66b7-f2f6-4b61-9b36-382989688e41 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:08.792505528Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1380,"session_id":"c142cce0-db5a-4edc-a766-df3054eeb60d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:08.792538402Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1380,"client":"0","":"downstairs","session_id":"c142cce0-db5a-4edc-a766-df3054eeb60d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:08.792586698Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1380,"client":"0","":"downstairs","session_id":"c142cce0-db5a-4edc-a766-df3054eeb60d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:08.792624493Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1380,"client":"1","":"downstairs","session_id":"c142cce0-db5a-4edc-a766-df3054eeb60d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:08.792698482Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1380,"client":"1","":"downstairs","session_id":"c142cce0-db5a-4edc-a766-df3054eeb60d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:08.792741038Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1380,"client":"2","":"downstairs","session_id":"c142cce0-db5a-4edc-a766-df3054eeb60d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:08.792942384Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1380,"client":"2","":"downstairs","session_id":"c142cce0-db5a-4edc-a766-df3054eeb60d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:08.79298621Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1380,"":"io task","client":"2","":"downstairs","session_id":"c142cce0-db5a-4edc-a766-df3054eeb60d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:08.793006632Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1380,"":"io task","client":"0","":"downstairs","session_id":"c142cce0-db5a-4edc-a766-df3054eeb60d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:08.793025504Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1380,"":"io task","client":"1","":"downstairs","session_id":"c142cce0-db5a-4edc-a766-df3054eeb60d"} {"msg":"ds_connection connected from Ok(127.0.0.1:49101)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:08.793041177Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1380,"":"io task","client":"0","":"downstairs","session_id":"c142cce0-db5a-4edc-a766-df3054eeb60d"} {"msg":"ds_connection connected from Ok(127.0.0.1:65183)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:08.793058729Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1380,"":"io task","client":"2","":"downstairs","session_id":"c142cce0-db5a-4edc-a766-df3054eeb60d"} {"msg":"ds_connection connected from Ok(127.0.0.1:39288)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:08.793073461Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1380,"":"io task","client":"1","":"downstairs","session_id":"c142cce0-db5a-4edc-a766-df3054eeb60d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:08.793602799Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1380,"client":"1","":"downstairs","session_id":"c142cce0-db5a-4edc-a766-df3054eeb60d"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:08.793641754Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1380,"client":"1","":"downstairs","session_id":"c142cce0-db5a-4edc-a766-df3054eeb60d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:08.793671968Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1380,"client":"0","":"downstairs","session_id":"c142cce0-db5a-4edc-a766-df3054eeb60d"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:08.793701572Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1380,"client":"0","":"downstairs","session_id":"c142cce0-db5a-4edc-a766-df3054eeb60d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:08.793732886Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1380,"client":"2","":"downstairs","session_id":"c142cce0-db5a-4edc-a766-df3054eeb60d"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:08.793762179Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1380,"client":"2","":"downstairs","session_id":"c142cce0-db5a-4edc-a766-df3054eeb60d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:08.794267244Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1380,"client":"1","":"downstairs","session_id":"c142cce0-db5a-4edc-a766-df3054eeb60d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:08.794370148Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1380,"client":"0","":"downstairs","session_id":"c142cce0-db5a-4edc-a766-df3054eeb60d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:08.7943911Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1380,"client":"2","":"downstairs","session_id":"c142cce0-db5a-4edc-a766-df3054eeb60d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:08.79485218Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1380,"client":"1","":"downstairs","session_id":"c142cce0-db5a-4edc-a766-df3054eeb60d"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:08.794880283Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1380,"client":"1","":"downstairs","session_id":"c142cce0-db5a-4edc-a766-df3054eeb60d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:08.794895276Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1380,"session_id":"c142cce0-db5a-4edc-a766-df3054eeb60d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:08.794963424Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1380,"client":"0","":"downstairs","session_id":"c142cce0-db5a-4edc-a766-df3054eeb60d"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:08.794986977Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1380,"client":"0","":"downstairs","session_id":"c142cce0-db5a-4edc-a766-df3054eeb60d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:08.795001039Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1380,"session_id":"c142cce0-db5a-4edc-a766-df3054eeb60d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:08.795013451Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1380,"client":"2","":"downstairs","session_id":"c142cce0-db5a-4edc-a766-df3054eeb60d"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:08.795026962Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1380,"client":"2","":"downstairs","session_id":"c142cce0-db5a-4edc-a766-df3054eeb60d"} {"msg":"[0]R flush_numbers[0..12]: [69, 69, 79, 78, 79, 69, 72, 71, 66, 76, 78, 79]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:08.795040684Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1380,"":"downstairs","":"downstairs","session_id":"c142cce0-db5a-4edc-a766-df3054eeb60d"} {"msg":"[0]R generation[0..12]: [42, 42, 49, 48, 49, 42, 44, 44, 40, 46, 48, 49]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:08.795054596Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1380,"":"downstairs","":"downstairs","session_id":"c142cce0-db5a-4edc-a766-df3054eeb60d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:08.795068848Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1380,"":"downstairs","":"downstairs","session_id":"c142cce0-db5a-4edc-a766-df3054eeb60d"} {"msg":"[1]R flush_numbers[0..12]: [69, 69, 79, 78, 79, 69, 72, 71, 66, 76, 78, 79]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:08.79508253Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1380,"":"downstairs","":"downstairs","session_id":"c142cce0-db5a-4edc-a766-df3054eeb60d"} {"msg":"[1]R generation[0..12]: [42, 42, 49, 48, 49, 42, 44, 44, 40, 46, 48, 49]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:08.795096161Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1380,"":"downstairs","":"downstairs","session_id":"c142cce0-db5a-4edc-a766-df3054eeb60d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:08.795110053Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1380,"":"downstairs","":"downstairs","session_id":"c142cce0-db5a-4edc-a766-df3054eeb60d"} {"msg":"[2]R flush_numbers[0..12]: [69, 69, 79, 78, 79, 69, 72, 71, 66, 76, 78, 79]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:08.795123815Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1380,"":"downstairs","":"downstairs","session_id":"c142cce0-db5a-4edc-a766-df3054eeb60d"} {"msg":"[2]R generation[0..12]: [42, 42, 49, 48, 49, 42, 44, 44, 40, 46, 48, 49]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:08.795137337Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1380,"":"downstairs","":"downstairs","session_id":"c142cce0-db5a-4edc-a766-df3054eeb60d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:08.795151338Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1380,"":"downstairs","":"downstairs","session_id":"c142cce0-db5a-4edc-a766-df3054eeb60d"} {"msg":"Max found gen is 50","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:08.79516451Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1380,"":"downstairs","":"downstairs","session_id":"c142cce0-db5a-4edc-a766-df3054eeb60d"} {"msg":"Generation requested: 50 >= found:50","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-01T16:52:08.795177992Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1380,"":"downstairs","":"downstairs","session_id":"c142cce0-db5a-4edc-a766-df3054eeb60d"} {"msg":"Next flush: 80","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:08.795202125Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1380,"":"downstairs","":"downstairs","session_id":"c142cce0-db5a-4edc-a766-df3054eeb60d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:08.795215137Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1380,"":"downstairs","":"downstairs","session_id":"c142cce0-db5a-4edc-a766-df3054eeb60d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:08.795228228Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1380,"session_id":"c142cce0-db5a-4edc-a766-df3054eeb60d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:08.79524043Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1380,"client":"0","":"downstairs","session_id":"c142cce0-db5a-4edc-a766-df3054eeb60d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:08.795253862Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1380,"client":"1","":"downstairs","session_id":"c142cce0-db5a-4edc-a766-df3054eeb60d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:08.795267093Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1380,"client":"2","":"downstairs","session_id":"c142cce0-db5a-4edc-a766-df3054eeb60d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:08.795280345Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1380,"session_id":"c142cce0-db5a-4edc-a766-df3054eeb60d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:08.795292947Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1380,"session_id":"c142cce0-db5a-4edc-a766-df3054eeb60d"} {"msg":"b18f66b7-f2f6-4b61-9b36-382989688e41 is now active with session: c142cce0-db5a-4edc-a766-df3054eeb60d","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:08.795306118Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1380,"session_id":"c142cce0-db5a-4edc-a766-df3054eeb60d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:08.79531831Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1380,"session_id":"c142cce0-db5a-4edc-a766-df3054eeb60d"} {"msg":"The guest has finished waiting for activation with:50","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:08.795330962Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1380} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 311 len 28672 data: 4 4 4 4 6 7 5 02/30 Read block 254 len 32768 03/30 Write block 162 len 16384 data: 5 5 4 4 04/30 Read block 440 len 16384 05/30 Read block 245 len 16384 06/30 Write block 169 len 8192 data: 3 3 07/30 Read block 574 len 32768 08/30 Read block 215 len 24576 09/30 Read block 17 len 16384 10/30 Read block 28 len 12288 11/30 Read block 408 len 36864 12/30 Read block 58 len 4096 13/30 Write block 307 len 12288 data: 8 8 4 14/30 Read block 383 len 32768 15/30 Write block 124 len 32768 data: 4 4 4 4 4 5 6 7 16/30 Write block 276 len 28672 data: 8 6 5 6 5 5 5 17/30 Read block 547 len 20480 18/30 Read block 294 len 16384 19/30 Read block 514 len 20480 20/30 Write block 439 len 40960 data: 5 5 5 5 7 7 7 6 7 6 21/30 Read block 515 len 4096 22/30 Write block 467 len 4096 data: 4 23/30 Write block 124 len 24576 data: 5 5 5 5 5 6 24/30 Write block 201 len 16384 data: 6 6 6 7 25/30 Flush 26/30 Write block 220 len 8192 data: 5 5 27/30 Read block 237 len 40960 28/30 Read block 591 len 4096 29/30 Read block 396 len 40960 30/30 Write block 342 len 24576 data: 4 5 6 5 5 4 ---------------------------------------------------------------- Crucible gen:50 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 2 Done Done Done false 1026 Acked Read 10 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Read 10 Done Done Done false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:52:08.818586688Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1380} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 42 42 42 69 69 69 F F F 1 020-039 42 42 42 69 69 69 F F F 2 040-059 49 49 49 79 79 79 F F F 3 060-079 48 48 48 78 78 78 F F F 4 080-099 49 49 49 79 79 79 F F F 5 100-119 42 42 42 69 69 69 F F F 6 120-139 50 50 50 80 80 80 F F F 7 140-159 44 44 44 71 71 71 F F F 8 160-179 50 50 50 80 80 80 F F F 9 180-199 46 46 46 76 76 76 F F F 10 200-219 50 50 50 80 80 80 F F F 11 220-239 49 49 49 79 79 79 T T T 12 240-259 49 49 49 79 79 79 F F F 13 260-279 50 50 50 80 80 80 F F F 14 280-299 50 50 50 80 80 80 F F F 15 300-319 50 50 50 80 80 80 F F F 16 320-339 42 42 42 69 69 69 F F F 17 340-359 33 33 33 55 55 55 T T T 18 360-379 48 48 48 78 78 78 F F F 19 380-399 49 49 49 79 79 79 F F F 20 400-419 27 27 27 41 41 41 F F F 21 420-439 50 50 50 80 80 80 F F F 22 440-459 50 50 50 80 80 80 F F F 23 460-479 50 50 50 80 80 80 F F F 24 480-499 47 47 47 77 77 77 F F F 25 500-519 48 48 48 78 78 78 F F F 26 520-539 44 44 44 72 72 72 F F F 27 540-559 45 45 45 73 73 73 F F F 28 560-579 48 48 48 78 78 78 F F F 29 580-599 49 49 49 79 79 79 F F F Max gen: 50, Max flush: 80 A Difference in extent metadata is expected here On loop 25 Verifying data now No change after verify Loop: 25 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 42 42 42 69 69 69 F F F 1 020-039 42 42 42 69 69 69 F F F 2 040-059 49 49 49 79 79 79 F F F 3 060-079 48 48 48 78 78 78 F F F 4 080-099 49 49 49 79 79 79 F F F 5 100-119 42 42 42 69 69 69 F F F 6 120-139 50 50 50 80 80 80 F F F 7 140-159 44 44 44 71 71 71 F F F 8 160-179 50 50 50 80 80 80 F F F 9 180-199 46 46 46 76 76 76 F F F 10 200-219 50 50 50 80 80 80 F F F 11 220-239 51 51 51 81 81 81 F F F 12 240-259 49 49 49 79 79 79 F F F 13 260-279 50 50 50 80 80 80 F F F 14 280-299 50 50 50 80 80 80 F F F 15 300-319 50 50 50 80 80 80 F F F 16 320-339 42 42 42 69 69 69 F F F 17 340-359 51 51 51 81 81 81 F F F 18 360-379 48 48 48 78 78 78 F F F 19 380-399 49 49 49 79 79 79 F F F 20 400-419 27 27 27 41 41 41 F F F 21 420-439 50 50 50 80 80 80 F F F 22 440-459 50 50 50 80 80 80 F F F 23 460-479 50 50 50 80 80 80 F F F 24 480-499 47 47 47 77 77 77 F F F 25 500-519 48 48 48 78 78 78 F F F 26 520-539 44 44 44 72 72 72 F F F 27 540-559 45 45 45 73 73 73 F F F 28 560-579 48 48 48 78 78 78 F F F 29 580-599 49 49 49 79 79 79 F F F Max gen: 51, Max flush: 81 Begin loop 26 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:13.115271475Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1388,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:52:13.158862982Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1388,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:13.159253792Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1388,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:13.191726316Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1388,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:13.192266366Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1388} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"3818f0fc76fe787c6cc412af30fd59b930203885\",\n git_commit_timestamp: \"2024-11-01T16:08:30.000000000Z\",\n git_branch: \"renovate/hyper-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:13.19230224Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1388} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:13.192316302Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1388} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:13.192330764Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1388} {"msg":"Crucible 2089f9f4-0cb7-4566-bbb7-79cbea57bebb has session id: be2229f6-1e48-4330-bc8c-727b340176b0","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:13.192754929Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1388,"session_id":"be2229f6-1e48-4330-bc8c-727b340176b0"} {"msg":"Upstairs opts: Upstairs UUID: 2089f9f4-0cb7-4566-bbb7-79cbea57bebb, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:13.192802815Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1388,"session_id":"be2229f6-1e48-4330-bc8c-727b340176b0"} {"msg":"Crucible stats registered with UUID: 2089f9f4-0cb7-4566-bbb7-79cbea57bebb","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:13.192824738Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1388,"session_id":"be2229f6-1e48-4330-bc8c-727b340176b0"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:13.223549326Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1388,"task":"crutest"} {"msg":"The guest has requested activation with gen:52","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:13.22357893Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1388} {"msg":"2089f9f4-0cb7-4566-bbb7-79cbea57bebb active request set","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:13.223628456Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1388,"session_id":"be2229f6-1e48-4330-bc8c-727b340176b0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:13.223647949Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1388,"client":"0","":"downstairs","session_id":"be2229f6-1e48-4330-bc8c-727b340176b0"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:13.223675883Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1388,"client":"0","":"downstairs","session_id":"be2229f6-1e48-4330-bc8c-727b340176b0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:13.223693225Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1388,"client":"1","":"downstairs","session_id":"be2229f6-1e48-4330-bc8c-727b340176b0"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:13.223710967Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1388,"client":"1","":"downstairs","session_id":"be2229f6-1e48-4330-bc8c-727b340176b0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:13.223740791Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1388,"client":"2","":"downstairs","session_id":"be2229f6-1e48-4330-bc8c-727b340176b0"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:13.223773715Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1388,"client":"2","":"downstairs","session_id":"be2229f6-1e48-4330-bc8c-727b340176b0"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:13.22380613Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1388,"":"io task","client":"2","":"downstairs","session_id":"be2229f6-1e48-4330-bc8c-727b340176b0"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:13.223838674Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1388,"":"io task","client":"0","":"downstairs","session_id":"be2229f6-1e48-4330-bc8c-727b340176b0"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:13.224038379Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1388,"":"io task","client":"1","":"downstairs","session_id":"be2229f6-1e48-4330-bc8c-727b340176b0"} {"msg":"ds_connection connected from Ok(127.0.0.1:53048)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:13.224085906Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1388,"":"io task","client":"1","":"downstairs","session_id":"be2229f6-1e48-4330-bc8c-727b340176b0"} {"msg":"ds_connection connected from Ok(127.0.0.1:40807)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:13.224125431Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1388,"":"io task","client":"2","":"downstairs","session_id":"be2229f6-1e48-4330-bc8c-727b340176b0"} {"msg":"ds_connection connected from Ok(127.0.0.1:50793)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:13.224161335Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1388,"":"io task","client":"0","":"downstairs","session_id":"be2229f6-1e48-4330-bc8c-727b340176b0"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:13.224600022Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1388,"client":"2","":"downstairs","session_id":"be2229f6-1e48-4330-bc8c-727b340176b0"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:13.224648068Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1388,"client":"2","":"downstairs","session_id":"be2229f6-1e48-4330-bc8c-727b340176b0"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:13.224677012Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1388,"client":"1","":"downstairs","session_id":"be2229f6-1e48-4330-bc8c-727b340176b0"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:13.224701055Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1388,"client":"1","":"downstairs","session_id":"be2229f6-1e48-4330-bc8c-727b340176b0"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:13.224726598Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1388,"client":"0","":"downstairs","session_id":"be2229f6-1e48-4330-bc8c-727b340176b0"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:13.224756252Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1388,"client":"0","":"downstairs","session_id":"be2229f6-1e48-4330-bc8c-727b340176b0"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:13.225275379Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1388,"client":"1","":"downstairs","session_id":"be2229f6-1e48-4330-bc8c-727b340176b0"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:13.225406856Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1388,"client":"0","":"downstairs","session_id":"be2229f6-1e48-4330-bc8c-727b340176b0"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:13.225478265Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1388,"client":"2","":"downstairs","session_id":"be2229f6-1e48-4330-bc8c-727b340176b0"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:13.225779644Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1388,"client":"1","":"downstairs","session_id":"be2229f6-1e48-4330-bc8c-727b340176b0"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:13.225815398Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1388,"client":"1","":"downstairs","session_id":"be2229f6-1e48-4330-bc8c-727b340176b0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:13.225842852Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1388,"session_id":"be2229f6-1e48-4330-bc8c-727b340176b0"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:13.225898279Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1388,"client":"0","":"downstairs","session_id":"be2229f6-1e48-4330-bc8c-727b340176b0"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:13.225928183Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1388,"client":"0","":"downstairs","session_id":"be2229f6-1e48-4330-bc8c-727b340176b0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:13.225944025Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1388,"session_id":"be2229f6-1e48-4330-bc8c-727b340176b0"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:13.225965058Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1388,"client":"2","":"downstairs","session_id":"be2229f6-1e48-4330-bc8c-727b340176b0"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:13.225979439Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1388,"client":"2","":"downstairs","session_id":"be2229f6-1e48-4330-bc8c-727b340176b0"} {"msg":"[0]R flush_numbers[0..12]: [69, 69, 79, 78, 79, 69, 80, 71, 80, 76, 80, 81]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:13.225999472Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1388,"":"downstairs","":"downstairs","session_id":"be2229f6-1e48-4330-bc8c-727b340176b0"} {"msg":"[0]R generation[0..12]: [42, 42, 49, 48, 49, 42, 50, 44, 50, 46, 50, 51]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:13.226024705Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1388,"":"downstairs","":"downstairs","session_id":"be2229f6-1e48-4330-bc8c-727b340176b0"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:13.226040437Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1388,"":"downstairs","":"downstairs","session_id":"be2229f6-1e48-4330-bc8c-727b340176b0"} {"msg":"[1]R flush_numbers[0..12]: [69, 69, 79, 78, 79, 69, 80, 71, 80, 76, 80, 81]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:13.226054569Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1388,"":"downstairs","":"downstairs","session_id":"be2229f6-1e48-4330-bc8c-727b340176b0"} {"msg":"[1]R generation[0..12]: [42, 42, 49, 48, 49, 42, 50, 44, 50, 46, 50, 51]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:13.226068511Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1388,"":"downstairs","":"downstairs","session_id":"be2229f6-1e48-4330-bc8c-727b340176b0"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:13.226082643Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1388,"":"downstairs","":"downstairs","session_id":"be2229f6-1e48-4330-bc8c-727b340176b0"} {"msg":"[2]R flush_numbers[0..12]: [69, 69, 79, 78, 79, 69, 80, 71, 80, 76, 80, 81]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:13.226096385Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1388,"":"downstairs","":"downstairs","session_id":"be2229f6-1e48-4330-bc8c-727b340176b0"} {"msg":"[2]R generation[0..12]: [42, 42, 49, 48, 49, 42, 50, 44, 50, 46, 50, 51]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:13.226110046Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1388,"":"downstairs","":"downstairs","session_id":"be2229f6-1e48-4330-bc8c-727b340176b0"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:13.226124158Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1388,"":"downstairs","":"downstairs","session_id":"be2229f6-1e48-4330-bc8c-727b340176b0"} {"msg":"Max found gen is 52","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:13.22613721Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1388,"":"downstairs","":"downstairs","session_id":"be2229f6-1e48-4330-bc8c-727b340176b0"} {"msg":"Generation requested: 52 >= found:52","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:13.226150832Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1388,"":"downstairs","":"downstairs","session_id":"be2229f6-1e48-4330-bc8c-727b340176b0"} {"msg":"Next flush: 82","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:13.226164073Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1388,"":"downstairs","":"downstairs","session_id":"be2229f6-1e48-4330-bc8c-727b340176b0"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:13.226177125Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1388,"":"downstairs","":"downstairs","session_id":"be2229f6-1e48-4330-bc8c-727b340176b0"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:13.226190187Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1388,"session_id":"be2229f6-1e48-4330-bc8c-727b340176b0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-01T16:52:13.226204388Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1388,"client":"0","":"downstairs","session_id":"be2229f6-1e48-4330-bc8c-727b340176b0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:13.226229532Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1388,"client":"1","":"downstairs","session_id":"be2229f6-1e48-4330-bc8c-727b340176b0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:13.226243764Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1388,"client":"2","":"downstairs","session_id":"be2229f6-1e48-4330-bc8c-727b340176b0"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:13.226256955Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1388,"session_id":"be2229f6-1e48-4330-bc8c-727b340176b0"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:13.226269267Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1388,"session_id":"be2229f6-1e48-4330-bc8c-727b340176b0"} {"msg":"2089f9f4-0cb7-4566-bbb7-79cbea57bebb is now active with session: be2229f6-1e48-4330-bc8c-727b340176b0","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:13.226282138Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1388,"session_id":"be2229f6-1e48-4330-bc8c-727b340176b0"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:13.22629546Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1388,"session_id":"be2229f6-1e48-4330-bc8c-727b340176b0"} {"msg":"The guest has finished waiting for activation with:52","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:13.226308152Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1388} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 536 len 12288 data: 5 5 4 02/30 Read block 77 len 12288 03/30 Read block 25 len 12288 04/30 Write block 495 len 32768 data: 3 4 4 4 5 5 4 4 05/30 Write block 0 len 4096 data: 3 06/30 Write block 309 len 24576 data: 5 4 5 5 5 5 07/30 Read block 128 len 16384 08/30 Read block 7 len 8192 09/30 Write block 357 len 16384 data: 5 4 4 3 10/30 Read block 471 len 40960 11/30 Write block 37 len 40960 data: 5 7 6 5 6 8 10 9 7 8 12/30 Write block 425 len 16384 data: 7 7 6 7 13/30 Write block 262 len 4096 data: 7 14/30 Write block 498 len 16384 data: 5 6 6 5 15/30 Flush 16/30 Write block 197 len 20480 data: 7 6 7 5 7 17/30 Write block 155 len 28672 data: 7 7 5 5 3 5 5 18/30 Read block 582 len 24576 19/30 Write block 447 len 32768 data: 8 7 7 8 8 6 7 6 20/30 Read block 73 len 36864 21/30 Read block 279 len 8192 22/30 Write block 43 len 20480 data: 11 10 8 9 5 23/30 Write block 223 len 40960 data: 7 7 6 5 5 3 4 4 4 4 24/30 Read block 367 len 4096 25/30 Write block 31 len 24576 data: 4 4 2 3 4 4 26/30 Read block 32 len 24576 27/30 Write block 578 len 36864 data: 5 6 6 5 5 5 5 7 7 28/30 Read block 42 len 32768 29/30 Read block 89 len 28672 30/30 Write block 81 len 8192 data: 5 8 ---------------------------------------------------------------- Crucible gen:52 GIO:true work queues: Upstairs:0 downstairs:15 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1015 Acked Write 5 Done Done Done false 1016 Acked Write 7 Done Done Done false 1017 Acked Read 6 Done Done Done false 1018 Acked Write 8 Done Done Done false 1019 Acked Read 9 Done Done Done false 1020 Acked Read 2 Done Done Done false 1021 Acked Write 5 Done Done Done false 1022 Acked Write 10 Done Done Done false 1023 Acked Read 1 Done Done Done false 1024 Acked Write 6 Done Done Done false 1025 Acked Read 6 Done Done Done false 1026 Acked Write 9 Done Done Done false 1027 Acked Read 8 Done Done Done false 1028 Acked Read 7 Done Done Done false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 14 14 14 42 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1014 1014 1014 Downstairs last five completed: 1014 1013 1012 1011 1010 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:52:13.249180319Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1388} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 52 52 52 82 82 82 F F F 1 020-039 52 52 52 82 82 82 T T T 2 040-059 52 52 52 82 82 82 T T T 3 060-079 48 48 48 78 78 78 F F F 4 080-099 49 49 49 79 79 79 T T T 5 100-119 42 42 42 69 69 69 F F F 6 120-139 50 50 50 80 80 80 F F F 7 140-159 44 44 44 71 71 71 T T T 8 160-179 50 50 50 80 80 80 T T T 9 180-199 46 46 46 76 76 76 T T T 10 200-219 50 50 50 80 80 80 T T T 11 220-239 51 51 51 81 81 81 T T T 12 240-259 49 49 49 79 79 79 F F F 13 260-279 52 52 52 82 82 82 F F F 14 280-299 50 50 50 80 80 80 F F F 15 300-319 52 52 52 82 82 82 F F F 16 320-339 42 42 42 69 69 69 F F F 17 340-359 52 52 52 82 82 82 F F F 18 360-379 52 52 52 82 82 82 F F F 19 380-399 49 49 49 79 79 79 F F F 20 400-419 27 27 27 41 41 41 F F F 21 420-439 52 52 52 82 82 82 F F F 22 440-459 50 50 50 80 80 80 T T T 23 460-479 50 50 50 80 80 80 F F F 24 480-499 52 52 52 82 82 82 F F F 25 500-519 52 52 52 82 82 82 F F F 26 520-539 52 52 52 82 82 82 F F F 27 540-559 45 45 45 73 73 73 F F F 28 560-579 48 48 48 78 78 78 T T T 29 580-599 49 49 49 79 79 79 T T T Max gen: 52, Max flush: 82 A Difference in extent metadata is expected here On loop 26 Verifying data now No change after verify Loop: 26 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 52 52 52 82 82 82 F F F 1 020-039 53 53 53 83 83 83 F F F 2 040-059 53 53 53 83 83 83 F F F 3 060-079 48 48 48 78 78 78 F F F 4 080-099 53 53 53 83 83 83 F F F 5 100-119 42 42 42 69 69 69 F F F 6 120-139 50 50 50 80 80 80 F F F 7 140-159 53 53 53 83 83 83 F F F 8 160-179 53 53 53 83 83 83 F F F 9 180-199 53 53 53 83 83 83 F F F 10 200-219 53 53 53 83 83 83 F F F 11 220-239 53 53 53 83 83 83 F F F 12 240-259 49 49 49 79 79 79 F F F 13 260-279 52 52 52 82 82 82 F F F 14 280-299 50 50 50 80 80 80 F F F 15 300-319 52 52 52 82 82 82 F F F 16 320-339 42 42 42 69 69 69 F F F 17 340-359 52 52 52 82 82 82 F F F 18 360-379 52 52 52 82 82 82 F F F 19 380-399 49 49 49 79 79 79 F F F 20 400-419 27 27 27 41 41 41 F F F 21 420-439 52 52 52 82 82 82 F F F 22 440-459 53 53 53 83 83 83 F F F 23 460-479 50 50 50 80 80 80 F F F 24 480-499 52 52 52 82 82 82 F F F 25 500-519 52 52 52 82 82 82 F F F 26 520-539 52 52 52 82 82 82 F F F 27 540-559 45 45 45 73 73 73 F F F 28 560-579 53 53 53 83 83 83 F F F 29 580-599 53 53 53 83 83 83 F F F Max gen: 53, Max flush: 83 Begin loop 27 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:17.649970508Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1396,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:52:17.692610282Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1396,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:17.692973449Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1396,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:17.724604774Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1396,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:17.725123421Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1396} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"3818f0fc76fe787c6cc412af30fd59b930203885\",\n git_commit_timestamp: \"2024-11-01T16:08:30.000000000Z\",\n git_branch: \"renovate/hyper-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:17.725168177Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1396} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:17.725186789Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1396} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:17.725204911Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1396} {"msg":"Crucible 2e62be61-08ca-4e10-81bc-19f6cfcbfab7 has session id: ea262d31-0277-4013-acef-536bdc1e8531","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:17.725644778Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1396,"session_id":"ea262d31-0277-4013-acef-536bdc1e8531"} {"msg":"Upstairs opts: Upstairs UUID: 2e62be61-08ca-4e10-81bc-19f6cfcbfab7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:17.725679472Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1396,"session_id":"ea262d31-0277-4013-acef-536bdc1e8531"} {"msg":"Crucible stats registered with UUID: 2e62be61-08ca-4e10-81bc-19f6cfcbfab7","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:17.725706236Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1396,"session_id":"ea262d31-0277-4013-acef-536bdc1e8531"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:17.756572003Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1396,"task":"crutest"} {"msg":"The guest has requested activation with gen:54","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:17.756614838Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1396} {"msg":"2e62be61-08ca-4e10-81bc-19f6cfcbfab7 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:17.756683407Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1396,"session_id":"ea262d31-0277-4013-acef-536bdc1e8531"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:17.756725563Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1396,"client":"0","":"downstairs","session_id":"ea262d31-0277-4013-acef-536bdc1e8531"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:17.756774439Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1396,"client":"0","":"downstairs","session_id":"ea262d31-0277-4013-acef-536bdc1e8531"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:17.756811304Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1396,"client":"1","":"downstairs","session_id":"ea262d31-0277-4013-acef-536bdc1e8531"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:17.756846708Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1396,"client":"1","":"downstairs","session_id":"ea262d31-0277-4013-acef-536bdc1e8531"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:17.756882043Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1396,"client":"2","":"downstairs","session_id":"ea262d31-0277-4013-acef-536bdc1e8531"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:17.756916327Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1396,"client":"2","":"downstairs","session_id":"ea262d31-0277-4013-acef-536bdc1e8531"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:17.756951972Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1396,"":"io task","client":"2","":"downstairs","session_id":"ea262d31-0277-4013-acef-536bdc1e8531"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:17.756986326Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1396,"":"io task","client":"0","":"downstairs","session_id":"ea262d31-0277-4013-acef-536bdc1e8531"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:17.757058416Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1396,"":"io task","client":"1","":"downstairs","session_id":"ea262d31-0277-4013-acef-536bdc1e8531"} {"msg":"ds_connection connected from Ok(127.0.0.1:38408)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:17.757147077Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1396,"":"io task","client":"2","":"downstairs","session_id":"ea262d31-0277-4013-acef-536bdc1e8531"} {"msg":"ds_connection connected from Ok(127.0.0.1:36163)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:17.757184302Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1396,"":"io task","client":"1","":"downstairs","session_id":"ea262d31-0277-4013-acef-536bdc1e8531"} {"msg":"ds_connection connected from Ok(127.0.0.1:44469)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:17.757204625Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1396,"":"io task","client":"0","":"downstairs","session_id":"ea262d31-0277-4013-acef-536bdc1e8531"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:17.757837316Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1396,"client":"1","":"downstairs","session_id":"ea262d31-0277-4013-acef-536bdc1e8531"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:17.757877841Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1396,"client":"1","":"downstairs","session_id":"ea262d31-0277-4013-acef-536bdc1e8531"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:17.757912166Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1396,"client":"2","":"downstairs","session_id":"ea262d31-0277-4013-acef-536bdc1e8531"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:17.75794324Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1396,"client":"2","":"downstairs","session_id":"ea262d31-0277-4013-acef-536bdc1e8531"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:17.757973604Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1396,"client":"0","":"downstairs","session_id":"ea262d31-0277-4013-acef-536bdc1e8531"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:17.758005638Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1396,"client":"0","":"downstairs","session_id":"ea262d31-0277-4013-acef-536bdc1e8531"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:17.758459716Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1396,"client":"1","":"downstairs","session_id":"ea262d31-0277-4013-acef-536bdc1e8531"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:17.758610046Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1396,"client":"0","":"downstairs","session_id":"ea262d31-0277-4013-acef-536bdc1e8531"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:17.758650011Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1396,"client":"2","":"downstairs","session_id":"ea262d31-0277-4013-acef-536bdc1e8531"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:17.758966441Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1396,"client":"1","":"downstairs","session_id":"ea262d31-0277-4013-acef-536bdc1e8531"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:17.758999346Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1396,"client":"1","":"downstairs","session_id":"ea262d31-0277-4013-acef-536bdc1e8531"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:17.759026359Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1396,"session_id":"ea262d31-0277-4013-acef-536bdc1e8531"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:17.759167598Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1396,"client":"2","":"downstairs","session_id":"ea262d31-0277-4013-acef-536bdc1e8531"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:17.759197001Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1396,"client":"2","":"downstairs","session_id":"ea262d31-0277-4013-acef-536bdc1e8531"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:17.759212113Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1396,"session_id":"ea262d31-0277-4013-acef-536bdc1e8531"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:17.759236866Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1396,"client":"0","":"downstairs","session_id":"ea262d31-0277-4013-acef-536bdc1e8531"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:17.759251558Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1396,"client":"0","":"downstairs","session_id":"ea262d31-0277-4013-acef-536bdc1e8531"} {"msg":"[0]R flush_numbers[0..12]: [82, 83, 83, 78, 83, 69, 80, 83, 83, 83, 83, 83]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:17.759269411Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1396,"":"downstairs","":"downstairs","session_id":"ea262d31-0277-4013-acef-536bdc1e8531"} {"msg":"[0]R generation[0..12]: [52, 53, 53, 48, 53, 42, 50, 53, 53, 53, 53, 53]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:17.759283682Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1396,"":"downstairs","":"downstairs","session_id":"ea262d31-0277-4013-acef-536bdc1e8531"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:17.759297814Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1396,"":"downstairs","":"downstairs","session_id":"ea262d31-0277-4013-acef-536bdc1e8531"} {"msg":"[1]R flush_numbers[0..12]: [82, 83, 83, 78, 83, 69, 80, 83, 83, 83, 83, 83]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:17.759311686Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1396,"":"downstairs","":"downstairs","session_id":"ea262d31-0277-4013-acef-536bdc1e8531"} {"msg":"[1]R generation[0..12]: [52, 53, 53, 48, 53, 42, 50, 53, 53, 53, 53, 53]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:17.759325358Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1396,"":"downstairs","":"downstairs","session_id":"ea262d31-0277-4013-acef-536bdc1e8531"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:17.75933928Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1396,"":"downstairs","":"downstairs","session_id":"ea262d31-0277-4013-acef-536bdc1e8531"} {"msg":"[2]R flush_numbers[0..12]: [82, 83, 83, 78, 83, 69, 80, 83, 83, 83, 83, 83]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:17.759353201Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1396,"":"downstairs","":"downstairs","session_id":"ea262d31-0277-4013-acef-536bdc1e8531"} {"msg":"[2]R generation[0..12]: [52, 53, 53, 48, 53, 42, 50, 53, 53, 53, 53, 53]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:17.759367073Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1396,"":"downstairs","":"downstairs","session_id":"ea262d31-0277-4013-acef-536bdc1e8531"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:17.759381095Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1396,"":"downstairs","":"downstairs","session_id":"ea262d31-0277-4013-acef-536bdc1e8531"} {"msg":"Max found gen is 54","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:17.759394227Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1396,"":"downstairs","":"downstairs","session_id":"ea262d31-0277-4013-acef-536bdc1e8531"} {"msg":"Generation requested: 54 >= found:54","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:17.759407368Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1396,"":"downstairs","":"downstairs","session_id":"ea262d31-0277-4013-acef-536bdc1e8531"} {"msg":"Next flush: 84","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:17.75942033Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1396,"":"downstairs","":"downstairs","session_id":"ea262d31-0277-4013-acef-536bdc1e8531"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:17.759433512Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1396,"":"downstairs","":"downstairs","session_id":"ea262d31-0277-4013-acef-536bdc1e8531"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:17.759458125Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1396,"session_id":"ea262d31-0277-4013-acef-536bdc1e8531"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:17.759473167Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1396,"client":"0","":"downstairs","session_id":"ea262d31-0277-4013-acef-536bdc1e8531"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-01T16:52:17.759486799Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1396,"client":"1","":"downstairs","session_id":"ea262d31-0277-4013-acef-536bdc1e8531"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:17.759509131Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1396,"client":"2","":"downstairs","session_id":"ea262d31-0277-4013-acef-536bdc1e8531"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:17.759522473Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1396,"session_id":"ea262d31-0277-4013-acef-536bdc1e8531"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:17.759534815Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1396,"session_id":"ea262d31-0277-4013-acef-536bdc1e8531"} {"msg":"2e62be61-08ca-4e10-81bc-19f6cfcbfab7 is now active with session: ea262d31-0277-4013-acef-536bdc1e8531","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:17.759547647Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1396,"session_id":"ea262d31-0277-4013-acef-536bdc1e8531"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:17.759559708Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1396,"session_id":"ea262d31-0277-4013-acef-536bdc1e8531"} {"msg":"The guest has finished waiting for activation with:54","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:17.759571959Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1396} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 69 len 16384 data: 9 12 10 10 02/30 Write block 266 len 36864 data: 6 5 5 5 4 6 6 8 10 03/30 Write block 59 len 4096 data: 3 04/30 Write block 539 len 8192 data: 3 3 05/30 Read block 236 len 40960 06/30 Read block 313 len 16384 07/30 Flush 08/30 Write block 81 len 4096 data: 6 09/30 Flush 10/30 Write block 56 len 40960 data: 4 4 4 4 3 4 4 4 6 6 11/30 Read block 85 len 4096 12/30 Read block 449 len 12288 13/30 Write block 145 len 32768 data: 4 3 3 3 5 5 4 5 14/30 Read block 27 len 4096 15/30 Read block 466 len 40960 16/30 Read block 76 len 16384 17/30 Write block 270 len 28672 data: 5 7 7 9 11 10 9 18/30 Read block 460 len 16384 19/30 Write block 110 len 16384 data: 5 4 4 3 20/30 Write block 7 len 36864 data: 4 4 3 2 4 3 3 4 4 21/30 Write block 118 len 40960 data: 3 3 5 6 7 5 6 6 6 6 22/30 Write block 587 len 16384 data: 7 5 6 5 23/30 Write block 385 len 8192 data: 7 7 24/30 Write block 385 len 4096 data: 8 25/30 Read block 570 len 20480 26/30 Write block 461 len 4096 data: 3 27/30 Read block 42 len 32768 28/30 Write block 48 len 12288 data: 4 7 5 29/30 Read block 477 len 8192 30/30 Read block 113 len 36864 ---------------------------------------------------------------- Crucible gen:54 GIO:true work queues: Upstairs:0 downstairs:21 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1009 Acked Write 10 Done Done Done false 1010 Acked Read 1 Done Done Done false 1011 Acked Read 3 Done Done Done false 1012 Acked Write 8 Done Done Done false 1013 Acked Read 1 Done Done Done false 1014 Acked Read 10 Done Done Done false 1015 Acked Read 4 Done Done Done false 1016 Acked Write 7 Done Done Done false 1017 Acked Read 4 Done Done Done false 1018 Acked Write 4 Done Done Done false 1019 Acked Write 9 Done Done Done false 1020 Acked Write 10 Done Done Done false 1021 Acked Write 4 Done Done Done false 1022 Acked Write 2 Done Done Done false 1023 Acked Write 1 Done Done Done false 1024 Acked Read 5 Done Done Done false 1025 Acked Write 1 Done Done Done false 1026 Acked Read 8 Done Done Done false 1027 Acked Write 3 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Read 9 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 0 2 Done 20 20 21 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1008 1008 1008 Downstairs last five completed: 1008 1007 1006 1005 1004 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:52:17.779546283Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1396} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 52 52 52 82 82 82 T T T 1 020-039 53 53 53 83 83 83 F F F 2 040-059 54 54 54 84 84 84 T T T 3 060-079 54 54 54 84 84 84 T T T 4 080-099 54 54 54 85 85 85 F F F 5 100-119 42 42 42 69 69 69 T T T 6 120-139 50 50 50 80 80 80 T T T 7 140-159 53 53 53 83 83 83 T T T 8 160-179 53 53 53 83 83 83 F F F 9 180-199 53 53 53 83 83 83 F F F 10 200-219 53 53 53 83 83 83 F F F 11 220-239 53 53 53 83 83 83 F F F 12 240-259 49 49 49 79 79 79 F F F 13 260-279 54 54 54 84 84 84 T T T 14 280-299 50 50 50 80 80 80 F F F 15 300-319 52 52 52 82 82 82 F F F 16 320-339 42 42 42 69 69 69 F F F 17 340-359 52 52 52 82 82 82 F F F 18 360-379 52 52 52 82 82 82 F F F 19 380-399 49 49 49 79 79 79 T T T 20 400-419 27 27 27 41 41 41 F F F 21 420-439 52 52 52 82 82 82 F F F 22 440-459 53 53 53 83 83 83 F F F 23 460-479 50 50 50 80 80 80 T T T 24 480-499 52 52 52 82 82 82 F F F 25 500-519 52 52 52 82 82 82 F F F 26 520-539 54 54 54 84 84 84 F F F 27 540-559 54 54 54 84 84 84 F F F 28 560-579 53 53 53 83 83 83 F F F 29 580-599 53 53 53 83 83 83 T T T Max gen: 54, Max flush: 85 A Difference in extent metadata is expected here On loop 27 Verifying data now No change after verify Loop: 27 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 55 55 55 86 86 86 F F F 1 020-039 53 53 53 83 83 83 F F F 2 040-059 55 55 55 86 86 86 F F F 3 060-079 55 55 55 86 86 86 F F F 4 080-099 54 54 54 85 85 85 F F F 5 100-119 55 55 55 86 86 86 F F F 6 120-139 55 55 55 86 86 86 F F F 7 140-159 55 55 55 86 86 86 F F F 8 160-179 53 53 53 83 83 83 F F F 9 180-199 53 53 53 83 83 83 F F F 10 200-219 53 53 53 83 83 83 F F F 11 220-239 53 53 53 83 83 83 F F F 12 240-259 49 49 49 79 79 79 F F F 13 260-279 55 55 55 86 86 86 F F F 14 280-299 50 50 50 80 80 80 F F F 15 300-319 52 52 52 82 82 82 F F F 16 320-339 42 42 42 69 69 69 F F F 17 340-359 52 52 52 82 82 82 F F F 18 360-379 52 52 52 82 82 82 F F F 19 380-399 55 55 55 86 86 86 F F F 20 400-419 27 27 27 41 41 41 F F F 21 420-439 52 52 52 82 82 82 F F F 22 440-459 53 53 53 83 83 83 F F F 23 460-479 55 55 55 86 86 86 F F F 24 480-499 52 52 52 82 82 82 F F F 25 500-519 52 52 52 82 82 82 F F F 26 520-539 54 54 54 84 84 84 F F F 27 540-559 54 54 54 84 84 84 F F F 28 560-579 53 53 53 83 83 83 F F F 29 580-599 55 55 55 86 86 86 F F F Max gen: 55, Max flush: 86 Begin loop 28 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:22.181496543Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1404,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:52:22.224528728Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1404,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:22.224904326Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1404,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:22.257177604Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1404,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:22.257711703Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1404} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"3818f0fc76fe787c6cc412af30fd59b930203885\",\n git_commit_timestamp: \"2024-11-01T16:08:30.000000000Z\",\n git_branch: \"renovate/hyper-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:22.257745908Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1404} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:22.257759959Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1404} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:22.257774161Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1404} {"msg":"Crucible 4e14e69b-7160-4914-af81-f7aec0c7c160 has session id: 22559ebf-2858-4301-8d40-9c8053613f7a","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:22.258255383Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1404,"session_id":"22559ebf-2858-4301-8d40-9c8053613f7a"} {"msg":"Upstairs opts: Upstairs UUID: 4e14e69b-7160-4914-af81-f7aec0c7c160, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:22.258297379Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1404,"session_id":"22559ebf-2858-4301-8d40-9c8053613f7a"} {"msg":"Crucible stats registered with UUID: 4e14e69b-7160-4914-af81-f7aec0c7c160","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:22.258323802Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1404,"session_id":"22559ebf-2858-4301-8d40-9c8053613f7a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:22.289692084Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1404,"task":"crutest"} {"msg":"The guest has requested activation with gen:56","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:22.289723638Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1404} {"msg":"4e14e69b-7160-4914-af81-f7aec0c7c160 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:22.289757042Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1404,"session_id":"22559ebf-2858-4301-8d40-9c8053613f7a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:22.289774014Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1404,"client":"0","":"downstairs","session_id":"22559ebf-2858-4301-8d40-9c8053613f7a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:22.289803768Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1404,"client":"0","":"downstairs","session_id":"22559ebf-2858-4301-8d40-9c8053613f7a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:22.2898217Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1404,"client":"1","":"downstairs","session_id":"22559ebf-2858-4301-8d40-9c8053613f7a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:22.289839603Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1404,"client":"1","":"downstairs","session_id":"22559ebf-2858-4301-8d40-9c8053613f7a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:22.289856325Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1404,"client":"2","":"downstairs","session_id":"22559ebf-2858-4301-8d40-9c8053613f7a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:22.289881078Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1404,"client":"2","":"downstairs","session_id":"22559ebf-2858-4301-8d40-9c8053613f7a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:22.289912682Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1404,"":"io task","client":"2","":"downstairs","session_id":"22559ebf-2858-4301-8d40-9c8053613f7a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:22.289939196Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1404,"":"io task","client":"0","":"downstairs","session_id":"22559ebf-2858-4301-8d40-9c8053613f7a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:22.289965989Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1404,"":"io task","client":"1","":"downstairs","session_id":"22559ebf-2858-4301-8d40-9c8053613f7a"} {"msg":"ds_connection connected from Ok(127.0.0.1:53025)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:22.289994873Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1404,"":"io task","client":"2","":"downstairs","session_id":"22559ebf-2858-4301-8d40-9c8053613f7a"} {"msg":"ds_connection connected from Ok(127.0.0.1:42998)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:22.290023106Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1404,"":"io task","client":"1","":"downstairs","session_id":"22559ebf-2858-4301-8d40-9c8053613f7a"} {"msg":"ds_connection connected from Ok(127.0.0.1:52731)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:22.290203899Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1404,"":"io task","client":"0","":"downstairs","session_id":"22559ebf-2858-4301-8d40-9c8053613f7a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:22.290762402Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1404,"client":"1","":"downstairs","session_id":"22559ebf-2858-4301-8d40-9c8053613f7a"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:22.290798846Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1404,"client":"1","":"downstairs","session_id":"22559ebf-2858-4301-8d40-9c8053613f7a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:22.290962227Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1404,"client":"0","":"downstairs","session_id":"22559ebf-2858-4301-8d40-9c8053613f7a"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:22.290988711Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1404,"client":"0","":"downstairs","session_id":"22559ebf-2858-4301-8d40-9c8053613f7a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:22.291019115Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1404,"client":"2","":"downstairs","session_id":"22559ebf-2858-4301-8d40-9c8053613f7a"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:22.291047378Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1404,"client":"2","":"downstairs","session_id":"22559ebf-2858-4301-8d40-9c8053613f7a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:22.291433738Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1404,"client":"1","":"downstairs","session_id":"22559ebf-2858-4301-8d40-9c8053613f7a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:22.291475434Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1404,"client":"2","":"downstairs","session_id":"22559ebf-2858-4301-8d40-9c8053613f7a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:22.291745208Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1404,"client":"0","":"downstairs","session_id":"22559ebf-2858-4301-8d40-9c8053613f7a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:22.292096223Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1404,"client":"1","":"downstairs","session_id":"22559ebf-2858-4301-8d40-9c8053613f7a"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:22.292121006Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1404,"client":"1","":"downstairs","session_id":"22559ebf-2858-4301-8d40-9c8053613f7a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:22.292135508Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1404,"session_id":"22559ebf-2858-4301-8d40-9c8053613f7a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:22.29214781Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1404,"client":"2","":"downstairs","session_id":"22559ebf-2858-4301-8d40-9c8053613f7a"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:22.292161562Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1404,"client":"2","":"downstairs","session_id":"22559ebf-2858-4301-8d40-9c8053613f7a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:22.292175013Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1404,"session_id":"22559ebf-2858-4301-8d40-9c8053613f7a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:22.29230621Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1404,"client":"0","":"downstairs","session_id":"22559ebf-2858-4301-8d40-9c8053613f7a"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:22.292328543Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1404,"client":"0","":"downstairs","session_id":"22559ebf-2858-4301-8d40-9c8053613f7a"} {"msg":"[0]R flush_numbers[0..12]: [86, 83, 86, 86, 85, 86, 86, 86, 83, 83, 83, 83]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:22.292342815Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1404,"":"downstairs","":"downstairs","session_id":"22559ebf-2858-4301-8d40-9c8053613f7a"} {"msg":"[0]R generation[0..12]: [55, 53, 55, 55, 54, 55, 55, 55, 53, 53, 53, 53]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:22.292356687Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1404,"":"downstairs","":"downstairs","session_id":"22559ebf-2858-4301-8d40-9c8053613f7a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:22.292370889Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1404,"":"downstairs","":"downstairs","session_id":"22559ebf-2858-4301-8d40-9c8053613f7a"} {"msg":"[1]R flush_numbers[0..12]: [86, 83, 86, 86, 85, 86, 86, 86, 83, 83, 83, 83]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:22.29238442Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1404,"":"downstairs","":"downstairs","session_id":"22559ebf-2858-4301-8d40-9c8053613f7a"} {"msg":"[1]R generation[0..12]: [55, 53, 55, 55, 54, 55, 55, 55, 53, 53, 53, 53]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:22.292397812Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1404,"":"downstairs","":"downstairs","session_id":"22559ebf-2858-4301-8d40-9c8053613f7a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:22.292411814Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1404,"":"downstairs","":"downstairs","session_id":"22559ebf-2858-4301-8d40-9c8053613f7a"} {"msg":"[2]R flush_numbers[0..12]: [86, 83, 86, 86, 85, 86, 86, 86, 83, 83, 83, 83]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:22.292425296Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1404,"":"downstairs","":"downstairs","session_id":"22559ebf-2858-4301-8d40-9c8053613f7a"} {"msg":"[2]R generation[0..12]: [55, 53, 55, 55, 54, 55, 55, 55, 53, 53, 53, 53]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:22.292438758Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1404,"":"downstairs","":"downstairs","session_id":"22559ebf-2858-4301-8d40-9c8053613f7a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:22.292452759Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1404,"":"downstairs","":"downstairs","session_id":"22559ebf-2858-4301-8d40-9c8053613f7a"} {"msg":"Max found gen is 56","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:22.292465721Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1404,"":"downstairs","":"downstairs","session_id":"22559ebf-2858-4301-8d40-9c8053613f7a"} {"msg":"Generation requested: 56 >= found:56","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:22.292478903Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1404,"":"downstairs","":"downstairs","session_id":"22559ebf-2858-4301-8d40-9c8053613f7a"} {"msg":"Next flush: 87","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:22.292491784Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1404,"":"downstairs","":"downstairs","session_id":"22559ebf-2858-4301-8d40-9c8053613f7a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:22.292504746Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1404,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO 22559ebf-2858-4301-8d40-9c8053613f7a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:22.292527799Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1404,"session_id":"22559ebf-2858-4301-8d40-9c8053613f7a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:22.292540031Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1404,"client":"0","":"downstairs","session_id":"22559ebf-2858-4301-8d40-9c8053613f7a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:22.292553302Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1404,"client":"1","":"downstairs","session_id":"22559ebf-2858-4301-8d40-9c8053613f7a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:22.292566394Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1404,"client":"2","":"downstairs","session_id":"22559ebf-2858-4301-8d40-9c8053613f7a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:22.292579606Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1404,"session_id":"22559ebf-2858-4301-8d40-9c8053613f7a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:22.292591757Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1404,"session_id":"22559ebf-2858-4301-8d40-9c8053613f7a"} {"msg":"4e14e69b-7160-4914-af81-f7aec0c7c160 is now active with session: 22559ebf-2858-4301-8d40-9c8053613f7a","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:22.292604659Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1404,"session_id":"22559ebf-2858-4301-8d40-9c8053613f7a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:22.292616741Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1404,"session_id":"22559ebf-2858-4301-8d40-9c8053613f7a"} {"msg":"The guest has finished waiting for activation with:56","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:22.292629002Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1404} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 506 len 36864 data: 4 4 6 5 6 6 6 3 2 02/30 Read block 544 len 8192 03/30 Write block 335 len 24576 data: 6 5 6 5 3 4 04/30 Read block 115 len 24576 05/30 Write block 194 len 24576 data: 7 7 7 8 7 8 06/30 Write block 183 len 24576 data: 4 4 4 5 5 6 07/30 Write block 335 len 4096 data: 7 08/30 Read block 24 len 8192 09/30 Write block 203 len 36864 data: 7 8 9 10 10 8 7 8 9 10/30 Read block 221 len 8192 11/30 Write block 313 len 32768 data: 6 6 7 8 6 7 7 7 12/30 Read block 466 len 24576 13/30 Write block 223 len 16384 data: 8 8 7 6 14/30 Read block 32 len 24576 15/30 Write block 529 len 32768 data: 5 7 6 6 6 6 5 6 16/30 Read block 539 len 40960 17/30 Read block 420 len 24576 18/30 Read block 21 len 32768 19/30 Write block 148 len 32768 data: 4 6 6 5 6 5 7 8 20/30 Write block 111 len 36864 data: 5 5 4 4 3 3 4 4 4 21/30 Write block 477 len 20480 data: 7 9 8 8 6 22/30 Write block 564 len 40960 data: 5 3 3 4 4 5 4 4 4 3 23/30 Read block 400 len 20480 24/30 Write block 468 len 28672 data: 4 5 6 8 8 7 7 25/30 Read block 505 len 36864 26/30 Write block 147 len 40960 data: 4 5 7 7 6 7 6 8 9 8 27/30 Read block 286 len 16384 28/30 Read block 54 len 36864 29/30 Read block 157 len 20480 30/30 Read block 589 len 20480 ---------------------------------------------------------------- Crucible gen:56 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 9 Done Done Done false 1001 Acked Read 2 Done Done Done false 1002 Acked Write 6 Done Done Done false 1003 Acked Read 6 Done Done Done false 1004 Acked Write 6 Done Done Done false 1005 Acked Write 6 Done Done Done false 1006 Acked Write 1 Done Done Done false 1007 Acked Read 2 Done Done Done false 1008 Acked Write 9 Done Done Done false 1009 Acked Read 2 Done Done Done false 1010 Acked Write 8 Done Done Done false 1011 Acked Read 6 Done Done Done false 1012 Acked Write 4 Done Done Done false 1013 Acked Read 6 Done Done Done false 1014 Acked Write 8 Done Done Done false 1015 Acked Read 10 Done Done Done false 1016 Acked Read 6 Done Done Done false 1017 Acked Read 8 Done Done Done false 1018 Acked Write 8 Done Done Done false 1019 Acked Write 9 Done Done Done false 1020 Acked Write 5 Done Done Done false 1021 Acked Write 10 Done Done Done false 1022 Acked Read 5 Done Done Done false 1023 Acked Write 7 Done Done Done false 1024 Acked Read 9 Done Done Done false 1025 Acked Write 10 Done Done Done false 1026 Acked Read 4 Done Done Done false 1027 Acked Read 9 Done Done Done false 1028 Acked Read 5 Done Done Done false 1029 Acked Read 5 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 30 30 30 90 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 0 0 0 Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:52:22.313736892Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1404} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 55 55 55 86 86 86 F F F 1 020-039 53 53 53 83 83 83 F F F 2 040-059 55 55 55 86 86 86 F F F 3 060-079 55 55 55 86 86 86 F F F 4 080-099 54 54 54 85 85 85 F F F 5 100-119 55 55 55 86 86 86 T T T 6 120-139 55 55 55 86 86 86 F F F 7 140-159 55 55 55 86 86 86 T T T 8 160-179 53 53 53 83 83 83 F F F 9 180-199 53 53 53 83 83 83 T T T 10 200-219 53 53 53 83 83 83 T T T 11 220-239 53 53 53 83 83 83 T T T 12 240-259 49 49 49 79 79 79 F F F 13 260-279 55 55 55 86 86 86 F F F 14 280-299 50 50 50 80 80 80 F F F 15 300-319 52 52 52 82 82 82 T T T 16 320-339 42 42 42 69 69 69 T T T 17 340-359 52 52 52 82 82 82 T T T 18 360-379 52 52 52 82 82 82 F F F 19 380-399 55 55 55 86 86 86 F F F 20 400-419 27 27 27 41 41 41 F F F 21 420-439 52 52 52 82 82 82 F F F 22 440-459 53 53 53 83 83 83 F F F 23 460-479 55 55 55 86 86 86 T T T 24 480-499 52 52 52 82 82 82 T T T 25 500-519 52 52 52 82 82 82 T T T 26 520-539 54 54 54 84 84 84 T T T 27 540-559 54 54 54 84 84 84 F F F 28 560-579 53 53 53 83 83 83 T T T 29 580-599 55 55 55 86 86 86 F F F Max gen: 55, Max flush: 86 A Difference in extent metadata is expected here On loop 28 Verifying data now No change after verify Loop: 28 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 55 55 55 86 86 86 F F F 1 020-039 53 53 53 83 83 83 F F F 2 040-059 55 55 55 86 86 86 F F F 3 060-079 55 55 55 86 86 86 F F F 4 080-099 54 54 54 85 85 85 F F F 5 100-119 56 56 56 87 87 87 F F F 6 120-139 55 55 55 86 86 86 F F F 7 140-159 56 56 56 87 87 87 F F F 8 160-179 53 53 53 83 83 83 F F F 9 180-199 56 56 56 87 87 87 F F F 10 200-219 56 56 56 87 87 87 F F F 11 220-239 56 56 56 87 87 87 F F F 12 240-259 49 49 49 79 79 79 F F F 13 260-279 55 55 55 86 86 86 F F F 14 280-299 50 50 50 80 80 80 F F F 15 300-319 56 56 56 87 87 87 F F F 16 320-339 56 56 56 87 87 87 F F F 17 340-359 56 56 56 87 87 87 F F F 18 360-379 52 52 52 82 82 82 F F F 19 380-399 55 55 55 86 86 86 F F F 20 400-419 27 27 27 41 41 41 F F F 21 420-439 52 52 52 82 82 82 F F F 22 440-459 53 53 53 83 83 83 F F F 23 460-479 56 56 56 87 87 87 F F F 24 480-499 56 56 56 87 87 87 F F F 25 500-519 56 56 56 87 87 87 F F F 26 520-539 56 56 56 87 87 87 F F F 27 540-559 54 54 54 84 84 84 F F F 28 560-579 56 56 56 87 87 87 F F F 29 580-599 55 55 55 86 86 86 F F F Max gen: 56, Max flush: 87 Begin loop 29 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:26.751447018Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1412,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:26.795951331Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1412,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:26.796505813Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1412} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"3818f0fc76fe787c6cc412af30fd59b930203885\",\n git_commit_timestamp: \"2024-11-01T16:08:30.000000000Z\",\n git_branch: \"renovate/hyper-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:26.796546618Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1412} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:26.79656249Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1412} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:26.796577162Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1412} {"msg":"Crucible e92ec8c9-cb27-421b-857f-06fcffa2396b has session id: 6fb3a10c-513f-4ac6-8ec0-b1f80e3af149","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:26.797008227Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1412,"session_id":"6fb3a10c-513f-4ac6-8ec0-b1f80e3af149"} {"msg":"Upstairs opts: Upstairs UUID: e92ec8c9-cb27-421b-857f-06fcffa2396b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:26.797051173Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1412,"session_id":"6fb3a10c-513f-4ac6-8ec0-b1f80e3af149"} {"msg":"Crucible stats registered with UUID: e92ec8c9-cb27-421b-857f-06fcffa2396b","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:26.797077216Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1412,"session_id":"6fb3a10c-513f-4ac6-8ec0-b1f80e3af149"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:26.828370408Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1412,"task":"crutest"} {"msg":"The guest has requested activation with gen:58","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:26.828429386Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1412} {"msg":"e92ec8c9-cb27-421b-857f-06fcffa2396b active request set","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:26.828499545Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1412,"session_id":"6fb3a10c-513f-4ac6-8ec0-b1f80e3af149"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:26.828534309Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1412,"client":"0","":"downstairs","session_id":"6fb3a10c-513f-4ac6-8ec0-b1f80e3af149"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:26.828565953Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1412,"client":"0","":"downstairs","session_id":"6fb3a10c-513f-4ac6-8ec0-b1f80e3af149"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:26.828584506Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1412,"client":"1","":"downstairs","session_id":"6fb3a10c-513f-4ac6-8ec0-b1f80e3af149"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:26.828603158Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1412,"client":"1","":"downstairs","session_id":"6fb3a10c-513f-4ac6-8ec0-b1f80e3af149"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:26.828619511Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1412,"client":"2","":"downstairs","session_id":"6fb3a10c-513f-4ac6-8ec0-b1f80e3af149"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:26.828636343Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1412,"client":"2","":"downstairs","session_id":"6fb3a10c-513f-4ac6-8ec0-b1f80e3af149"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:26.828666357Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1412,"":"io task","client":"2","":"downstairs","session_id":"6fb3a10c-513f-4ac6-8ec0-b1f80e3af149"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:26.828700841Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1412,"":"io task","client":"0","":"downstairs","session_id":"6fb3a10c-513f-4ac6-8ec0-b1f80e3af149"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:26.828854761Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1412,"":"io task","client":"1","":"downstairs","session_id":"6fb3a10c-513f-4ac6-8ec0-b1f80e3af149"} {"msg":"ds_connection connected from Ok(127.0.0.1:63750)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:26.828952583Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1412,"":"io task","client":"0","":"downstairs","session_id":"6fb3a10c-513f-4ac6-8ec0-b1f80e3af149"} {"msg":"ds_connection connected from Ok(127.0.0.1:47706)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:26.829053716Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1412,"":"io task","client":"2","":"downstairs","session_id":"6fb3a10c-513f-4ac6-8ec0-b1f80e3af149"} {"msg":"ds_connection connected from Ok(127.0.0.1:48237)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:26.829089341Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1412,"":"io task","client":"1","":"downstairs","session_id":"6fb3a10c-513f-4ac6-8ec0-b1f80e3af149"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:26.829588795Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1412,"client":"0","":"downstairs","session_id":"6fb3a10c-513f-4ac6-8ec0-b1f80e3af149"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:26.829665215Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1412,"client":"0","":"downstairs","session_id":"6fb3a10c-513f-4ac6-8ec0-b1f80e3af149"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:26.82970261Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1412,"client":"1","":"downstairs","session_id":"6fb3a10c-513f-4ac6-8ec0-b1f80e3af149"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:26.829733554Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1412,"client":"1","":"downstairs","session_id":"6fb3a10c-513f-4ac6-8ec0-b1f80e3af149"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:26.829811314Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1412,"client":"2","":"downstairs","session_id":"6fb3a10c-513f-4ac6-8ec0-b1f80e3af149"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:26.829843258Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1412,"client":"2","":"downstairs","session_id":"6fb3a10c-513f-4ac6-8ec0-b1f80e3af149"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:26.830221907Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1412,"client":"0","":"downstairs","session_id":"6fb3a10c-513f-4ac6-8ec0-b1f80e3af149"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:26.830251351Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1412,"client":"1","":"downstairs","session_id":"6fb3a10c-513f-4ac6-8ec0-b1f80e3af149"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:26.83032198Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1412,"client":"2","":"downstairs","session_id":"6fb3a10c-513f-4ac6-8ec0-b1f80e3af149"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:26.830842747Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1412,"client":"2","":"downstairs","session_id":"6fb3a10c-513f-4ac6-8ec0-b1f80e3af149"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:26.830871061Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1412,"client":"2","":"downstairs","session_id":"6fb3a10c-513f-4ac6-8ec0-b1f80e3af149"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:26.830886773Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1412,"session_id":"6fb3a10c-513f-4ac6-8ec0-b1f80e3af149"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:26.830899474Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1412,"client":"0","":"downstairs","session_id":"6fb3a10c-513f-4ac6-8ec0-b1f80e3af149"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:26.830913186Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1412,"client":"0","":"downstairs","session_id":"6fb3a10c-513f-4ac6-8ec0-b1f80e3af149"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:26.830927328Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1412,"session_id":"6fb3a10c-513f-4ac6-8ec0-b1f80e3af149"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:26.83094258Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1412,"client":"1","":"downstairs","session_id":"6fb3a10c-513f-4ac6-8ec0-b1f80e3af149"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:26.830956552Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1412,"client":"1","":"downstairs","session_id":"6fb3a10c-513f-4ac6-8ec0-b1f80e3af149"} {"msg":"[0]R flush_numbers[0..12]: [86, 83, 86, 86, 85, 87, 86, 87, 83, 87, 87, 87]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:26.830970373Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1412,"":"downstairs","":"downstairs","session_id":"6fb3a10c-513f-4ac6-8ec0-b1f80e3af149"} {"msg":"[0]R generation[0..12]: [55, 53, 55, 55, 54, 56, 55, 56, 53, 56, 56, 56]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:26.830984215Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1412,"":"downstairs","":"downstairs","session_id":"6fb3a10c-513f-4ac6-8ec0-b1f80e3af149"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:26.831012079Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1412,"":"downstairs","":"downstairs","session_id":"6fb3a10c-513f-4ac6-8ec0-b1f80e3af149"} {"msg":"[1]R flush_numbers[0..12]: [86, 83, 86, 86, 85, 87, 86, 87, 83, 87, 87, 87]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:26.831027111Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1412,"":"downstairs","":"downstairs","session_id":"6fb3a10c-513f-4ac6-8ec0-b1f80e3af149"} {"msg":"[1]R generation[0..12]: [55, 53, 55, 55, 54, 56, 55, 56, 53, 56, 56, 56]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:26.831040772Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1412,"":"downstairs","":"downstairs","session_id":"6fb3a10c-513f-4ac6-8ec0-b1f80e3af149"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:26.831054814Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1412,"":"downstairs","":"downstairs","session_id":"6fb3a10c-513f-4ac6-8ec0-b1f80e3af149"} {"msg":"[2]R flush_numbers[0..12]: [86, 83, 86, 86, 85, 87, 86, 87, 83, 87, 87, 87]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:26.831068286Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1412,"":"downstairs","":"downstairs","session_id":"6fb3a10c-513f-4ac6-8ec0-b1f80e3af149"} {"msg":"Wait for a query_work_queue command to finish before sending IO [2]R generation[0..12]: [55, 53, 55, 55, 54, 56, 55, 56, 53, 56, 56, 56]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:26.831090329Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1412,"":"downstairs","":"downstairs","session_id":"6fb3a10c-513f-4ac6-8ec0-b1f80e3af149"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:26.831104591Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1412,"":"downstairs","":"downstairs","session_id":"6fb3a10c-513f-4ac6-8ec0-b1f80e3af149"} {"msg":"Max found gen is 57","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:26.831117602Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1412,"":"downstairs","":"downstairs","session_id":"6fb3a10c-513f-4ac6-8ec0-b1f80e3af149"} {"msg":"Generation requested: 58 >= found:57","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:26.831130854Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1412,"":"downstairs","":"downstairs","session_id":"6fb3a10c-513f-4ac6-8ec0-b1f80e3af149"} {"msg":"Next flush: 88","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:26.831143896Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1412,"":"downstairs","":"downstairs","session_id":"6fb3a10c-513f-4ac6-8ec0-b1f80e3af149"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:26.831159808Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1412,"":"downstairs","":"downstairs","session_id":"6fb3a10c-513f-4ac6-8ec0-b1f80e3af149"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:26.831173729Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1412,"session_id":"6fb3a10c-513f-4ac6-8ec0-b1f80e3af149"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:26.831185911Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1412,"client":"0","":"downstairs","session_id":"6fb3a10c-513f-4ac6-8ec0-b1f80e3af149"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:26.831199243Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1412,"client":"1","":"downstairs","session_id":"6fb3a10c-513f-4ac6-8ec0-b1f80e3af149"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2024-11-01T16:52:26.831212525Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1412,"client":"2","":"downstairs","session_id":"6fb3a10c-513f-4ac6-8ec0-b1f80e3af149"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:26.831228637Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1412,"session_id":"6fb3a10c-513f-4ac6-8ec0-b1f80e3af149"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:26.831241438Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1412,"session_id":"6fb3a10c-513f-4ac6-8ec0-b1f80e3af149"} {"msg":"e92ec8c9-cb27-421b-857f-06fcffa2396b is now active with session: 6fb3a10c-513f-4ac6-8ec0-b1f80e3af149","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:26.8312544Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1412,"session_id":"6fb3a10c-513f-4ac6-8ec0-b1f80e3af149"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:26.831266641Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1412,"session_id":"6fb3a10c-513f-4ac6-8ec0-b1f80e3af149"} {"msg":"The guest has finished waiting for activation with:58","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:26.831278853Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1412} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 305 len 36864 data: 8 8 9 9 6 5 6 6 7 02/30 Read block 105 len 12288 03/30 Read block 528 len 32768 04/30 Write block 211 len 28672 data: 10 8 7 6 4 8 8 05/30 Read block 198 len 28672 06/30 Write block 29 len 36864 data: 3 3 5 5 3 4 5 5 6 07/30 Read block 217 len 4096 08/30 Read block 547 len 12288 09/30 Write block 36 len 4096 data: 6 10/30 Read block 323 len 20480 11/30 Flush 12/30 Write block 583 len 24576 data: 6 6 8 8 8 6 13/30 Read block 196 len 40960 14/30 Read block 141 len 4096 15/30 Read block 173 len 16384 16/30 Write block 65 len 20480 data: 7 10 9 9 10 17/30 Read block 487 len 28672 18/30 Read block 510 len 28672 19/30 Read block 448 len 32768 20/30 Write block 504 len 24576 data: 4 4 5 5 7 6 21/30 Read block 92 len 24576 22/30 Write block 466 len 36864 data: 5 5 5 6 7 9 9 8 8 23/30 Read block 494 len 12288 24/30 Read block 180 len 8192 25/30 Flush 26/30 Write block 391 len 8192 data: 4 4 27/30 Read block 159 len 32768 28/30 Read block 562 len 4096 29/30 Read block 168 len 12288 30/30 Write block 150 len 4096 data: 8 ---------------------------------------------------------------- Crucible gen:58 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 2 Done Done Done false 1026 Acked Read 8 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Read 3 Done Done Done false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:52:26.85539509Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1412} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 55 55 55 86 86 86 F F F 1 020-039 58 58 58 88 88 88 F F F 2 040-059 55 55 55 86 86 86 F F F 3 060-079 58 58 58 89 89 89 F F F 4 080-099 54 54 54 85 85 85 F F F 5 100-119 56 56 56 87 87 87 F F F 6 120-139 55 55 55 86 86 86 F F F 7 140-159 56 56 56 87 87 87 T T T 8 160-179 53 53 53 83 83 83 F F F 9 180-199 56 56 56 87 87 87 F F F 10 200-219 58 58 58 88 88 88 F F F 11 220-239 56 56 56 87 87 87 F F F 12 240-259 49 49 49 79 79 79 F F F 13 260-279 55 55 55 86 86 86 F F F 14 280-299 50 50 50 80 80 80 F F F 15 300-319 58 58 58 88 88 88 F F F 16 320-339 56 56 56 87 87 87 F F F 17 340-359 56 56 56 87 87 87 F F F 18 360-379 52 52 52 82 82 82 F F F 19 380-399 55 55 55 86 86 86 T T T 20 400-419 27 27 27 41 41 41 F F F 21 420-439 52 52 52 82 82 82 F F F 22 440-459 53 53 53 83 83 83 F F F 23 460-479 58 58 58 89 89 89 F F F 24 480-499 56 56 56 87 87 87 F F F 25 500-519 58 58 58 89 89 89 F F F 26 520-539 56 56 56 87 87 87 F F F 27 540-559 54 54 54 84 84 84 F F F 28 560-579 56 56 56 87 87 87 F F F 29 580-599 58 58 58 89 89 89 F F F Max gen: 58, Max flush: 89 A Difference in extent metadata is expected here On loop 29 Verifying data now No change after verify Loop: 29 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 55 55 55 86 86 86 F F F 1 020-039 58 58 58 88 88 88 F F F 2 040-059 55 55 55 86 86 86 F F F 3 060-079 58 58 58 89 89 89 F F F 4 080-099 54 54 54 85 85 85 F F F 5 100-119 56 56 56 87 87 87 F F F 6 120-139 55 55 55 86 86 86 F F F 7 140-159 59 59 59 90 90 90 F F F 8 160-179 53 53 53 83 83 83 F F F 9 180-199 56 56 56 87 87 87 F F F 10 200-219 58 58 58 88 88 88 F F F 11 220-239 56 56 56 87 87 87 F F F 12 240-259 49 49 49 79 79 79 F F F 13 260-279 55 55 55 86 86 86 F F F 14 280-299 50 50 50 80 80 80 F F F 15 300-319 58 58 58 88 88 88 F F F 16 320-339 56 56 56 87 87 87 F F F 17 340-359 56 56 56 87 87 87 F F F 18 360-379 52 52 52 82 82 82 F F F 19 380-399 59 59 59 90 90 90 F F F 20 400-419 27 27 27 41 41 41 F F F 21 420-439 52 52 52 82 82 82 F F F 22 440-459 53 53 53 83 83 83 F F F 23 460-479 58 58 58 89 89 89 F F F 24 480-499 56 56 56 87 87 87 F F F 25 500-519 58 58 58 89 89 89 F F F 26 520-539 56 56 56 87 87 87 F F F 27 540-559 54 54 54 84 84 84 F F F 28 560-579 56 56 56 87 87 87 F F F 29 580-599 58 58 58 89 89 89 F F F Max gen: 59, Max flush: 90 Begin loop 30 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:31.124476711Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1420,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:52:31.167057888Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1420,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:31.167445978Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1420,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:31.199221782Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1420,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:31.199741359Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1420} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"3818f0fc76fe787c6cc412af30fd59b930203885\",\n git_commit_timestamp: \"2024-11-01T16:08:30.000000000Z\",\n git_branch: \"renovate/hyper-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:31.199783734Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1420} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:31.199800746Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1420} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:31.199819819Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1420} {"msg":"Crucible d8e857b7-0304-4b1a-9a65-a24ba017ad63 has session id: a683fb15-16a4-47e9-8a38-8e12d6f2ee8d","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:31.200280488Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1420,"session_id":"a683fb15-16a4-47e9-8a38-8e12d6f2ee8d"} {"msg":"Upstairs opts: Upstairs UUID: d8e857b7-0304-4b1a-9a65-a24ba017ad63, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:31.200320523Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1420,"session_id":"a683fb15-16a4-47e9-8a38-8e12d6f2ee8d"} {"msg":"Crucible stats registered with UUID: d8e857b7-0304-4b1a-9a65-a24ba017ad63","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:31.200343466Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1420,"session_id":"a683fb15-16a4-47e9-8a38-8e12d6f2ee8d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:31.231206803Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1420,"task":"crutest"} {"msg":"The guest has requested activation with gen:60","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:31.231236397Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1420} {"msg":"d8e857b7-0304-4b1a-9a65-a24ba017ad63 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:31.231284973Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1420,"session_id":"a683fb15-16a4-47e9-8a38-8e12d6f2ee8d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:31.231309386Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1420,"client":"0","":"downstairs","session_id":"a683fb15-16a4-47e9-8a38-8e12d6f2ee8d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:31.2313416Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1420,"client":"0","":"downstairs","session_id":"a683fb15-16a4-47e9-8a38-8e12d6f2ee8d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:31.231374415Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1420,"client":"1","":"downstairs","session_id":"a683fb15-16a4-47e9-8a38-8e12d6f2ee8d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:31.231405899Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1420,"client":"1","":"downstairs","session_id":"a683fb15-16a4-47e9-8a38-8e12d6f2ee8d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:31.231432722Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1420,"client":"2","":"downstairs","session_id":"a683fb15-16a4-47e9-8a38-8e12d6f2ee8d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:31.231464626Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1420,"client":"2","":"downstairs","session_id":"a683fb15-16a4-47e9-8a38-8e12d6f2ee8d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:31.231498631Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1420,"":"io task","client":"2","":"downstairs","session_id":"a683fb15-16a4-47e9-8a38-8e12d6f2ee8d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:31.231534895Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1420,"":"io task","client":"0","":"downstairs","session_id":"a683fb15-16a4-47e9-8a38-8e12d6f2ee8d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:31.23157168Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1420,"":"io task","client":"1","":"downstairs","session_id":"a683fb15-16a4-47e9-8a38-8e12d6f2ee8d"} {"msg":"ds_connection connected from Ok(127.0.0.1:41979)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:31.231606954Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1420,"":"io task","client":"0","":"downstairs","session_id":"a683fb15-16a4-47e9-8a38-8e12d6f2ee8d"} {"msg":"ds_connection connected from Ok(127.0.0.1:41577)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:31.231666662Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1420,"":"io task","client":"2","":"downstairs","session_id":"a683fb15-16a4-47e9-8a38-8e12d6f2ee8d"} {"msg":"ds_connection connected from Ok(127.0.0.1:33866)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:31.231795769Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1420,"":"io task","client":"1","":"downstairs","session_id":"a683fb15-16a4-47e9-8a38-8e12d6f2ee8d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:31.23226901Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1420,"client":"1","":"downstairs","session_id":"a683fb15-16a4-47e9-8a38-8e12d6f2ee8d"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:31.232297003Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1420,"client":"1","":"downstairs","session_id":"a683fb15-16a4-47e9-8a38-8e12d6f2ee8d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:31.232323907Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1420,"client":"0","":"downstairs","session_id":"a683fb15-16a4-47e9-8a38-8e12d6f2ee8d"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:31.232380674Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1420,"client":"0","":"downstairs","session_id":"a683fb15-16a4-47e9-8a38-8e12d6f2ee8d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:31.232408628Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1420,"client":"2","":"downstairs","session_id":"a683fb15-16a4-47e9-8a38-8e12d6f2ee8d"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:31.232438592Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1420,"client":"2","":"downstairs","session_id":"a683fb15-16a4-47e9-8a38-8e12d6f2ee8d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:31.232899191Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1420,"client":"2","":"downstairs","session_id":"a683fb15-16a4-47e9-8a38-8e12d6f2ee8d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:31.233027857Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1420,"client":"1","":"downstairs","session_id":"a683fb15-16a4-47e9-8a38-8e12d6f2ee8d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:31.233315214Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1420,"client":"0","":"downstairs","session_id":"a683fb15-16a4-47e9-8a38-8e12d6f2ee8d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:31.233442271Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1420,"client":"2","":"downstairs","session_id":"a683fb15-16a4-47e9-8a38-8e12d6f2ee8d"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:31.233469714Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1420,"client":"2","":"downstairs","session_id":"a683fb15-16a4-47e9-8a38-8e12d6f2ee8d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:31.233484996Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1420,"session_id":"a683fb15-16a4-47e9-8a38-8e12d6f2ee8d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:31.233636276Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1420,"client":"1","":"downstairs","session_id":"a683fb15-16a4-47e9-8a38-8e12d6f2ee8d"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:31.23366683Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1420,"client":"1","":"downstairs","session_id":"a683fb15-16a4-47e9-8a38-8e12d6f2ee8d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:31.233681962Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1420,"session_id":"a683fb15-16a4-47e9-8a38-8e12d6f2ee8d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:31.233694914Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1420,"client":"0","":"downstairs","session_id":"a683fb15-16a4-47e9-8a38-8e12d6f2ee8d"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:31.233708765Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1420,"client":"0","":"downstairs","session_id":"a683fb15-16a4-47e9-8a38-8e12d6f2ee8d"} {"msg":"[0]R flush_numbers[0..12]: [86, 88, 86, 89, 85, 87, 86, 90, 83, 87, 88, 87]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:31.233723327Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1420,"":"downstairs","":"downstairs","session_id":"a683fb15-16a4-47e9-8a38-8e12d6f2ee8d"} {"msg":"[0]R generation[0..12]: [55, 58, 55, 58, 54, 56, 55, 59, 53, 56, 58, 56]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:31.233741129Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1420,"":"downstairs","":"downstairs","session_id":"a683fb15-16a4-47e9-8a38-8e12d6f2ee8d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:31.233756662Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1420,"":"downstairs","":"downstairs","session_id":"a683fb15-16a4-47e9-8a38-8e12d6f2ee8d"} {"msg":"[1]R flush_numbers[0..12]: [86, 88, 86, 89, 85, 87, 86, 90, 83, 87, 88, 87]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:31.233770443Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1420,"":"downstairs","":"downstairs","session_id":"a683fb15-16a4-47e9-8a38-8e12d6f2ee8d"} {"msg":"[1]R generation[0..12]: [55, 58, 55, 58, 54, 56, 55, 59, 53, 56, 58, 56]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:31.233784125Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1420,"":"downstairs","":"downstairs","session_id":"a683fb15-16a4-47e9-8a38-8e12d6f2ee8d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:31.233797937Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1420,"":"downstairs","":"downstairs","session_id":"a683fb15-16a4-47e9-8a38-8e12d6f2ee8d"} {"msg":"[2]R flush_numbers[0..12]: [86, 88, 86, 89, 85, 87, 86, 90, 83, 87, 88, 87]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:31.233811589Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1420,"":"downstairs","":"downstairs","session_id":"a683fb15-16a4-47e9-8a38-8e12d6f2ee8d"} {"msg":"[2]R generation[0..12]: [55, 58, 55, 58, 54, 56, 55, 59, 53, 56, 58, 56]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:31.23382503Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1420,"":"downstairs","":"downstairs","session_id":"a683fb15-16a4-47e9-8a38-8e12d6f2ee8d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:31.233838892Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1420,"":"downstairs","":"downstairs","session_id":"a683fb15-16a4-47e9-8a38-8e12d6f2ee8d"} {"msg":"Max found gen is 60","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:31.233851934Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1420,"":"downstairs","":"downstairs","session_id":"a683fb15-16a4-47e9-8a38-8e12d6f2ee8d"} {"msg":"Generation requested: 60 >= found:60","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:31.233864925Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1420,"":"downstairs","":"downstairs","session_id":"a683fb15-16a4-47e9-8a38-8e12d6f2ee8d"} {"msg":"Next flush: 91","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:31.233877817Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1420,"":"downstairs","":"downstairs","session_id":"a683fb15-16a4-47e9-8a38-8e12d6f2ee8d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:31.233890849Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1420,"":"downstairs","":"downstairs","session_id":"a683fb15-16a4-47e9-8a38-8e12d6f2ee8d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:31.233904181Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1420,"session_id":"a683fb15-16a4-47e9-8a38-8e12d6f2ee8d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-01T16:52:31.233916212Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1420,"client":"0","":"downstairs","session_id":"a683fb15-16a4-47e9-8a38-8e12d6f2ee8d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:31.233934694Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1420,"client":"1","":"downstairs","session_id":"a683fb15-16a4-47e9-8a38-8e12d6f2ee8d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:31.233950996Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1420,"client":"2","":"downstairs","session_id":"a683fb15-16a4-47e9-8a38-8e12d6f2ee8d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:31.233964588Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1420,"session_id":"a683fb15-16a4-47e9-8a38-8e12d6f2ee8d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:31.2339769Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1420,"session_id":"a683fb15-16a4-47e9-8a38-8e12d6f2ee8d"} {"msg":"d8e857b7-0304-4b1a-9a65-a24ba017ad63 is now active with session: a683fb15-16a4-47e9-8a38-8e12d6f2ee8d","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:31.233989721Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1420,"session_id":"a683fb15-16a4-47e9-8a38-8e12d6f2ee8d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:31.234008594Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1420,"session_id":"a683fb15-16a4-47e9-8a38-8e12d6f2ee8d"} {"msg":"The guest has finished waiting for activation with:60","v":0,"name":"crucible","level":30,"time"Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 :"2024-11-01T16:52:31.234027726Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1420} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 103 len 16384 data: 5 5 5 6 02/30 Read block 403 len 24576 03/30 Write block 96 len 28672 data: 6 6 6 7 6 6 6 04/30 Read block 371 len 36864 05/30 Read block 445 len 4096 06/30 Read block 237 len 24576 07/30 Write block 217 len 36864 data: 9 10 6 6 6 6 9 9 8 08/30 Read block 202 len 8192 09/30 Write block 157 len 40960 data: 6 6 4 6 6 6 6 5 5 4 10/30 Read block 328 len 32768 11/30 Read block 345 len 40960 12/30 Read block 365 len 32768 13/30 Flush 14/30 Write block 266 len 28672 data: 7 6 6 6 6 8 8 15/30 Read block 398 len 12288 16/30 Read block 572 len 8192 17/30 Write block 566 len 8192 data: 4 5 18/30 Read block 198 len 32768 19/30 Read block 260 len 28672 20/30 Read block 520 len 28672 21/30 Write block 352 len 16384 data: 5 6 5 5 22/30 Read block 434 len 32768 23/30 Read block 248 len 24576 24/30 Read block 290 len 8192 25/30 Write block 294 len 12288 data: 7 5 6 26/30 Write block 79 len 40960 data: 7 7 7 9 9 8 9 7 8 8 27/30 Read block 530 len 8192 28/30 Write block 537 len 20480 data: 6 5 4 4 4 29/30 Read block 501 len 36864 30/30 Write block 477 len 32768 data: 8 10 9 9 7 6 5 5 ---------------------------------------------------------------- Crucible gen:60 GIO:true work queues: Upstairs:0 downstairs:17 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1013 Acked Write 7 Done Done Done false 1014 Acked Read 3 Done Done Done false 1015 Acked Read 2 Done Done Done false 1016 Acked Write 2 Done Done Done false 1017 Acked Read 8 Done Done Done false 1018 Acked Read 7 Done Done Done false 1019 Acked Read 7 Done Done Done false 1020 Acked Write 4 Done Done Done false 1021 Acked Read 8 Done Done Done false 1022 Acked Read 6 Done Done Done false 1023 Acked Read 2 Done Done Done false 1024 Acked Write 3 Done Done Done false 1025 Acked Write 10 Done Done Done false 1026 Acked Read 2 Done Done Done false 1027 Acked Write 5 Done Done Done false 1028 Acked Read 9 Done Done Done false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 16 16 16 48 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1012 1012 1012 Downstairs last five completed: 1012 1011 1010 1009 1008 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:52:31.256620388Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1420} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 55 55 55 86 86 86 F F F 1 020-039 58 58 58 88 88 88 F F F 2 040-059 55 55 55 86 86 86 F F F 3 060-079 58 58 58 89 89 89 T T T 4 080-099 60 60 60 91 91 91 T T T 5 100-119 60 60 60 91 91 91 F F F 6 120-139 55 55 55 86 86 86 F F F 7 140-159 60 60 60 91 91 91 F F F 8 160-179 60 60 60 91 91 91 F F F 9 180-199 56 56 56 87 87 87 F F F 10 200-219 60 60 60 91 91 91 F F F 11 220-239 60 60 60 91 91 91 F F F 12 240-259 49 49 49 79 79 79 F F F 13 260-279 55 55 55 86 86 86 T T T 14 280-299 50 50 50 80 80 80 T T T 15 300-319 58 58 58 88 88 88 F F F 16 320-339 56 56 56 87 87 87 F F F 17 340-359 56 56 56 87 87 87 T T T 18 360-379 52 52 52 82 82 82 F F F 19 380-399 59 59 59 90 90 90 F F F 20 400-419 27 27 27 41 41 41 F F F 21 420-439 52 52 52 82 82 82 F F F 22 440-459 53 53 53 83 83 83 F F F 23 460-479 58 58 58 89 89 89 T T T 24 480-499 56 56 56 87 87 87 T T T 25 500-519 58 58 58 89 89 89 F F F 26 520-539 56 56 56 87 87 87 T T T 27 540-559 54 54 54 84 84 84 T T T 28 560-579 56 56 56 87 87 87 T T T 29 580-599 58 58 58 89 89 89 F F F Max gen: 60, Max flush: 91 A Difference in extent metadata is expected here On loop 30 Verifying data now No change after verify Loop: 30 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 55 55 55 86 86 86 F F F 1 020-039 58 58 58 88 88 88 F F F 2 040-059 55 55 55 86 86 86 F F F 3 060-079 61 61 61 92 92 92 F F F 4 080-099 61 61 61 92 92 92 F F F 5 100-119 60 60 60 91 91 91 F F F 6 120-139 55 55 55 86 86 86 F F F 7 140-159 60 60 60 91 91 91 F F F 8 160-179 60 60 60 91 91 91 F F F 9 180-199 56 56 56 87 87 87 F F F 10 200-219 60 60 60 91 91 91 F F F 11 220-239 60 60 60 91 91 91 F F F 12 240-259 49 49 49 79 79 79 F F F 13 260-279 61 61 61 92 92 92 F F F 14 280-299 61 61 61 92 92 92 F F F 15 300-319 58 58 58 88 88 88 F F F 16 320-339 56 56 56 87 87 87 F F F 17 340-359 61 61 61 92 92 92 F F F 18 360-379 52 52 52 82 82 82 F F F 19 380-399 59 59 59 90 90 90 F F F 20 400-419 27 27 27 41 41 41 F F F 21 420-439 52 52 52 82 82 82 F F F 22 440-459 53 53 53 83 83 83 F F F 23 460-479 61 61 61 92 92 92 F F F 24 480-499 61 61 61 92 92 92 F F F 25 500-519 58 58 58 89 89 89 F F F 26 520-539 61 61 61 92 92 92 F F F 27 540-559 61 61 61 92 92 92 F F F 28 560-579 61 61 61 92 92 92 F F F 29 580-599 58 58 58 89 89 89 F F F Max gen: 61, Max flush: 92 Begin loop 31 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:35.654665104Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1428,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:35.698729451Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1428,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:35.699322478Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1428} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"3818f0fc76fe787c6cc412af30fd59b930203885\",\n git_commit_timestamp: \"2024-11-01T16:08:30.000000000Z\",\n git_branch: \"renovate/hyper-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:35.699373814Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1428} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:35.699392007Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1428} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:35.699409719Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1428} {"msg":"Crucible c9987fa8-983c-44e9-8f59-d5cde01731b1 has session id: a8e44bc2-7205-4ff7-b44d-1defbecfb36d","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:35.699872738Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1428,"session_id":"a8e44bc2-7205-4ff7-b44d-1defbecfb36d"} {"msg":"Upstairs opts: Upstairs UUID: c9987fa8-983c-44e9-8f59-d5cde01731b1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:35.699913014Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1428,"session_id":"a8e44bc2-7205-4ff7-b44d-1defbecfb36d"} {"msg":"Crucible stats registered with UUID: c9987fa8-983c-44e9-8f59-d5cde01731b1","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:35.699937237Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1428,"session_id":"a8e44bc2-7205-4ff7-b44d-1defbecfb36d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:35.730663006Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1428,"task":"crutest"} {"msg":"The guest has requested activation with gen:62","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:35.73069479Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1428} {"msg":"c9987fa8-983c-44e9-8f59-d5cde01731b1 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:35.730725184Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1428,"session_id":"a8e44bc2-7205-4ff7-b44d-1defbecfb36d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:35.730742526Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1428,"client":"0","":"downstairs","session_id":"a8e44bc2-7205-4ff7-b44d-1defbecfb36d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:35.73077073Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1428,"client":"0","":"downstairs","session_id":"a8e44bc2-7205-4ff7-b44d-1defbecfb36d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:35.730793002Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1428,"client":"1","":"downstairs","session_id":"a8e44bc2-7205-4ff7-b44d-1defbecfb36d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:35.730810535Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1428,"client":"1","":"downstairs","session_id":"a8e44bc2-7205-4ff7-b44d-1defbecfb36d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:35.730827017Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1428,"client":"2","":"downstairs","session_id":"a8e44bc2-7205-4ff7-b44d-1defbecfb36d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:35.730843919Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1428,"client":"2","":"downstairs","session_id":"a8e44bc2-7205-4ff7-b44d-1defbecfb36d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:35.730860451Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1428,"":"io task","client":"2","":"downstairs","session_id":"a8e44bc2-7205-4ff7-b44d-1defbecfb36d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:35.730885074Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1428,"":"io task","client":"0","":"downstairs","session_id":"a8e44bc2-7205-4ff7-b44d-1defbecfb36d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:35.730918039Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1428,"":"io task","client":"1","":"downstairs","session_id":"a8e44bc2-7205-4ff7-b44d-1defbecfb36d"} {"msg":"ds_connection connected from Ok(127.0.0.1:36603)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:35.731196654Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1428,"":"io task","client":"2","":"downstairs","session_id":"a8e44bc2-7205-4ff7-b44d-1defbecfb36d"} {"msg":"ds_connection connected from Ok(127.0.0.1:54240)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:35.73123795Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1428,"":"io task","client":"1","":"downstairs","session_id":"a8e44bc2-7205-4ff7-b44d-1defbecfb36d"} {"msg":"ds_connection connected from Ok(127.0.0.1:34951)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:35.731270754Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1428,"":"io task","client":"0","":"downstairs","session_id":"a8e44bc2-7205-4ff7-b44d-1defbecfb36d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:35.731807763Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1428,"client":"0","":"downstairs","session_id":"a8e44bc2-7205-4ff7-b44d-1defbecfb36d"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:35.731847368Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1428,"client":"0","":"downstairs","session_id":"a8e44bc2-7205-4ff7-b44d-1defbecfb36d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:35.731878412Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1428,"client":"1","":"downstairs","session_id":"a8e44bc2-7205-4ff7-b44d-1defbecfb36d"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:35.731908626Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1428,"client":"1","":"downstairs","session_id":"a8e44bc2-7205-4ff7-b44d-1defbecfb36d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:35.7319358Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1428,"client":"2","":"downstairs","session_id":"a8e44bc2-7205-4ff7-b44d-1defbecfb36d"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:35.731964184Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1428,"client":"2","":"downstairs","session_id":"a8e44bc2-7205-4ff7-b44d-1defbecfb36d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:35.732491111Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1428,"client":"1","":"downstairs","session_id":"a8e44bc2-7205-4ff7-b44d-1defbecfb36d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:35.732532597Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1428,"client":"0","":"downstairs","session_id":"a8e44bc2-7205-4ff7-b44d-1defbecfb36d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:35.732568801Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1428,"client":"2","":"downstairs","session_id":"a8e44bc2-7205-4ff7-b44d-1defbecfb36d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:35.733087728Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1428,"client":"1","":"downstairs","session_id":"a8e44bc2-7205-4ff7-b44d-1defbecfb36d"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:35.733123073Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1428,"client":"1","":"downstairs","session_id":"a8e44bc2-7205-4ff7-b44d-1defbecfb36d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:35.733139155Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1428,"session_id":"a8e44bc2-7205-4ff7-b44d-1defbecfb36d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:35.733151737Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1428,"client":"2","":"downstairs","session_id":"a8e44bc2-7205-4ff7-b44d-1defbecfb36d"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:35.733165618Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1428,"client":"2","":"downstairs","session_id":"a8e44bc2-7205-4ff7-b44d-1defbecfb36d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:35.73317922Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1428,"session_id":"a8e44bc2-7205-4ff7-b44d-1defbecfb36d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:35.733191782Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1428,"client":"0","":"downstairs","session_id":"a8e44bc2-7205-4ff7-b44d-1defbecfb36d"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:35.733205303Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1428,"client":"0","":"downstairs","session_id":"a8e44bc2-7205-4ff7-b44d-1defbecfb36d"} {"msg":"[0]R flush_numbers[0..12]: [86, 88, 86, 92, 92, 91, 86, 91, 91, 87, 91, 91]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:35.733219085Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1428,"":"downstairs","":"downstairs","session_id":"a8e44bc2-7205-4ff7-b44d-1defbecfb36d"} {"msg":"[0]R generation[0..12]: [55, 58, 55, 61, 61, 60, 55, 60, 60, 56, 60, 60]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:35.733233187Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1428,"":"downstairs","":"downstairs","session_id":"a8e44bc2-7205-4ff7-b44d-1defbecfb36d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:35.733247359Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1428,"":"downstairs","":"downstairs","session_id":"a8e44bc2-7205-4ff7-b44d-1defbecfb36d"} {"msg":"[1]R flush_numbers[0..12]: [86, 88, 86, 92, 92, 91, 86, 91, 91, 87, 91, 91]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:35.733265321Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1428,"":"downstairs","":"downstairs","session_id":"a8e44bc2-7205-4ff7-b44d-1defbecfb36d"} {"msg":"[1]R generation[0..12]: [55, 58, 55, 61, 61, 60, 55, 60, 60, 56, 60, 60]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:35.733280273Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1428,"":"downstairs","":"downstairs","session_id":"a8e44bc2-7205-4ff7-b44d-1defbecfb36d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:35.733294305Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1428,"":"downstairs","":"downstairs","session_id":"a8e44bc2-7205-4ff7-b44d-1defbecfb36d"} {"msg":"[2]R flush_numbers[0..12]: [86, 88, 86, 92, 92, 91, 86, 91, 91, 87, 91, 91]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:35.733307966Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1428,"":"downstairs","":"downstairs","session_id":"a8e44bc2-7205-4ff7-b44d-1defbecfb36d"} {"msg":"[2]R generation[0..12]: [55, 58, 55, 61, 61, 60, 55, 60, 60, 56, 60, 60]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:35.73333287Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1428,"":"downstairs","":"downstairs","session_id":"a8e44bc2-7205-4ff7-b44d-1defbecfb36d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-01T16:52:35.733347422Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1428,"":"downstairs","":"downstairs","session_id":"a8e44bc2-7205-4ff7-b44d-1defbecfb36d"} {"msg":"Max found gen is 62","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:35.733373765Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1428,"":"downstairs","":"downstairs","session_id":"a8e44bc2-7205-4ff7-b44d-1defbecfb36d"} {"msg":"Generation requested: 62 >= found:62","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:35.733388587Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1428,"":"downstairs","":"downstairs","session_id":"a8e44bc2-7205-4ff7-b44d-1defbecfb36d"} {"msg":"Next flush: 93","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:35.733401939Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1428,"":"downstairs","":"downstairs","session_id":"a8e44bc2-7205-4ff7-b44d-1defbecfb36d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:35.7334152Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1428,"":"downstairs","":"downstairs","session_id":"a8e44bc2-7205-4ff7-b44d-1defbecfb36d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:35.733428402Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1428,"session_id":"a8e44bc2-7205-4ff7-b44d-1defbecfb36d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:35.733440504Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1428,"client":"0","":"downstairs","session_id":"a8e44bc2-7205-4ff7-b44d-1defbecfb36d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:35.733454095Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1428,"client":"1","":"downstairs","session_id":"a8e44bc2-7205-4ff7-b44d-1defbecfb36d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:35.733467437Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1428,"client":"2","":"downstairs","session_id":"a8e44bc2-7205-4ff7-b44d-1defbecfb36d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:35.733480869Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1428,"session_id":"a8e44bc2-7205-4ff7-b44d-1defbecfb36d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:35.733496951Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1428,"session_id":"a8e44bc2-7205-4ff7-b44d-1defbecfb36d"} {"msg":"c9987fa8-983c-44e9-8f59-d5cde01731b1 is now active with session: a8e44bc2-7205-4ff7-b44d-1defbecfb36d","v":0,"name":"crucible","level":Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 30,"time":"2024-11-01T16:52:35.733513563Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1428,"session_id":"a8e44bc2-7205-4ff7-b44d-1defbecfb36d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:35.733525975Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1428,"session_id":"a8e44bc2-7205-4ff7-b44d-1defbecfb36d"} {"msg":"The guest has finished waiting for activation with:62","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:35.733538236Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1428} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 18 len 8192 data: 5 5 02/30 Write block 408 len 40960 data: 4 3 3 3 3 3 3 3 2 4 03/30 Write block 126 len 32768 data: 7 7 6 7 7 8 9 8 04/30 Read block 536 len 32768 05/30 Write block 269 len 24576 data: 7 7 9 9 10 12 06/30 Flush 07/30 Write block 484 len 20480 data: 6 5 5 6 7 08/30 Write block 356 len 40960 data: 5 6 5 5 4 3 3 3 4 6 09/30 Write block 331 len 8192 data: 4 4 10/30 Read block 398 len 20480 11/30 Read block 397 len 8192 12/30 Flush 13/30 Write block 119 len 16384 data: 5 6 7 8 14/30 Read block 518 len 28672 15/30 Read block 430 len 12288 16/30 Write block 70 len 40960 data: 13 11 11 8 7 6 2 2 4 8 17/30 Write block 569 len 40960 data: 6 5 5 5 4 4 4 4 5 6 18/30 Write block 563 len 4096 data: 5 19/30 Write block 190 len 28672 data: 6 4 5 6 8 8 8 20/30 Read block 527 len 36864 21/30 Read block 460 len 40960 22/30 Write block 322 len 40960 data: 7 6 6 5 5 3 2 4 4 5 23/30 Flush 24/30 Write block 593 len 4096 data: 6 25/30 Read block 397 len 24576 26/30 Read block 472 len 4096 27/30 Read block 35 len 28672 28/30 Read block 557 len 24576 29/30 Write block 558 len 28672 data: 8 9 6 5 5 6 6 30/30 Write block 421 len 12288 data: 6 6 6 ---------------------------------------------------------------- Crucible gen:62 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 1 Done Done Done false 1024 Acked Read 6 Done Done Done false 1025 Acked Read 1 Done Done Done false 1026 Acked Read 7 Done Done Done false 1027 Acked Read 6 Done Done Done false 1028 Acked Write 7 Sent Sent Sent false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:52:35.759775567Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1428} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 62 62 62 93 93 93 F F F 1 020-039 58 58 58 88 88 88 F F F 2 040-059 55 55 55 86 86 86 F F F 3 060-079 62 62 62 95 95 95 F F F 4 080-099 61 61 61 92 92 92 F F F 5 100-119 62 62 62 95 95 95 F F F 6 120-139 62 62 62 95 95 95 F F F 7 140-159 60 60 60 91 91 91 F F F 8 160-179 60 60 60 91 91 91 F F F 9 180-199 62 62 62 95 95 95 F F F 10 200-219 60 60 60 91 91 91 F F F 11 220-239 60 60 60 91 91 91 F F F 12 240-259 49 49 49 79 79 79 F F F 13 260-279 62 62 62 93 93 93 F F F 14 280-299 61 61 61 92 92 92 F F F 15 300-319 58 58 58 88 88 88 F F F 16 320-339 62 62 62 95 95 95 F F F 17 340-359 62 62 62 94 94 94 F F F 18 360-379 62 62 62 94 94 94 F F F 19 380-399 59 59 59 90 90 90 F F F 20 400-419 62 62 62 93 93 93 F F F 21 420-439 52 52 52 82 82 82 T T T 22 440-459 53 53 53 83 83 83 F F F 23 460-479 61 61 61 92 92 92 F F F 24 480-499 62 62 62 94 94 94 F F F 25 500-519 58 58 58 89 89 89 F F F 26 520-539 61 61 61 92 92 92 F F F 27 540-559 61 61 61 92 92 92 T T T 28 560-579 62 62 62 95 95 95 T T T 29 580-599 58 58 58 89 89 89 T T T Max gen: 62, Max flush: 95 A Difference in extent metadata is expected here On loop 31 Verifying data now No change after verify Loop: 31 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 62 62 62 93 93 93 F F F 1 020-039 58 58 58 88 88 88 F F F 2 040-059 55 55 55 86 86 86 F F F 3 060-079 62 62 62 95 95 95 F F F 4 080-099 61 61 61 92 92 92 F F F 5 100-119 62 62 62 95 95 95 F F F 6 120-139 62 62 62 95 95 95 F F F 7 140-159 60 60 60 91 91 91 F F F 8 160-179 60 60 60 91 91 91 F F F 9 180-199 62 62 62 95 95 95 F F F 10 200-219 60 60 60 91 91 91 F F F 11 220-239 60 60 60 91 91 91 F F F 12 240-259 49 49 49 79 79 79 F F F 13 260-279 62 62 62 93 93 93 F F F 14 280-299 61 61 61 92 92 92 F F F 15 300-319 58 58 58 88 88 88 F F F 16 320-339 62 62 62 95 95 95 F F F 17 340-359 62 62 62 94 94 94 F F F 18 360-379 62 62 62 94 94 94 F F F 19 380-399 59 59 59 90 90 90 F F F 20 400-419 62 62 62 93 93 93 F F F 21 420-439 63 63 63 96 96 96 F F F 22 440-459 53 53 53 83 83 83 F F F 23 460-479 61 61 61 92 92 92 F F F 24 480-499 62 62 62 94 94 94 F F F 25 500-519 58 58 58 89 89 89 F F F 26 520-539 61 61 61 92 92 92 F F F 27 540-559 63 63 63 96 96 96 F F F 28 560-579 63 63 63 96 96 96 F F F 29 580-599 63 63 63 96 96 96 F F F Max gen: 63, Max flush: 96 Begin loop 32 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:40.04736031Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1436,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:40.090741189Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1436,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:40.091282419Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1436} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"3818f0fc76fe787c6cc412af30fd59b930203885\",\n git_commit_timestamp: \"2024-11-01T16:08:30.000000000Z\",\n git_branch: \"renovate/hyper-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:40.091323555Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1436} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:40.091343057Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1436} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:40.091358649Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1436} {"msg":"Crucible 906b3ce5-af80-4cb1-8dfd-29cf5dfa960c has session id: ce0dd030-87ac-413a-b1d6-dcb0e08b3cc3","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:40.091734908Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1436,"session_id":"ce0dd030-87ac-413a-b1d6-dcb0e08b3cc3"} {"msg":"Upstairs opts: Upstairs UUID: 906b3ce5-af80-4cb1-8dfd-29cf5dfa960c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:40.091776543Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1436,"session_id":"ce0dd030-87ac-413a-b1d6-dcb0e08b3cc3"} {"msg":"Crucible stats registered with UUID: 906b3ce5-af80-4cb1-8dfd-29cf5dfa960c","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:40.091801316Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1436,"session_id":"ce0dd030-87ac-413a-b1d6-dcb0e08b3cc3"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:40.12295462Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1436,"task":"crutest"} {"msg":"The guest has requested activation with gen:64","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:40.122988184Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1436} {"msg":"906b3ce5-af80-4cb1-8dfd-29cf5dfa960c active request set","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:40.123049042Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1436,"session_id":"ce0dd030-87ac-413a-b1d6-dcb0e08b3cc3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:40.123076396Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1436,"client":"0","":"downstairs","session_id":"ce0dd030-87ac-413a-b1d6-dcb0e08b3cc3"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:40.123100459Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1436,"client":"0","":"downstairs","session_id":"ce0dd030-87ac-413a-b1d6-dcb0e08b3cc3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:40.123115291Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1436,"client":"1","":"downstairs","session_id":"ce0dd030-87ac-413a-b1d6-dcb0e08b3cc3"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:40.123129302Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1436,"client":"1","":"downstairs","session_id":"ce0dd030-87ac-413a-b1d6-dcb0e08b3cc3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:40.123143084Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1436,"client":"2","":"downstairs","session_id":"ce0dd030-87ac-413a-b1d6-dcb0e08b3cc3"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:40.123156656Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1436,"client":"2","":"downstairs","session_id":"ce0dd030-87ac-413a-b1d6-dcb0e08b3cc3"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:40.123173048Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1436,"":"io task","client":"2","":"downstairs","session_id":"ce0dd030-87ac-413a-b1d6-dcb0e08b3cc3"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:40.12318744Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1436,"":"io task","client":"0","":"downstairs","session_id":"ce0dd030-87ac-413a-b1d6-dcb0e08b3cc3"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:40.123201382Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1436,"":"io task","client":"1","":"downstairs","session_id":"ce0dd030-87ac-413a-b1d6-dcb0e08b3cc3"} {"msg":"ds_connection connected from Ok(127.0.0.1:46224)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:40.123348821Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1436,"":"io task","client":"2","":"downstairs","session_id":"ce0dd030-87ac-413a-b1d6-dcb0e08b3cc3"} {"msg":"ds_connection connected from Ok(127.0.0.1:41444)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:40.123470026Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1436,"":"io task","client":"1","":"downstairs","session_id":"ce0dd030-87ac-413a-b1d6-dcb0e08b3cc3"} {"msg":"ds_connection connected from Ok(127.0.0.1:33396)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:40.123520743Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1436,"":"io task","client":"0","":"downstairs","session_id":"ce0dd030-87ac-413a-b1d6-dcb0e08b3cc3"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:40.124110789Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1436,"client":"0","":"downstairs","session_id":"ce0dd030-87ac-413a-b1d6-dcb0e08b3cc3"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:40.124139683Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1436,"client":"0","":"downstairs","session_id":"ce0dd030-87ac-413a-b1d6-dcb0e08b3cc3"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:40.124156655Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1436,"client":"1","":"downstairs","session_id":"ce0dd030-87ac-413a-b1d6-dcb0e08b3cc3"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:40.124170957Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1436,"client":"1","":"downstairs","session_id":"ce0dd030-87ac-413a-b1d6-dcb0e08b3cc3"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:40.124252707Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1436,"client":"2","":"downstairs","session_id":"ce0dd030-87ac-413a-b1d6-dcb0e08b3cc3"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:40.12426954Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1436,"client":"2","":"downstairs","session_id":"ce0dd030-87ac-413a-b1d6-dcb0e08b3cc3"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:40.124695084Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1436,"client":"0","":"downstairs","session_id":"ce0dd030-87ac-413a-b1d6-dcb0e08b3cc3"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:40.124820891Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1436,"client":"1","":"downstairs","session_id":"ce0dd030-87ac-413a-b1d6-dcb0e08b3cc3"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:40.124850144Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1436,"client":"2","":"downstairs","session_id":"ce0dd030-87ac-413a-b1d6-dcb0e08b3cc3"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:40.125213641Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1436,"client":"0","":"downstairs","session_id":"ce0dd030-87ac-413a-b1d6-dcb0e08b3cc3"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:40.125249156Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1436,"client":"0","":"downstairs","session_id":"ce0dd030-87ac-413a-b1d6-dcb0e08b3cc3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:40.125277699Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1436,"session_id":"ce0dd030-87ac-413a-b1d6-dcb0e08b3cc3"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:40.125347618Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1436,"client":"2","":"downstairs","session_id":"ce0dd030-87ac-413a-b1d6-dcb0e08b3cc3"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:40.125368061Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1436,"client":"2","":"downstairs","session_id":"ce0dd030-87ac-413a-b1d6-dcb0e08b3cc3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:40.125381923Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1436,"session_id":"ce0dd030-87ac-413a-b1d6-dcb0e08b3cc3"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:40.125441671Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1436,"client":"1","":"downstairs","session_id":"ce0dd030-87ac-413a-b1d6-dcb0e08b3cc3"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:40.125461323Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1436,"client":"1","":"downstairs","session_id":"ce0dd030-87ac-413a-b1d6-dcb0e08b3cc3"} {"msg":"[0]R flush_numbers[0..12]: [93, 88, 86, 95, 92, 95, 95, 91, 91, 95, 91, 91]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:40.125475415Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1436,"":"downstairs","":"downstairs","session_id":"ce0dd030-87ac-413a-b1d6-dcb0e08b3cc3"} {"msg":"[0]R generation[0..12]: [62, 58, 55, 62, 61, 62, 62, 60, 60, 62, 60, 60]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:40.125489487Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1436,"":"downstairs","":"downstairs","session_id":"ce0dd030-87ac-413a-b1d6-dcb0e08b3cc3"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:40.125503988Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1436,"":"downstairs","":"downstairs","session_id":"ce0dd030-87ac-413a-b1d6-dcb0e08b3cc3"} {"msg":"[1]R flush_numbers[0..12]: [93, 88, 86, 95, 92, 95, 95, 91, 91, 95, 91, 91]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:40.12551773Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1436,"":"downstairs","":"downstairs","session_id":"ce0dd030-87ac-413a-b1d6-dcb0e08b3cc3"} {"msg":"[1]R generation[0..12]: [62, 58, 55, 62, 61, 62, 62, 60, 60, 62, 60, 60]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:40.125531482Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1436,"":"downstairs","":"downstairs","session_id":"ce0dd030-87ac-413a-b1d6-dcb0e08b3cc3"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:40.125545504Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1436,"":"downstairs","":"downstairs","session_id":"ce0dd030-87ac-413a-b1d6-dcb0e08b3cc3"} {"msg":"[2]R flush_numbers[0..12]: [93, 88, 86, 95, 92, 95, 95, 91, 91, 95, 91, 91]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:40.125559336Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1436,"":"downstairs","":"downstairs","session_id":"ce0dd030-87ac-413a-b1d6-dcb0e08b3cc3"} {"msg":"[2]R generation[0..12]: [62, 58, 55, 62, 61, 62, 62, 60, 60, 62, 60, 60]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:40.125576038Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1436,"":"downstairs","":"downstairs","session_id":"ce0dd030-87ac-413a-b1d6-dcb0e08b3cc3"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:40.12559125Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1436,"":"downstairs","":"downstairs","session_id":"ce0dd030-87ac-413a-b1d6-dcb0e08b3cc3"} {"msg":"Max found gen is 64","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:40.125604541Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1436,"":"downstairs","":"downstairs","session_id":"ce0dd030-87ac-413a-b1d6-dcb0e08b3cc3"} {"msg":"Generation requested: 64 >= found:64","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:40.125618063Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1436,"":"downstairs","":"downstairs","session_id":"ce0dd030-87ac-413a-b1d6-dcb0e08b3cc3"} {"msg":"Next flush: 97","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:40.125631165Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1436,"":"downstairs","":"downstairs","session_id":"ce0dd030-87ac-413a-b1d6-dcb0e08b3cc3"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:40.125644297Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1436,"":"downstairs","":"downstairs","session_id":"ce0dd030-87ac-413a-b1d6-dcb0e08b3cc3"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:40.125657858Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1436,"session_id":"ce0dd030-87ac-413a-b1d6-dcb0e08b3cc3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-01T16:52:40.12566996Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1436,"client":"0","":"downstairs","session_id":"ce0dd030-87ac-413a-b1d6-dcb0e08b3cc3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:40.125690423Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1436,"client":"1","":"downstairs","session_id":"ce0dd030-87ac-413a-b1d6-dcb0e08b3cc3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:40.125703614Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1436,"client":"2","":"downstairs","session_id":"ce0dd030-87ac-413a-b1d6-dcb0e08b3cc3"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:40.125716866Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1436,"session_id":"ce0dd030-87ac-413a-b1d6-dcb0e08b3cc3"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:40.125729128Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1436,"session_id":"ce0dd030-87ac-413a-b1d6-dcb0e08b3cc3"} {"msg":"906b3ce5-af80-4cb1-8dfd-29cf5dfa960c is now active with session: ce0dd030-87ac-413a-b1d6-dcb0e08b3cc3","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:40.125742029Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1436,"session_id":"ce0dd030-87ac-413a-b1d6-dcb0e08b3cc3"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:40.125754111Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1436,"session_id":"ce0dd030-87ac-413a-b1d6-dcb0e08b3cc3"} {"msg":"The guest has finished waiting for activation with:64","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:40.125766272Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1436} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 498 len 40960 data: 6 7 7 6 5 3 5 5 6 6 02/30 Read block 36 len 36864 03/30 Write block 168 len 40960 data: 5 4 4 3 3 3 3 4 5 6 04/30 Write block 470 len 36864 data: 8 10 10 9 9 8 6 9 11 05/30 Read block 401 len 8192 06/30 Read block 18 len 28672 07/30 Read block 35 len 32768 08/30 Read block 71 len 4096 09/30 Write block 357 len 8192 data: 7 6 10/30 Write block 229 len 40960 data: 5 5 5 5 4 5 5 6 5 5 11/30 Read block 2 len 32768 12/30 Write block 315 len 12288 data: 8 9 7 13/30 Read block 566 len 4096 14/30 Write block 585 len 4096 data: 9 15/30 Flush 16/30 Write block 182 len 40960 data: 4 5 5 5 6 6 7 6 7 5 17/30 Write block 589 len 40960 data: 7 6 5 5 7 5 5 4 4 3 18/30 Write block 261 len 20480 data: 8 8 7 6 6 19/30 Write block 133 len 24576 data: 9 8 7 5 6 6 20/30 Write block 266 len 4096 data: 8 21/30 Flush 22/30 Write block 184 len 36864 data: 6 6 7 7 8 7 8 6 6 23/30 Write block 89 len 24576 data: 9 12 8 7 9 8 24/30 Read block 336 len 8192 25/30 Read block 189 len 40960 26/30 Read block 257 len 28672 27/30 Write block 253 len 24576 data: 7 5 4 4 3 5 28/30 Write block 25 len 36864 data: 5 4 4 4 4 4 6 6 4 29/30 Write block 259 len 12288 data: 6 6 9 30/30 Write block 577 len 20480 data: 6 7 7 7 6 ---------------------------------------------------------------- Crucible gen:64 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 9 Done Done Done false 1022 Acked Write 6 Done Done Done false 1023 Acked Read 2 Done Done Done false 1024 Acked Read 10 Done Done Done false 1025 Acked Read 7 Done Done Done false 1026 Acked Write 6 Done Done Done false 1027 Acked Write 9 Done Done Sent false 1028 Acked Write 3 Sent Sent Sent false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 3 7 Done 7 7 6 20 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1020 1020 1020 Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:52:40.150080595Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1436} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 62 62 62 93 93 93 F F F 1 020-039 58 58 58 88 88 88 T T T 2 040-059 55 55 55 86 86 86 F F F 3 060-079 62 62 62 95 95 95 F F F 4 080-099 61 61 61 92 92 92 T T T 5 100-119 62 62 62 95 95 95 F F F 6 120-139 64 64 64 98 98 98 F F F 7 140-159 60 60 60 91 91 91 F F F 8 160-179 64 64 64 97 97 97 F F F 9 180-199 64 64 64 98 98 98 T T T 10 200-219 60 60 60 91 91 91 F F F 11 220-239 64 64 64 97 97 97 F F F 12 240-259 49 49 49 79 79 79 T T T 13 260-279 64 64 64 98 98 98 T T T 14 280-299 61 61 61 92 92 92 F F F 15 300-319 64 64 64 97 97 97 F F F 16 320-339 62 62 62 95 95 95 F F F 17 340-359 64 64 64 97 97 97 F F F 18 360-379 62 62 62 94 94 94 F F F 19 380-399 59 59 59 90 90 90 F F F 20 400-419 62 62 62 93 93 93 F F F 21 420-439 63 63 63 96 96 96 F F F 22 440-459 53 53 53 83 83 83 F F F 23 460-479 64 64 64 97 97 97 F F F 24 480-499 64 64 64 97 97 97 F F F 25 500-519 64 64 64 97 97 97 F F F 26 520-539 61 61 61 92 92 92 F F F 27 540-559 63 63 63 96 96 96 F F F 28 560-579 63 63 63 96 96 96 T T T 29 580-599 64 64 64 98 98 98 T T T Max gen: 64, Max flush: 98 A Difference in extent metadata is expected here On loop 32 Verifying data now No change after verify Loop: 32 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 62 62 62 93 93 93 F F F 1 020-039 65 65 65 99 99 99 F F F 2 040-059 55 55 55 86 86 86 F F F 3 060-079 62 62 62 95 95 95 F F F 4 080-099 65 65 65 99 99 99 F F F 5 100-119 62 62 62 95 95 95 F F F 6 120-139 64 64 64 98 98 98 F F F 7 140-159 60 60 60 91 91 91 F F F 8 160-179 64 64 64 97 97 97 F F F 9 180-199 65 65 65 99 99 99 F F F 10 200-219 60 60 60 91 91 91 F F F 11 220-239 64 64 64 97 97 97 F F F 12 240-259 65 65 65 99 99 99 F F F 13 260-279 65 65 65 99 99 99 F F F 14 280-299 61 61 61 92 92 92 F F F 15 300-319 64 64 64 97 97 97 F F F 16 320-339 62 62 62 95 95 95 F F F 17 340-359 64 64 64 97 97 97 F F F 18 360-379 62 62 62 94 94 94 F F F 19 380-399 59 59 59 90 90 90 F F F 20 400-419 62 62 62 93 93 93 F F F 21 420-439 63 63 63 96 96 96 F F F 22 440-459 53 53 53 83 83 83 F F F 23 460-479 64 64 64 97 97 97 F F F 24 480-499 64 64 64 97 97 97 F F F 25 500-519 64 64 64 97 97 97 F F F 26 520-539 61 61 61 92 92 92 F F F 27 540-559 63 63 63 96 96 96 F F F 28 560-579 65 65 65 99 99 99 F F F 29 580-599 65 65 65 99 99 99 F F F Max gen: 65, Max flush: 99 Begin loop 33 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:44.48174873Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1444,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:44.525738248Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1444,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:44.526421046Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1444} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"3818f0fc76fe787c6cc412af30fd59b930203885\",\n git_commit_timestamp: \"2024-11-01T16:08:30.000000000Z\",\n git_branch: \"renovate/hyper-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:44.526462761Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1444} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:44.526477943Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1444} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:44.526492415Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1444} {"msg":"Crucible 6b16c29f-baf0-45fd-843c-b00964ff037c has session id: 6904c0e0-2643-41e2-bb0a-79d4b8e63f81","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:44.526850681Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1444,"session_id":"6904c0e0-2643-41e2-bb0a-79d4b8e63f81"} {"msg":"Upstairs opts: Upstairs UUID: 6b16c29f-baf0-45fd-843c-b00964ff037c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:44.526891016Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1444,"session_id":"6904c0e0-2643-41e2-bb0a-79d4b8e63f81"} {"msg":"Crucible stats registered with UUID: 6b16c29f-baf0-45fd-843c-b00964ff037c","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:44.526914969Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1444,"session_id":"6904c0e0-2643-41e2-bb0a-79d4b8e63f81"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:44.558092596Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1444,"task":"crutest"} {"msg":"The guest has requested activation with gen:66","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:44.558135432Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1444} {"msg":"6b16c29f-baf0-45fd-843c-b00964ff037c active request set","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:44.558207321Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1444,"session_id":"6904c0e0-2643-41e2-bb0a-79d4b8e63f81"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:44.558255617Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1444,"client":"0","":"downstairs","session_id":"6904c0e0-2643-41e2-bb0a-79d4b8e63f81"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:44.558290442Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1444,"client":"0","":"downstairs","session_id":"6904c0e0-2643-41e2-bb0a-79d4b8e63f81"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:44.558324366Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1444,"client":"1","":"downstairs","session_id":"6904c0e0-2643-41e2-bb0a-79d4b8e63f81"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:44.558385064Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1444,"client":"1","":"downstairs","session_id":"6904c0e0-2643-41e2-bb0a-79d4b8e63f81"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:44.558421839Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1444,"client":"2","":"downstairs","session_id":"6904c0e0-2643-41e2-bb0a-79d4b8e63f81"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:44.558455263Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1444,"client":"2","":"downstairs","session_id":"6904c0e0-2643-41e2-bb0a-79d4b8e63f81"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:44.558481637Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1444,"":"io task","client":"2","":"downstairs","session_id":"6904c0e0-2643-41e2-bb0a-79d4b8e63f81"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:44.558514741Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1444,"":"io task","client":"0","":"downstairs","session_id":"6904c0e0-2643-41e2-bb0a-79d4b8e63f81"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:44.558547025Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1444,"":"io task","client":"1","":"downstairs","session_id":"6904c0e0-2643-41e2-bb0a-79d4b8e63f81"} {"msg":"ds_connection connected from Ok(127.0.0.1:42020)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:44.55858339Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1444,"":"io task","client":"2","":"downstairs","session_id":"6904c0e0-2643-41e2-bb0a-79d4b8e63f81"} {"msg":"ds_connection connected from Ok(127.0.0.1:52703)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:44.558670401Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1444,"":"io task","client":"0","":"downstairs","session_id":"6904c0e0-2643-41e2-bb0a-79d4b8e63f81"} {"msg":"ds_connection connected from Ok(127.0.0.1:51893)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:44.558696854Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1444,"":"io task","client":"1","":"downstairs","session_id":"6904c0e0-2643-41e2-bb0a-79d4b8e63f81"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:44.55928255Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1444,"client":"2","":"downstairs","session_id":"6904c0e0-2643-41e2-bb0a-79d4b8e63f81"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:44.559308443Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1444,"client":"2","":"downstairs","session_id":"6904c0e0-2643-41e2-bb0a-79d4b8e63f81"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:44.559323675Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1444,"client":"1","":"downstairs","session_id":"6904c0e0-2643-41e2-bb0a-79d4b8e63f81"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:44.559337257Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1444,"client":"1","":"downstairs","session_id":"6904c0e0-2643-41e2-bb0a-79d4b8e63f81"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:44.559350799Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1444,"client":"0","":"downstairs","session_id":"6904c0e0-2643-41e2-bb0a-79d4b8e63f81"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:44.55936418Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1444,"client":"0","":"downstairs","session_id":"6904c0e0-2643-41e2-bb0a-79d4b8e63f81"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:44.560006943Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1444,"client":"2","":"downstairs","session_id":"6904c0e0-2643-41e2-bb0a-79d4b8e63f81"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:44.560098345Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1444,"client":"0","":"downstairs","session_id":"6904c0e0-2643-41e2-bb0a-79d4b8e63f81"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:44.560121558Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1444,"client":"1","":"downstairs","session_id":"6904c0e0-2643-41e2-bb0a-79d4b8e63f81"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:44.560710514Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1444,"client":"1","":"downstairs","session_id":"6904c0e0-2643-41e2-bb0a-79d4b8e63f81"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:44.560748649Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1444,"client":"1","":"downstairs","session_id":"6904c0e0-2643-41e2-bb0a-79d4b8e63f81"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:44.560776132Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1444,"session_id":"6904c0e0-2643-41e2-bb0a-79d4b8e63f81"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:44.560796495Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1444,"client":"0","":"downstairs","session_id":"6904c0e0-2643-41e2-bb0a-79d4b8e63f81"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:44.560810967Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1444,"client":"0","":"downstairs","session_id":"6904c0e0-2643-41e2-bb0a-79d4b8e63f81"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:44.560824408Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1444,"session_id":"6904c0e0-2643-41e2-bb0a-79d4b8e63f81"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:44.56083686Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1444,"client":"2","":"downstairs","session_id":"6904c0e0-2643-41e2-bb0a-79d4b8e63f81"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:44.560850642Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1444,"client":"2","":"downstairs","session_id":"6904c0e0-2643-41e2-bb0a-79d4b8e63f81"} {"msg":"[0]R flush_numbers[0..12]: [93, 99, 86, 95, 99, 95, 98, 91, 97, 99, 91, 97]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:44.560864384Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1444,"":"downstairs","":"downstairs","session_id":"6904c0e0-2643-41e2-bb0a-79d4b8e63f81"} {"msg":"[0]R generation[0..12]: [62, 65, 55, 62, 65, 62, 64, 60, 64, 65, 60, 64]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:44.560878146Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1444,"":"downstairs","":"downstairs","session_id":"6904c0e0-2643-41e2-bb0a-79d4b8e63f81"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:44.560892307Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1444,"":"downstairs","":"downstairs","session_id":"6904c0e0-2643-41e2-bb0a-79d4b8e63f81"} {"msg":"[1]R flush_numbers[0..12]: [93, 99, 86, 95, 99, 95, 98, 91, 97, 99, 91, 97]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:44.560905989Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1444,"":"downstairs","":"downstairs","session_id":"6904c0e0-2643-41e2-bb0a-79d4b8e63f81"} {"msg":"[1]R generation[0..12]: [62, 65, 55, 62, 65, 62, 64, 60, 64, 65, 60, 64]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:44.560919781Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1444,"":"downstairs","":"downstairs","session_id":"6904c0e0-2643-41e2-bb0a-79d4b8e63f81"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:44.560933873Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1444,"":"downstairs","":"downstairs","session_id":"6904c0e0-2643-41e2-bb0a-79d4b8e63f81"} {"msg":"[2]R flush_numbers[0..12]: [93, 99, 86, 95, 99, 95, 98, 91, 97, 99, 91, 97]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:44.560947624Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1444,"":"downstairs","":"downstairs","session_id":"6904c0e0-2643-41e2-bb0a-79d4b8e63f81"} {"msg":"[2]R generation[0..12]: [62, 65, 55, 62, 65, 62, 64, 60, 64, 65, 60, 64]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:44.560961136Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1444,"":"downstairs","":"downstairs","session_id":"6904c0e0-2643-41e2-bb0a-79d4b8e63f81"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:44.560975048Z","hostname":"Wait for a query_work_queue command to finish before sending IO w-01JBMA2T0XD4W63AXFF321JBBP","pid":1444,"":"downstairs","":"downstairs","session_id":"6904c0e0-2643-41e2-bb0a-79d4b8e63f81"} {"msg":"Max found gen is 66","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:44.561010103Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1444,"":"downstairs","":"downstairs","session_id":"6904c0e0-2643-41e2-bb0a-79d4b8e63f81"} {"msg":"Generation requested: 66 >= found:66","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:44.561037266Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1444,"":"downstairs","":"downstairs","session_id":"6904c0e0-2643-41e2-bb0a-79d4b8e63f81"} {"msg":"Next flush: 100","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:44.561058509Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1444,"":"downstairs","":"downstairs","session_id":"6904c0e0-2643-41e2-bb0a-79d4b8e63f81"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:44.561081052Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1444,"":"downstairs","":"downstairs","session_id":"6904c0e0-2643-41e2-bb0a-79d4b8e63f81"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:44.561106525Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1444,"session_id":"6904c0e0-2643-41e2-bb0a-79d4b8e63f81"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:44.561122577Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1444,"client":"0","":"downstairs","session_id":"6904c0e0-2643-41e2-bb0a-79d4b8e63f81"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:44.561136339Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1444,"client":"1","":"downstairs","session_id":"6904c0e0-2643-41e2-bb0a-79d4b8e63f81"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:44.561149741Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1444,"client":"2","":"downstairs","session_id":"6904c0e0-2643-41e2-bb0a-79d4b8e63f81"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:44.561163332Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1444,"session_id":"6904c0e0-2643-41e2-bb0a-79d4b8e63f81"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:44.561175744Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1444,"session_id":"6904c0e0-2643-41e2-bb0a-79d4b8e63f81"} {"msg":"6b16c29f-baf0-45fd-843c-b00964ff037c is now active with session: 6904c0e0-2643-41e2-bb0a-79d4b8e63f81","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:44.561188325Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1444,"session_id":"6904c0e0-2643-41e2-bb0a-79d4b8e63f81"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:44.561200217Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1444,"session_id":"6904c0e0-2643-41e2-bb0a-79d4b8e63f81"} {"msg":"The guest has finished waiting for activation with:66","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:44.561212629Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1444} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 175 len 24576 data: 5 6 7 6 6 6 02/30 Read block 423 len 20480 03/30 Write block 377 len 24576 data: 3 3 3 3 5 7 04/30 Write block 587 len 20480 data: 9 7 8 7 6 05/30 Read block 450 len 28672 06/30 Read block 219 len 36864 07/30 Read block 314 len 20480 08/30 Read block 521 len 36864 09/30 Write block 292 len 4096 data: 5 10/30 Write block 456 len 36864 data: 3 2 2 2 2 4 5 4 4 11/30 Read block 507 len 4096 12/30 Read block 62 len 16384 13/30 Read block 440 len 28672 14/30 Read block 134 len 20480 15/30 Read block 287 len 8192 16/30 Read block 228 len 28672 17/30 Write block 12 len 32768 data: 4 4 5 5 5 5 6 6 18/30 Read block 65 len 8192 19/30 Write block 53 len 32768 data: 5 4 4 5 5 5 5 4 20/30 Read block 41 len 12288 21/30 Flush 22/30 Write block 533 len 16384 data: 7 7 6 7 23/30 Read block 179 len 12288 24/30 Read block 257 len 20480 25/30 Write block 362 len 4096 data: 4 26/30 Write block 511 len 40960 data: 7 7 4 3 2 3 3 3 4 4 27/30 Read block 179 len 36864 28/30 Write block 310 len 28672 data: 6 7 7 8 7 9 10 29/30 Write block 418 len 40960 data: 6 6 6 7 7 7 7 8 8 7 30/30 Read block 479 len 4096 ---------------------------------------------------------------- Crucible gen:66 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 4 Done Done Done false 1022 Acked Read 3 Done Done Done false 1023 Acked Read 5 Done Done Done false 1024 Acked Write 1 Done Done Done false 1025 Acked Write 10 Done Done Done false 1026 Acked Read 9 Done Done Done false 1027 Acked Write 7 Sent Done Done false 1028 Acked Write 10 Sent Sent Done false 1029 Acked Read 1 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 2 0 5 Done 6 7 9 22 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1020 1020 1020 Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:52:44.585244515Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1444} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 66 66 66 100 100 100 F F F 1 020-039 65 65 65 99 99 99 F F F 2 040-059 66 66 66 100 100 100 F F F 3 060-079 66 66 66 100 100 100 F F F 4 080-099 65 65 65 99 99 99 F F F 5 100-119 62 62 62 95 95 95 F F F 6 120-139 64 64 64 98 98 98 F F F 7 140-159 60 60 60 91 91 91 F F F 8 160-179 66 66 66 100 100 100 F F F 9 180-199 66 66 66 100 100 100 F F F 10 200-219 60 60 60 91 91 91 F F F 11 220-239 64 64 64 97 97 97 F F F 12 240-259 65 65 65 99 99 99 F F F 13 260-279 65 65 65 99 99 99 F F F 14 280-299 66 66 66 100 100 100 F F F 15 300-319 64 64 64 97 97 97 T T T 16 320-339 62 62 62 95 95 95 F F F 17 340-359 64 64 64 97 97 97 F F F 18 360-379 66 66 66 100 100 100 T T T 19 380-399 66 66 66 100 100 100 F F F 20 400-419 62 62 62 93 93 93 T T T 21 420-439 63 63 63 96 96 96 T T T 22 440-459 66 66 66 100 100 100 F F F 23 460-479 66 66 66 100 100 100 F F F 24 480-499 64 64 64 97 97 97 F F F 25 500-519 64 64 64 97 97 97 T T T 26 520-539 61 61 61 92 92 92 T T T 27 540-559 63 63 63 96 96 96 F F F 28 560-579 65 65 65 99 99 99 F F F 29 580-599 66 66 66 100 100 100 F F F Max gen: 66, Max flush: 100 A Difference in extent metadata is expected here On loop 33 Verifying data now No change after verify Loop: 33 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 66 66 66 100 100 100 F F F 1 020-039 65 65 65 99 99 99 F F F 2 040-059 66 66 66 100 100 100 F F F 3 060-079 66 66 66 100 100 100 F F F 4 080-099 65 65 65 99 99 99 F F F 5 100-119 62 62 62 95 95 95 F F F 6 120-139 64 64 64 98 98 98 F F F 7 140-159 60 60 60 91 91 91 F F F 8 160-179 66 66 66 100 100 100 F F F 9 180-199 66 66 66 100 100 100 F F F 10 200-219 60 60 60 91 91 91 F F F 11 220-239 64 64 64 97 97 97 F F F 12 240-259 65 65 65 99 99 99 F F F 13 260-279 65 65 65 99 99 99 F F F 14 280-299 66 66 66 100 100 100 F F F 15 300-319 67 67 67 101 101 101 F F F 16 320-339 62 62 62 95 95 95 F F F 17 340-359 64 64 64 97 97 97 F F F 18 360-379 67 67 67 101 101 101 F F F 19 380-399 66 66 66 100 100 100 F F F 20 400-419 67 67 67 101 101 101 F F F 21 420-439 67 67 67 101 101 101 F F F 22 440-459 66 66 66 100 100 100 F F F 23 460-479 66 66 66 100 100 100 F F F 24 480-499 64 64 64 97 97 97 F F F 25 500-519 67 67 67 101 101 101 F F F 26 520-539 67 67 67 101 101 101 F F F 27 540-559 63 63 63 96 96 96 F F F 28 560-579 65 65 65 99 99 99 F F F 29 580-599 66 66 66 100 100 100 F F F Max gen: 67, Max flush: 101 Begin loop 34 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:48.902072656Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1452,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:48.946213953Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1452,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:48.946877729Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1452} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"3818f0fc76fe787c6cc412af30fd59b930203885\",\n git_commit_timestamp: \"2024-11-01T16:08:30.000000000Z\",\n git_branch: \"renovate/hyper-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:48.946917544Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1452} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:48.946932446Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1452} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:48.946946928Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1452} {"msg":"Crucible 660fcb52-c0d3-404a-a018-77c06fbb5008 has session id: 3ac6d3b1-8236-45cc-a4bc-20abe6b1b715","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:48.947346269Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1452,"session_id":"3ac6d3b1-8236-45cc-a4bc-20abe6b1b715"} {"msg":"Upstairs opts: Upstairs UUID: 660fcb52-c0d3-404a-a018-77c06fbb5008, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:48.947397536Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1452,"session_id":"3ac6d3b1-8236-45cc-a4bc-20abe6b1b715"} {"msg":"Crucible stats registered with UUID: 660fcb52-c0d3-404a-a018-77c06fbb5008","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:48.947423309Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1452,"session_id":"3ac6d3b1-8236-45cc-a4bc-20abe6b1b715"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:48.97839738Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1452,"task":"crutest"} {"msg":"The guest has requested activation with gen:68","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:48.978431455Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1452} {"msg":"660fcb52-c0d3-404a-a018-77c06fbb5008 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:48.978466379Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1452,"session_id":"3ac6d3b1-8236-45cc-a4bc-20abe6b1b715"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:48.978483291Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1452,"client":"0","":"downstairs","session_id":"3ac6d3b1-8236-45cc-a4bc-20abe6b1b715"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:48.978511745Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1452,"client":"0","":"downstairs","session_id":"3ac6d3b1-8236-45cc-a4bc-20abe6b1b715"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:48.978529297Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1452,"client":"1","":"downstairs","session_id":"3ac6d3b1-8236-45cc-a4bc-20abe6b1b715"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:48.978546439Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1452,"client":"1","":"downstairs","session_id":"3ac6d3b1-8236-45cc-a4bc-20abe6b1b715"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:48.978562912Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1452,"client":"2","":"downstairs","session_id":"3ac6d3b1-8236-45cc-a4bc-20abe6b1b715"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:48.97863054Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1452,"client":"2","":"downstairs","session_id":"3ac6d3b1-8236-45cc-a4bc-20abe6b1b715"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:48.978670605Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1452,"":"io task","client":"2","":"downstairs","session_id":"3ac6d3b1-8236-45cc-a4bc-20abe6b1b715"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:48.97870997Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1452,"":"io task","client":"0","":"downstairs","session_id":"3ac6d3b1-8236-45cc-a4bc-20abe6b1b715"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:48.978741655Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1452,"":"io task","client":"1","":"downstairs","session_id":"3ac6d3b1-8236-45cc-a4bc-20abe6b1b715"} {"msg":"ds_connection connected from Ok(127.0.0.1:51266)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:48.978770768Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1452,"":"io task","client":"2","":"downstairs","session_id":"3ac6d3b1-8236-45cc-a4bc-20abe6b1b715"} {"msg":"ds_connection connected from Ok(127.0.0.1:63315)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:48.978799242Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1452,"":"io task","client":"1","":"downstairs","session_id":"3ac6d3b1-8236-45cc-a4bc-20abe6b1b715"} {"msg":"ds_connection connected from Ok(127.0.0.1:40859)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:48.978828126Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1452,"":"io task","client":"0","":"downstairs","session_id":"3ac6d3b1-8236-45cc-a4bc-20abe6b1b715"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:48.979622988Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1452,"client":"0","":"downstairs","session_id":"3ac6d3b1-8236-45cc-a4bc-20abe6b1b715"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:48.979650792Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1452,"client":"0","":"downstairs","session_id":"3ac6d3b1-8236-45cc-a4bc-20abe6b1b715"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:48.979665963Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1452,"client":"1","":"downstairs","session_id":"3ac6d3b1-8236-45cc-a4bc-20abe6b1b715"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:48.979680115Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1452,"client":"1","":"downstairs","session_id":"3ac6d3b1-8236-45cc-a4bc-20abe6b1b715"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:48.979693687Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1452,"client":"2","":"downstairs","session_id":"3ac6d3b1-8236-45cc-a4bc-20abe6b1b715"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:48.979707679Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1452,"client":"2","":"downstairs","session_id":"3ac6d3b1-8236-45cc-a4bc-20abe6b1b715"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:48.980350132Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1452,"client":"0","":"downstairs","session_id":"3ac6d3b1-8236-45cc-a4bc-20abe6b1b715"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:48.980485019Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1452,"client":"1","":"downstairs","session_id":"3ac6d3b1-8236-45cc-a4bc-20abe6b1b715"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:48.980508362Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1452,"client":"2","":"downstairs","session_id":"3ac6d3b1-8236-45cc-a4bc-20abe6b1b715"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:48.98088396Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1452,"client":"0","":"downstairs","session_id":"3ac6d3b1-8236-45cc-a4bc-20abe6b1b715"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:48.980912564Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1452,"client":"0","":"downstairs","session_id":"3ac6d3b1-8236-45cc-a4bc-20abe6b1b715"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:48.980927436Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1452,"session_id":"3ac6d3b1-8236-45cc-a4bc-20abe6b1b715"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:48.981058523Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1452,"client":"1","":"downstairs","session_id":"3ac6d3b1-8236-45cc-a4bc-20abe6b1b715"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:48.981081336Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1452,"client":"1","":"downstairs","session_id":"3ac6d3b1-8236-45cc-a4bc-20abe6b1b715"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:48.981096278Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1452,"session_id":"3ac6d3b1-8236-45cc-a4bc-20abe6b1b715"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:48.981121351Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1452,"client":"2","":"downstairs","session_id":"3ac6d3b1-8236-45cc-a4bc-20abe6b1b715"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:48.981138503Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1452,"client":"2","":"downstairs","session_id":"3ac6d3b1-8236-45cc-a4bc-20abe6b1b715"} {"msg":"[0]R flush_numbers[0..12]: [100, 99, 100, 100, 99, 95, 98, 91, 100, 100, 91, 97]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:48.981156826Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1452,"":"downstairs","":"downstairs","session_id":"3ac6d3b1-8236-45cc-a4bc-20abe6b1b715"} {"msg":"[0]R generation[0..12]: [66, 65, 66, 66, 65, 62, 64, 60, 66, 66, 60, 64]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:48.981170837Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1452,"":"downstairs","":"downstairs","session_id":"3ac6d3b1-8236-45cc-a4bc-20abe6b1b715"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:48.981184919Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1452,"":"downstairs","":"downstairs","session_id":"3ac6d3b1-8236-45cc-a4bc-20abe6b1b715"} {"msg":"[1]R flush_numbers[0..12]: [100, 99, 100, 100, 99, 95, 98, 91, 100, 100, 91, 97]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:48.981198731Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1452,"":"downstairs","":"downstairs","session_id":"3ac6d3b1-8236-45cc-a4bc-20abe6b1b715"} {"msg":"[1]R generation[0..12]: [66, 65, 66, 66, 65, 62, 64, 60, 66, 66, 60, 64]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:48.981212283Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1452,"":"downstairs","":"downstairs","session_id":"3ac6d3b1-8236-45cc-a4bc-20abe6b1b715"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:48.981226435Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1452,"":"downstairs","":"downstairs","session_id":"3ac6d3b1-8236-45cc-a4bc-20abe6b1b715"} {"msg":"[2]R flush_numbers[0..12]: [100, 99, 100, 100, 99, 95, 98, 91, 100, 100, 91, 97]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:48.981240046Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1452,"":"downstairs","":"downstairs","session_id":"3ac6d3b1-8236-45cc-a4bc-20abe6b1b715"} {"msg":"[2]R generation[0..12]: [66, 65, 66, 66, 65, 62, 64, 60, 66, 66, 60, 64]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:48.981253558Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1452,"":"downstairs","":"downstairs","session_id":"3ac6d3b1-8236-45cc-a4bc-20abe6b1b715"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:48.98126739Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1452,"":"downstairs","":"downstairs","session_id":"3ac6d3b1-8236-45cc-a4bc-20abe6b1b715"} {"msg":"Max found gen is 68","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:48.981280442Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1452,"":"downstairs","":"downstairs","session_id":"3ac6d3b1-8236-45cc-a4bc-20abe6b1b715"} {"msg":"Generation requested: 68 >= found:68","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:48.981293543Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1452,"":"downstairs","":"downstairs","session_id":"3ac6d3b1-8236-45cc-a4bc-20abe6b1b715"} {"msg":"Next flush: 102","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:48.981306385Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1452,"":"downstairs","":"downstairs","session_id":"3ac6d3b1-8236-45cc-a4bc-20abe6b1b715"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:48.981319277Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1452,"":"downstairs","":"downstairs","session_id":"3ac6d3b1-8236-45cc-a4bc-20abe6b1b715"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:48.981335629Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1452,"session_id":"3ac6d3b1-8236-45cc-a4bc-20abe6b1b715"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-01T16:52:48.98134797Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1452,"client":"0","":"downstairs","session_id":"3ac6d3b1-8236-45cc-a4bc-20abe6b1b715"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:48.981369323Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1452,"client":"1","":"downstairs","session_id":"3ac6d3b1-8236-45cc-a4bc-20abe6b1b715"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:48.981382665Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1452,"client":"2","":"downstairs","session_id":"3ac6d3b1-8236-45cc-a4bc-20abe6b1b715"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:48.981395926Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1452,"session_id":"3ac6d3b1-8236-45cc-a4bc-20abe6b1b715"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:48.981408108Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1452,"session_id":"3ac6d3b1-8236-45cc-a4bc-20abe6b1b715"} {"msg":"660fcb52-c0d3-404a-a018-77c06fbb5008 is now active with session: 3ac6d3b1-8236-45cc-a4bc-20abe6b1b715","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:48.98142084Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1452,"session_id":"3ac6d3b1-8236-45cc-a4bc-20abe6b1b715"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:48.981432951Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1452,"session_id":"3ac6d3b1-8236-45cc-a4bc-20abe6b1b715"} {"msg":"The guest has finished waiting for activation with:68","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:48.981445053Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1452} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 193 len 24576 data: 7 9 9 9 9 8 02/30 Read block 377 len 24576 03/30 Read block 424 len 16384 04/30 Write block 473 len 4096 data: 10 05/30 Write block 491 len 20480 data: 7 5 3 3 4 06/30 Write block 373 len 8192 data: 5 4 07/30 Read block 564 len 36864 08/30 Read block 527 len 32768 09/30 Read block 531 len 24576 10/30 Write block 43 len 16384 data: 12 11 9 10 11/30 Read block 388 len 8192 12/30 Write block 291 len 36864 data: 4 6 6 8 6 7 6 5 5 13/30 Read block 518 len 20480 14/30 Read block 339 len 12288 15/30 Flush 16/30 Write block 217 len 40960 data: 10 11 7 7 7 7 10 10 9 7 17/30 Read block 223 len 20480 18/30 Write block 2 len 28672 data: 4 4 4 5 6 5 5 19/30 Write block 427 len 20480 data: 8 8 7 6 6 20/30 Read block 261 len 20480 21/30 Flush 22/30 Write block 555 len 20480 data: 7 7 7 9 10 23/30 Write block 535 len 8192 data: 7 8 24/30 Read block 118 len 12288 25/30 Flush 26/30 Write block 127 len 28672 data: 8 7 8 8 9 10 10 27/30 Read block 32 len 20480 28/30 Write block 351 len 12288 data: 4 6 7 29/30 Read block 343 len 28672 30/30 Read block 320 len 32768 ---------------------------------------------------------------- Crucible gen:68 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 7 Done Done Done false 1026 Acked Read 5 Done Done Done false 1027 Acked Write 3 Done Done Done false 1028 Acked Read 7 Done Done Done false 1029 Acked Read 8 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 1 2 Done 4 5 4 13 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:52:49.0063385Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1452} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 68 68 68 103 103 103 F F F 1 020-039 65 65 65 99 99 99 F F F 2 040-059 68 68 68 102 102 102 F F F 3 060-079 66 66 66 100 100 100 F F F 4 080-099 65 65 65 99 99 99 F F F 5 100-119 62 62 62 95 95 95 F F F 6 120-139 64 64 64 98 98 98 T T T 7 140-159 60 60 60 91 91 91 F F F 8 160-179 66 66 66 100 100 100 F F F 9 180-199 68 68 68 102 102 102 F F F 10 200-219 68 68 68 103 103 103 F F F 11 220-239 68 68 68 103 103 103 F F F 12 240-259 65 65 65 99 99 99 F F F 13 260-279 65 65 65 99 99 99 F F F 14 280-299 68 68 68 102 102 102 F F F 15 300-319 67 67 67 101 101 101 F F F 16 320-339 62 62 62 95 95 95 F F F 17 340-359 64 64 64 97 97 97 T T T 18 360-379 68 68 68 102 102 102 F F F 19 380-399 66 66 66 100 100 100 F F F 20 400-419 67 67 67 101 101 101 F F F 21 420-439 68 68 68 103 103 103 F F F 22 440-459 66 66 66 100 100 100 F F F 23 460-479 68 68 68 102 102 102 F F F 24 480-499 68 68 68 102 102 102 F F F 25 500-519 67 67 67 101 101 101 F F F 26 520-539 68 68 68 104 104 104 F F F 27 540-559 68 68 68 104 104 104 F F F 28 560-579 65 65 65 99 99 99 F F F 29 580-599 66 66 66 100 100 100 F F F Max gen: 68, Max flush: 104 A Difference in extent metadata is expected here On loop 34 Verifying data now No change after verify Loop: 34 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 68 68 68 103 103 103 F F F 1 020-039 65 65 65 99 99 99 F F F 2 040-059 68 68 68 102 102 102 F F F 3 060-079 66 66 66 100 100 100 F F F 4 080-099 65 65 65 99 99 99 F F F 5 100-119 62 62 62 95 95 95 F F F 6 120-139 69 69 69 105 105 105 F F F 7 140-159 60 60 60 91 91 91 F F F 8 160-179 66 66 66 100 100 100 F F F 9 180-199 68 68 68 102 102 102 F F F 10 200-219 68 68 68 103 103 103 F F F 11 220-239 68 68 68 103 103 103 F F F 12 240-259 65 65 65 99 99 99 F F F 13 260-279 65 65 65 99 99 99 F F F 14 280-299 68 68 68 102 102 102 F F F 15 300-319 67 67 67 101 101 101 F F F 16 320-339 62 62 62 95 95 95 F F F 17 340-359 69 69 69 105 105 105 F F F 18 360-379 68 68 68 102 102 102 F F F 19 380-399 66 66 66 100 100 100 F F F 20 400-419 67 67 67 101 101 101 F F F 21 420-439 68 68 68 103 103 103 F F F 22 440-459 66 66 66 100 100 100 F F F 23 460-479 68 68 68 102 102 102 F F F 24 480-499 68 68 68 102 102 102 F F F 25 500-519 67 67 67 101 101 101 F F F 26 520-539 68 68 68 104 104 104 F F F 27 540-559 68 68 68 104 104 104 F F F 28 560-579 65 65 65 99 99 99 F F F 29 580-599 66 66 66 100 100 100 F F F Max gen: 69, Max flush: 105 Begin loop 35 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:53.274716869Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1460,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:52:53.317415871Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1460,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:53.317807361Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1460,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:53.349787172Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1460,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:53.350312Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1460} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"3818f0fc76fe787c6cc412af30fd59b930203885\",\n git_commit_timestamp: \"2024-11-01T16:08:30.000000000Z\",\n git_branch: \"renovate/hyper-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:53.350354325Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1460} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:53.350371127Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1460} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:53.350388979Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1460} {"msg":"Crucible 2087c9a2-c9a0-47ed-817c-cb59eb9260c9 has session id: 1b04237b-3863-4745-ba80-a0b4845b99b6","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:53.350746996Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1460,"session_id":"1b04237b-3863-4745-ba80-a0b4845b99b6"} {"msg":"Upstairs opts: Upstairs UUID: 2087c9a2-c9a0-47ed-817c-cb59eb9260c9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:53.350794852Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1460,"session_id":"1b04237b-3863-4745-ba80-a0b4845b99b6"} {"msg":"Crucible stats registered with UUID: 2087c9a2-c9a0-47ed-817c-cb59eb9260c9","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:53.350823846Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1460,"session_id":"1b04237b-3863-4745-ba80-a0b4845b99b6"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:53.381686382Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1460,"task":"crutest"} {"msg":"The guest has requested activation with gen:70","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:53.381717766Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1460} {"msg":"2087c9a2-c9a0-47ed-817c-cb59eb9260c9 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:53.381754881Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1460,"session_id":"1b04237b-3863-4745-ba80-a0b4845b99b6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:53.381771783Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1460,"client":"0","":"downstairs","session_id":"1b04237b-3863-4745-ba80-a0b4845b99b6"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:53.381801317Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1460,"client":"0","":"downstairs","session_id":"1b04237b-3863-4745-ba80-a0b4845b99b6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:53.381819279Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1460,"client":"1","":"downstairs","session_id":"1b04237b-3863-4745-ba80-a0b4845b99b6"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:53.381836261Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1460,"client":"1","":"downstairs","session_id":"1b04237b-3863-4745-ba80-a0b4845b99b6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:53.381854084Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1460,"client":"2","":"downstairs","session_id":"1b04237b-3863-4745-ba80-a0b4845b99b6"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:53.381884778Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1460,"client":"2","":"downstairs","session_id":"1b04237b-3863-4745-ba80-a0b4845b99b6"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:53.381916182Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1460,"":"io task","client":"2","":"downstairs","session_id":"1b04237b-3863-4745-ba80-a0b4845b99b6"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:53.381949596Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1460,"":"io task","client":"0","":"downstairs","session_id":"1b04237b-3863-4745-ba80-a0b4845b99b6"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:53.38198101Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1460,"":"io task","client":"1","":"downstairs","session_id":"1b04237b-3863-4745-ba80-a0b4845b99b6"} {"msg":"ds_connection connected from Ok(127.0.0.1:49376)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:53.382194637Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1460,"":"io task","client":"2","":"downstairs","session_id":"1b04237b-3863-4745-ba80-a0b4845b99b6"} {"msg":"ds_connection connected from Ok(127.0.0.1:62351)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:53.382236893Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1460,"":"io task","client":"1","":"downstairs","session_id":"1b04237b-3863-4745-ba80-a0b4845b99b6"} {"msg":"ds_connection connected from Ok(127.0.0.1:58370)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:53.382261196Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1460,"":"io task","client":"0","":"downstairs","session_id":"1b04237b-3863-4745-ba80-a0b4845b99b6"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:53.382745439Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1460,"client":"1","":"downstairs","session_id":"1b04237b-3863-4745-ba80-a0b4845b99b6"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:53.382773652Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1460,"client":"1","":"downstairs","session_id":"1b04237b-3863-4745-ba80-a0b4845b99b6"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:53.382799595Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1460,"client":"0","":"downstairs","session_id":"1b04237b-3863-4745-ba80-a0b4845b99b6"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:53.38283063Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1460,"client":"0","":"downstairs","session_id":"1b04237b-3863-4745-ba80-a0b4845b99b6"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:53.382857733Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1460,"client":"2","":"downstairs","session_id":"1b04237b-3863-4745-ba80-a0b4845b99b6"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:53.382887947Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1460,"client":"2","":"downstairs","session_id":"1b04237b-3863-4745-ba80-a0b4845b99b6"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:53.383435408Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1460,"client":"1","":"downstairs","session_id":"1b04237b-3863-4745-ba80-a0b4845b99b6"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:53.383464571Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1460,"client":"2","":"downstairs","session_id":"1b04237b-3863-4745-ba80-a0b4845b99b6"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:53.383479813Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1460,"client":"0","":"downstairs","session_id":"1b04237b-3863-4745-ba80-a0b4845b99b6"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:53.383988769Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1460,"client":"1","":"downstairs","session_id":"1b04237b-3863-4745-ba80-a0b4845b99b6"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:53.384016832Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1460,"client":"1","":"downstairs","session_id":"1b04237b-3863-4745-ba80-a0b4845b99b6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:53.38407231Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1460,"session_id":"1b04237b-3863-4745-ba80-a0b4845b99b6"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:53.384098813Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1460,"client":"0","":"downstairs","session_id":"1b04237b-3863-4745-ba80-a0b4845b99b6"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:53.384113875Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1460,"client":"0","":"downstairs","session_id":"1b04237b-3863-4745-ba80-a0b4845b99b6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:53.384127377Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1460,"session_id":"1b04237b-3863-4745-ba80-a0b4845b99b6"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:53.384139678Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1460,"client":"2","":"downstairs","session_id":"1b04237b-3863-4745-ba80-a0b4845b99b6"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:53.38415311Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1460,"client":"2","":"downstairs","session_id":"1b04237b-3863-4745-ba80-a0b4845b99b6"} {"msg":"[0]R flush_numbers[0..12]: [103, 99, 102, 100, 99, 95, 105, 91, 100, 102, 103, 103]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:53.384167182Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1460,"":"downstairs","":"downstairs","session_id":"1b04237b-3863-4745-ba80-a0b4845b99b6"} {"msg":"[0]R generation[0..12]: [68, 65, 68, 66, 65, 62, 69, 60, 66, 68, 68, 68]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:53.384185134Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1460,"":"downstairs","":"downstairs","session_id":"1b04237b-3863-4745-ba80-a0b4845b99b6"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:53.384200576Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1460,"":"downstairs","":"downstairs","session_id":"1b04237b-3863-4745-ba80-a0b4845b99b6"} {"msg":"[1]R flush_numbers[0..12]: [103, 99, 102, 100, 99, 95, 105, 91, 100, 102, 103, 103]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:53.384214378Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1460,"":"downstairs","":"downstairs","session_id":"1b04237b-3863-4745-ba80-a0b4845b99b6"} {"msg":"[1]R generation[0..12]: [68, 65, 68, 66, 65, 62, 69, 60, 66, 68, 68, 68]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:53.384227939Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1460,"":"downstairs","":"downstairs","session_id":"1b04237b-3863-4745-ba80-a0b4845b99b6"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:53.384241821Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1460,"":"downstairs","":"downstairs","session_id":"1b04237b-3863-4745-ba80-a0b4845b99b6"} {"msg":"[2]R flush_numbers[0..12]: [103, 99, 102, 100, 99, 95, 105, 91, 100, 102, 103, 103]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:53.384255803Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1460,"":"downstairs","":"downstairs","session_id":"1b04237b-3863-4745-ba80-a0b4845b99b6"} {"msg":"[2]R generation[0..12]: [68, 65, 68, 66, 65, 62, 69, 60, 66, 68, 68, 68]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:53.384269465Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1460,"":"downstairs","":"downstairs","session_id":"1b04237b-3863-4745-ba80-a0b4845b99b6"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:53.384283157Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1460,"":"downstairs","":"downstairs","session_id":"1b04237b-3863-4745-ba80-a0b4845b99b6"} {"msg":"Max found gen is 70","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:53.384296138Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1460,"":"downstairs","":"downstairs","session_id":"1b04237b-3863-4745-ba80-a0b4845b99b6"} {"msg":"Generation requested: 70 >= found:70","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:53.3843105Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1460,"":"downstairs","":"downstairs","session_id":"1b04237b-3863-4745-ba80-a0b4845b99b6"} {"msg":"Next flush: 106","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:53.384323592Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1460,"":"downstairs","":"downstairs","session_id":"1b04237b-3863-4745-ba80-a0b4845b99b6"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:53.384336624Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1460,"":"downstairs","":"downstairs","session_id":"1b04237b-3863-4745-ba80-a0b4845b99b6"} {"msg":"No downstairs reconciliation required","v":0,"name":"Wait for a query_work_queue command to finish before sending IO crucible","level":30,"time":"2024-11-01T16:52:53.384357846Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1460,"session_id":"1b04237b-3863-4745-ba80-a0b4845b99b6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:53.384370058Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1460,"client":"0","":"downstairs","session_id":"1b04237b-3863-4745-ba80-a0b4845b99b6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:53.38438503Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1460,"client":"1","":"downstairs","session_id":"1b04237b-3863-4745-ba80-a0b4845b99b6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:53.384398492Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1460,"client":"2","":"downstairs","session_id":"1b04237b-3863-4745-ba80-a0b4845b99b6"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:53.384412093Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1460,"session_id":"1b04237b-3863-4745-ba80-a0b4845b99b6"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:53.384424445Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1460,"session_id":"1b04237b-3863-4745-ba80-a0b4845b99b6"} {"msg":"2087c9a2-c9a0-47ed-817c-cb59eb9260c9 is now active with session: 1b04237b-3863-4745-ba80-a0b4845b99b6","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:53.384437246Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1460,"session_id":"1b04237b-3863-4745-ba80-a0b4845b99b6"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:53.384449298Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1460,"session_id":"1b04237b-3863-4745-ba80-a0b4845b99b6"} {"msg":"The guest has finished waiting for activation with:70","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:53.3844616Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1460} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 97 len 16384 data: 7 7 8 7 02/30 Write block 81 len 8192 data: 8 10 03/30 Write block 584 len 24576 data: 7 10 9 10 8 9 04/30 Read block 292 len 24576 05/30 Write block 347 len 28672 data: 5 5 5 4 5 7 8 06/30 Write block 91 len 32768 data: 9 8 10 9 6 7 8 8 07/30 Read block 359 len 40960 08/30 Flush 09/30 Write block 395 len 40960 data: 5 7 8 9 9 8 7 7 6 4 10/30 Write block 199 len 8192 data: 9 6 11/30 Read block 369 len 28672 12/30 Read block 277 len 36864 13/30 Read block 507 len 8192 14/30 Write block 281 len 8192 data: 6 6 15/30 Read block 459 len 24576 16/30 Flush 17/30 Write block 74 len 28672 data: 8 7 3 3 5 9 8 18/30 Read block 325 len 4096 19/30 Write block 249 len 24576 data: 10 11 9 8 8 6 20/30 Read block 187 len 12288 21/30 Read block 32 len 28672 22/30 Read block 153 len 16384 23/30 Read block 160 len 32768 24/30 Write block 12 len 24576 data: 5 5 6 6 6 6 25/30 Read block 403 len 16384 26/30 Read block 471 len 36864 27/30 Write block 317 len 12288 data: 8 8 8 28/30 Write block 552 len 40960 data: 6 7 7 8 8 8 10 11 7 6 29/30 Write block 549 len 28672 data: 7 5 8 7 8 8 9 30/30 Write block 553 len 28672 data: 9 9 10 9 9 11 12 ---------------------------------------------------------------- Crucible gen:70 GIO:true work queues: Upstairs:0 downstairs:14 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1016 Acked Write 7 Done Done Done false 1017 Acked Read 1 Done Done Done false 1018 Acked Write 6 Done Done Done false 1019 Acked Read 3 Done Done Done false 1020 Acked Read 7 Done Done Done false 1021 Acked Read 4 Done Done Done false 1022 Acked Read 8 Done Done Done false 1023 Acked Write 6 Done Done Done false 1024 Acked Read 4 Done Done Done false 1025 Acked Read 9 Done Done Done false 1026 Acked Write 3 Done Done Done false 1027 Acked Write 10 Sent Done Done false 1028 Acked Write 7 Sent Sent Sent false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 2 2 7 Done 11 12 12 35 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1015 1015 1015 Downstairs last five completed: 1015 1014 1013 1012 1011 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:52:53.408271537Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1460} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 68 68 68 103 103 103 T T T 1 020-039 65 65 65 99 99 99 F F F 2 040-059 68 68 68 102 102 102 F F F 3 060-079 66 66 66 100 100 100 T T T 4 080-099 70 70 70 106 106 106 T T T 5 100-119 70 70 70 106 106 106 F F F 6 120-139 69 69 69 105 105 105 F F F 7 140-159 60 60 60 91 91 91 F F F 8 160-179 66 66 66 100 100 100 F F F 9 180-199 70 70 70 107 107 107 F F F 10 200-219 70 70 70 107 107 107 F F F 11 220-239 68 68 68 103 103 103 F F F 12 240-259 65 65 65 99 99 99 T T T 13 260-279 65 65 65 99 99 99 F F F 14 280-299 70 70 70 107 107 107 F F F 15 300-319 67 67 67 101 101 101 T T T 16 320-339 62 62 62 95 95 95 F F F 17 340-359 70 70 70 106 106 106 F F F 18 360-379 68 68 68 102 102 102 F F F 19 380-399 70 70 70 107 107 107 F F F 20 400-419 70 70 70 107 107 107 F F F 21 420-439 68 68 68 103 103 103 F F F 22 440-459 66 66 66 100 100 100 F F F 23 460-479 68 68 68 102 102 102 F F F 24 480-499 68 68 68 102 102 102 F F F 25 500-519 67 67 67 101 101 101 F F F 26 520-539 68 68 68 104 104 104 F F F 27 540-559 68 68 68 104 104 104 T T T 28 560-579 65 65 65 99 99 99 T T T 29 580-599 70 70 70 106 106 106 F F F Max gen: 70, Max flush: 107 A Difference in extent metadata is expected here On loop 35 Verifying data now No change after verify Loop: 35 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 71 71 71 108 108 108 F F F 1 020-039 65 65 65 99 99 99 F F F 2 040-059 68 68 68 102 102 102 F F F 3 060-079 71 71 71 108 108 108 F F F 4 080-099 71 71 71 108 108 108 F F F 5 100-119 70 70 70 106 106 106 F F F 6 120-139 69 69 69 105 105 105 F F F 7 140-159 60 60 60 91 91 91 F F F 8 160-179 66 66 66 100 100 100 F F F 9 180-199 70 70 70 107 107 107 F F F 10 200-219 70 70 70 107 107 107 F F F 11 220-239 68 68 68 103 103 103 F F F 12 240-259 71 71 71 108 108 108 F F F 13 260-279 65 65 65 99 99 99 F F F 14 280-299 70 70 70 107 107 107 F F F 15 300-319 71 71 71 108 108 108 F F F 16 320-339 62 62 62 95 95 95 F F F 17 340-359 70 70 70 106 106 106 F F F 18 360-379 68 68 68 102 102 102 F F F 19 380-399 70 70 70 107 107 107 F F F 20 400-419 70 70 70 107 107 107 F F F 21 420-439 68 68 68 103 103 103 F F F 22 440-459 66 66 66 100 100 100 F F F 23 460-479 68 68 68 102 102 102 F F F 24 480-499 68 68 68 102 102 102 F F F 25 500-519 67 67 67 101 101 101 F F F 26 520-539 68 68 68 104 104 104 F F F 27 540-559 71 71 71 108 108 108 F F F 28 560-579 71 71 71 108 108 108 F F F 29 580-599 70 70 70 106 106 106 F F F Max gen: 71, Max flush: 108 Begin loop 36 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:57.767303938Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1468,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:57.81187496Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1468,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:57.812476118Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1468} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"3818f0fc76fe787c6cc412af30fd59b930203885\",\n git_commit_timestamp: \"2024-11-01T16:08:30.000000000Z\",\n git_branch: \"renovate/hyper-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:57.812518723Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1468} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:57.812535045Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1468} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:57.812549697Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1468} {"msg":"Crucible fe937df7-69ab-421d-9ef4-1837c9f3954d has session id: 7e710c3f-b201-4e35-a9ff-088fe54f4b29","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:57.813011097Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1468,"session_id":"7e710c3f-b201-4e35-a9ff-088fe54f4b29"} {"msg":"Upstairs opts: Upstairs UUID: fe937df7-69ab-421d-9ef4-1837c9f3954d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:57.813054102Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1468,"session_id":"7e710c3f-b201-4e35-a9ff-088fe54f4b29"} {"msg":"Crucible stats registered with UUID: fe937df7-69ab-421d-9ef4-1837c9f3954d","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:57.813078856Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1468,"session_id":"7e710c3f-b201-4e35-a9ff-088fe54f4b29"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:57.843841319Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1468,"task":"crutest"} {"msg":"The guest has requested activation with gen:72","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:57.843907168Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1468} {"msg":"fe937df7-69ab-421d-9ef4-1837c9f3954d active request set","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:57.843972896Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1468,"session_id":"7e710c3f-b201-4e35-a9ff-088fe54f4b29"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:57.844040875Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1468,"client":"0","":"downstairs","session_id":"7e710c3f-b201-4e35-a9ff-088fe54f4b29"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:57.844109184Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1468,"client":"0","":"downstairs","session_id":"7e710c3f-b201-4e35-a9ff-088fe54f4b29"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:57.844147088Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1468,"client":"1","":"downstairs","session_id":"7e710c3f-b201-4e35-a9ff-088fe54f4b29"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:57.844180343Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1468,"client":"1","":"downstairs","session_id":"7e710c3f-b201-4e35-a9ff-088fe54f4b29"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:57.844211667Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1468,"client":"2","":"downstairs","session_id":"7e710c3f-b201-4e35-a9ff-088fe54f4b29"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:57.844270504Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1468,"client":"2","":"downstairs","session_id":"7e710c3f-b201-4e35-a9ff-088fe54f4b29"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:57.844332532Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1468,"":"io task","client":"2","":"downstairs","session_id":"7e710c3f-b201-4e35-a9ff-088fe54f4b29"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:57.844397991Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1468,"":"io task","client":"0","":"downstairs","session_id":"7e710c3f-b201-4e35-a9ff-088fe54f4b29"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:57.84446881Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1468,"":"io task","client":"1","":"downstairs","session_id":"7e710c3f-b201-4e35-a9ff-088fe54f4b29"} {"msg":"ds_connection connected from Ok(127.0.0.1:40401)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:57.844538279Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1468,"":"io task","client":"2","":"downstairs","session_id":"7e710c3f-b201-4e35-a9ff-088fe54f4b29"} {"msg":"ds_connection connected from Ok(127.0.0.1:53597)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:57.844599477Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1468,"":"io task","client":"1","":"downstairs","session_id":"7e710c3f-b201-4e35-a9ff-088fe54f4b29"} {"msg":"ds_connection connected from Ok(127.0.0.1:43845)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:57.844830797Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1468,"":"io task","client":"0","":"downstairs","session_id":"7e710c3f-b201-4e35-a9ff-088fe54f4b29"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:57.845508584Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1468,"client":"0","":"downstairs","session_id":"7e710c3f-b201-4e35-a9ff-088fe54f4b29"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:57.845538098Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1468,"client":"0","":"downstairs","session_id":"7e710c3f-b201-4e35-a9ff-088fe54f4b29"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:57.845560191Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1468,"client":"1","":"downstairs","session_id":"7e710c3f-b201-4e35-a9ff-088fe54f4b29"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:57.845586484Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1468,"client":"1","":"downstairs","session_id":"7e710c3f-b201-4e35-a9ff-088fe54f4b29"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:57.845663194Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1468,"client":"2","":"downstairs","session_id":"7e710c3f-b201-4e35-a9ff-088fe54f4b29"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:57.845681376Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1468,"client":"2","":"downstairs","session_id":"7e710c3f-b201-4e35-a9ff-088fe54f4b29"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:57.84617701Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1468,"client":"0","":"downstairs","session_id":"7e710c3f-b201-4e35-a9ff-088fe54f4b29"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:57.846326909Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1468,"client":"1","":"downstairs","session_id":"7e710c3f-b201-4e35-a9ff-088fe54f4b29"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:57.846378326Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1468,"client":"2","":"downstairs","session_id":"7e710c3f-b201-4e35-a9ff-088fe54f4b29"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:57.846719Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1468,"client":"0","":"downstairs","session_id":"7e710c3f-b201-4e35-a9ff-088fe54f4b29"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:57.846744863Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1468,"client":"0","":"downstairs","session_id":"7e710c3f-b201-4e35-a9ff-088fe54f4b29"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:57.846763536Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1468,"session_id":"7e710c3f-b201-4e35-a9ff-088fe54f4b29"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:57.846965982Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1468,"client":"1","":"downstairs","session_id":"7e710c3f-b201-4e35-a9ff-088fe54f4b29"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:57.846987605Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1468,"client":"1","":"downstairs","session_id":"7e710c3f-b201-4e35-a9ff-088fe54f4b29"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:57.847002086Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1468,"session_id":"7e710c3f-b201-4e35-a9ff-088fe54f4b29"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:57.847014838Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1468,"client":"2","":"downstairs","session_id":"7e710c3f-b201-4e35-a9ff-088fe54f4b29"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:57.84702837Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1468,"client":"2","":"downstairs","session_id":"7e710c3f-b201-4e35-a9ff-088fe54f4b29"} {"msg":"[0]R flush_numbers[0..12]: [108, 99, 102, 108, 108, 106, 105, 91, 100, 107, 107, 103]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:57.847042341Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1468,"":"downstairs","":"downstairs","session_id":"7e710c3f-b201-4e35-a9ff-088fe54f4b29"} {"msg":"[0]R generation[0..12]: [71, 65, 68, 71, 71, 70, 69, 60, 66, 70, 70, 68]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:57.847056213Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1468,"":"downstairs","":"downstairs","session_id":"7e710c3f-b201-4e35-a9ff-088fe54f4b29"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:57.847073576Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1468,"":"downstairs","":"downstairs","session_id":"7e710c3f-b201-4e35-a9ff-088fe54f4b29"} {"msg":"[1]R flush_numbers[0..12]: [108, 99, 102, 108, 108, 106, 105, 91, 100, 107, 107, 103]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:57.847087878Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1468,"":"downstairs","":"downstairs","session_id":"7e710c3f-b201-4e35-a9ff-088fe54f4b29"} {"msg":"[1]R generation[0..12]: [71, 65, 68, 71, 71, 70, 69, 60, 66, 70, 70, 68]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:57.847101539Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1468,"":"downstairs","":"downstairs","session_id":"7e710c3f-b201-4e35-a9ff-088fe54f4b29"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:57.847115631Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1468,"":"downstairs","":"downstairs","session_id":"7e710c3f-b201-4e35-a9ff-088fe54f4b29"} {"msg":"[2]R flush_numbers[0..12]: [108, 99, 102, 108, 108, 106, 105, 91, 100, 107, 107, 103]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:57.847129433Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1468,"":"downstairs","":"downstairs","session_id":"7e710c3f-b201-4e35-a9ff-088fe54f4b29"} {"msg":"[2]R generation[0..12]: [71, 65, 68, 71, 71, 70, 69, 60, 66, 70, 70, 68]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:57.847143105Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1468,"":"downstairs","":"downstairs","session_id":"7e710c3f-b201-4e35-a9ff-088fe54f4b29"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:57.847156976Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1468,"":"downstairs","":"downstairs","session_id":"7e710c3f-b201-4e35-a9ff-088fe54f4b29"} {"msg":"Max found gen is 72","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:57.847169958Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1468,"":"downstairs","":"downstairs","session_id":"7e710c3f-b201-4e35-a9ff-088fe54f4b29"} {"msg":"Generation requested: 72 >= found:72","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:57.84718325Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1468,"":"downstairs","":"downstairs","session_id":"7e710c3f-b201-4e35-a9ff-088fe54f4b29"} {"msg":"Next flush: 109","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:57.847196472Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1468,"":"downstairs","":"downstairs","session_id":"7e710c3f-b201-4e35-a9ff-088fe54f4b29"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:57.847211073Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1468,"":"downstairs","":"downstairs","session_id":"7e710c3f-b201-4e35-a9ff-088fe54f4b29"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:57.847224505Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1468,"session_id":"7e710c3f-b201-4e35-a9ff-088fe54f4b29"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-01T16:52:57.847236426Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1468,"client":"0","":"downstairs","session_id":"7e710c3f-b201-4e35-a9ff-088fe54f4b29"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:57.8472589Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1468,"client":"1","":"downstairs","session_id":"7e710c3f-b201-4e35-a9ff-088fe54f4b29"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:57.847272381Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1468,"client":"2","":"downstairs","session_id":"7e710c3f-b201-4e35-a9ff-088fe54f4b29"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:57.847288093Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1468,"session_id":"7e710c3f-b201-4e35-a9ff-088fe54f4b29"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:57.847300715Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1468,"session_id":"7e710c3f-b201-4e35-a9ff-088fe54f4b29"} {"msg":"fe937df7-69ab-421d-9ef4-1837c9f3954d is now active with session: 7e710c3f-b201-4e35-a9ff-088fe54f4b29","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:57.847313547Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1468,"session_id":"7e710c3f-b201-4e35-a9ff-088fe54f4b29"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:57.847325518Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1468,"session_id":"7e710c3f-b201-4e35-a9ff-088fe54f4b29"} {"msg":"The guest has finished waiting for activation with:72","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:57.84733763Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1468} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 93 len 40960 data: 11 10 7 8 9 9 9 8 7 7 02/30 Read block 423 len 8192 03/30 Read block 541 len 8192 04/30 Write block 377 len 32768 data: 4 4 4 4 6 8 7 6 05/30 Write block 485 len 32768 data: 6 6 7 8 7 7 8 6 06/30 Read block 104 len 24576 07/30 Flush 08/30 Write block 185 len 36864 data: 7 8 8 9 8 9 7 7 8 09/30 Write block 406 len 12288 data: 4 4 5 10/30 Write block 9 len 36864 data: 4 3 5 6 6 7 7 7 7 11/30 Read block 556 len 36864 12/30 Read block 580 len 28672 13/30 Read block 577 len 24576 14/30 Flush 15/30 Write block 531 len 40960 data: 7 7 8 8 8 9 7 6 5 5 16/30 Write block 24 len 20480 data: 6 6 5 5 5 17/30 Write block 374 len 40960 data: 5 3 3 5 5 5 5 7 9 8 18/30 Write block 392 len 28672 data: 5 5 4 6 8 9 10 19/30 Read block 206 len 24576 20/30 Flush 21/30 Write block 2 len 12288 data: 5 5 5 22/30 Read block 417 len 20480 23/30 Write block 259 len 28672 data: 7 7 10 9 8 7 7 24/30 Write block 484 len 20480 data: 7 7 7 8 9 25/30 Read block 82 len 12288 26/30 Read block 352 len 4096 27/30 Read block 34 len 4096 28/30 Read block 0 len 40960 29/30 Read block 28 len 12288 30/30 Read block 116 len 12288 ---------------------------------------------------------------- Crucible gen:72 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1020 Acked Write 3 Done Done Done false 1021 Acked Read 5 Done Done Done false 1022 Acked Write 7 Done Done Done false 1023 Acked Write 5 Done Done Done false 1024 Acked Read 3 Done Done Done false 1025 Acked Read 1 Done Done Done false 1026 Acked Read 1 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Read 3 Done Done Done false 1029 Acked Read 3 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 10 10 10 30 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1019 1019 1019 Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:52:57.873042072Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1468} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 72 72 72 110 110 110 T T T 1 020-039 72 72 72 111 111 111 F F F 2 040-059 68 68 68 102 102 102 F F F 3 060-079 71 71 71 108 108 108 F F F 4 080-099 72 72 72 109 109 109 F F F 5 100-119 72 72 72 109 109 109 F F F 6 120-139 69 69 69 105 105 105 F F F 7 140-159 60 60 60 91 91 91 F F F 8 160-179 66 66 66 100 100 100 F F F 9 180-199 72 72 72 110 110 110 F F F 10 200-219 70 70 70 107 107 107 F F F 11 220-239 68 68 68 103 103 103 F F F 12 240-259 71 71 71 108 108 108 T T T 13 260-279 65 65 65 99 99 99 T T T 14 280-299 70 70 70 107 107 107 F F F 15 300-319 71 71 71 108 108 108 F F F 16 320-339 62 62 62 95 95 95 F F F 17 340-359 70 70 70 106 106 106 F F F 18 360-379 72 72 72 111 111 111 F F F 19 380-399 72 72 72 111 111 111 F F F 20 400-419 72 72 72 110 110 110 F F F 21 420-439 68 68 68 103 103 103 F F F 22 440-459 66 66 66 100 100 100 F F F 23 460-479 68 68 68 102 102 102 F F F 24 480-499 72 72 72 109 109 109 T T T 25 500-519 67 67 67 101 101 101 F F F 26 520-539 72 72 72 111 111 111 F F F 27 540-559 72 72 72 111 111 111 F F F 28 560-579 71 71 71 108 108 108 F F F 29 580-599 70 70 70 106 106 106 F F F Max gen: 72, Max flush: 111 A Difference in extent metadata is expected here On loop 36 Verifying data now No change after verify Loop: 36 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 73 73 73 112 112 112 F F F 1 020-039 72 72 72 111 111 111 F F F 2 040-059 68 68 68 102 102 102 F F F 3 060-079 71 71 71 108 108 108 F F F 4 080-099 72 72 72 109 109 109 F F F 5 100-119 72 72 72 109 109 109 F F F 6 120-139 69 69 69 105 105 105 F F F 7 140-159 60 60 60 91 91 91 F F F 8 160-179 66 66 66 100 100 100 F F F 9 180-199 72 72 72 110 110 110 F F F 10 200-219 70 70 70 107 107 107 F F F 11 220-239 68 68 68 103 103 103 F F F 12 240-259 73 73 73 112 112 112 F F F 13 260-279 73 73 73 112 112 112 F F F 14 280-299 70 70 70 107 107 107 F F F 15 300-319 71 71 71 108 108 108 F F F 16 320-339 62 62 62 95 95 95 F F F 17 340-359 70 70 70 106 106 106 F F F 18 360-379 72 72 72 111 111 111 F F F 19 380-399 72 72 72 111 111 111 F F F 20 400-419 72 72 72 110 110 110 F F F 21 420-439 68 68 68 103 103 103 F F F 22 440-459 66 66 66 100 100 100 F F F 23 460-479 68 68 68 102 102 102 F F F 24 480-499 73 73 73 112 112 112 F F F 25 500-519 67 67 67 101 101 101 F F F 26 520-539 72 72 72 111 111 111 F F F 27 540-559 72 72 72 111 111 111 F F F 28 560-579 71 71 71 108 108 108 F F F 29 580-599 70 70 70 106 106 106 F F F Max gen: 73, Max flush: 112 Begin loop 37 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:02.15770449Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1476,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:02.201730302Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1476,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:02.202358863Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1476} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"3818f0fc76fe787c6cc412af30fd59b930203885\",\n git_commit_timestamp: \"2024-11-01T16:08:30.000000000Z\",\n git_branch: \"renovate/hyper-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:02.202397218Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1476} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:02.20241216Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1476} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:02.202426682Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1476} {"msg":"Crucible 771f5e02-8a5e-4717-a602-1f94488efc94 has session id: fe1d0cc1-d36f-497b-ae37-124d90da8f6c","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:02.202907704Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1476,"session_id":"fe1d0cc1-d36f-497b-ae37-124d90da8f6c"} {"msg":"Upstairs opts: Upstairs UUID: 771f5e02-8a5e-4717-a602-1f94488efc94, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:02.2029511Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1476,"session_id":"fe1d0cc1-d36f-497b-ae37-124d90da8f6c"} {"msg":"Crucible stats registered with UUID: 771f5e02-8a5e-4717-a602-1f94488efc94","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:02.202979043Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1476,"session_id":"fe1d0cc1-d36f-497b-ae37-124d90da8f6c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:02.234460269Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1476,"task":"crutest"} {"msg":"The guest has requested activation with gen:74","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:02.234513416Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1476} {"msg":"771f5e02-8a5e-4717-a602-1f94488efc94 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:02.234602418Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1476,"session_id":"fe1d0cc1-d36f-497b-ae37-124d90da8f6c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:02.234654594Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1476,"client":"0","":"downstairs","session_id":"fe1d0cc1-d36f-497b-ae37-124d90da8f6c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:02.23469933Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1476,"client":"0","":"downstairs","session_id":"fe1d0cc1-d36f-497b-ae37-124d90da8f6c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:02.234729044Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1476,"client":"1","":"downstairs","session_id":"fe1d0cc1-d36f-497b-ae37-124d90da8f6c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:02.234755187Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1476,"client":"1","":"downstairs","session_id":"fe1d0cc1-d36f-497b-ae37-124d90da8f6c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:02.234785931Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1476,"client":"2","":"downstairs","session_id":"fe1d0cc1-d36f-497b-ae37-124d90da8f6c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:02.234813105Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1476,"client":"2","":"downstairs","session_id":"fe1d0cc1-d36f-497b-ae37-124d90da8f6c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:02.2348553Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1476,"":"io task","client":"2","":"downstairs","session_id":"fe1d0cc1-d36f-497b-ae37-124d90da8f6c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:02.234885744Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1476,"":"io task","client":"0","":"downstairs","session_id":"fe1d0cc1-d36f-497b-ae37-124d90da8f6c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:02.234912547Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1476,"":"io task","client":"1","":"downstairs","session_id":"fe1d0cc1-d36f-497b-ae37-124d90da8f6c"} {"msg":"ds_connection connected from Ok(127.0.0.1:63342)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:02.234965164Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1476,"":"io task","client":"2","":"downstairs","session_id":"fe1d0cc1-d36f-497b-ae37-124d90da8f6c"} {"msg":"ds_connection connected from Ok(127.0.0.1:63058)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:02.23508597Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1476,"":"io task","client":"0","":"downstairs","session_id":"fe1d0cc1-d36f-497b-ae37-124d90da8f6c"} {"msg":"ds_connection connected from Ok(127.0.0.1:64159)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:02.235363166Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1476,"":"io task","client":"1","":"downstairs","session_id":"fe1d0cc1-d36f-497b-ae37-124d90da8f6c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:02.23586082Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1476,"client":"0","":"downstairs","session_id":"fe1d0cc1-d36f-497b-ae37-124d90da8f6c"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:02.235899275Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1476,"client":"0","":"downstairs","session_id":"fe1d0cc1-d36f-497b-ae37-124d90da8f6c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:02.235929729Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1476,"client":"2","":"downstairs","session_id":"fe1d0cc1-d36f-497b-ae37-124d90da8f6c"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:02.235956222Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1476,"client":"2","":"downstairs","session_id":"fe1d0cc1-d36f-497b-ae37-124d90da8f6c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:02.235981705Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1476,"client":"1","":"downstairs","session_id":"fe1d0cc1-d36f-497b-ae37-124d90da8f6c"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:02.236008739Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1476,"client":"1","":"downstairs","session_id":"fe1d0cc1-d36f-497b-ae37-124d90da8f6c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:02.236516484Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1476,"client":"1","":"downstairs","session_id":"fe1d0cc1-d36f-497b-ae37-124d90da8f6c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:02.236556399Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1476,"client":"2","":"downstairs","session_id":"fe1d0cc1-d36f-497b-ae37-124d90da8f6c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:02.236587723Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1476,"client":"0","":"downstairs","session_id":"fe1d0cc1-d36f-497b-ae37-124d90da8f6c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:02.237126503Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1476,"client":"1","":"downstairs","session_id":"fe1d0cc1-d36f-497b-ae37-124d90da8f6c"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:02.237153506Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1476,"client":"1","":"downstairs","session_id":"fe1d0cc1-d36f-497b-ae37-124d90da8f6c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:02.237168668Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1476,"session_id":"fe1d0cc1-d36f-497b-ae37-124d90da8f6c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:02.237231306Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1476,"client":"0","":"downstairs","session_id":"fe1d0cc1-d36f-497b-ae37-124d90da8f6c"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:02.237265261Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1476,"client":"0","":"downstairs","session_id":"fe1d0cc1-d36f-497b-ae37-124d90da8f6c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:02.237281713Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1476,"session_id":"fe1d0cc1-d36f-497b-ae37-124d90da8f6c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:02.237294694Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1476,"client":"2","":"downstairs","session_id":"fe1d0cc1-d36f-497b-ae37-124d90da8f6c"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:02.237308936Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1476,"client":"2","":"downstairs","session_id":"fe1d0cc1-d36f-497b-ae37-124d90da8f6c"} {"msg":"[0]R flush_numbers[0..12]: [112, 111, 102, 108, 109, 109, 105, 91, 100, 110, 107, 103]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:02.237329909Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1476,"":"downstairs","":"downstairs","session_id":"fe1d0cc1-d36f-497b-ae37-124d90da8f6c"} {"msg":"[0]R generation[0..12]: [73, 72, 68, 71, 72, 72, 69, 60, 66, 72, 70, 68]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:02.237345051Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1476,"":"downstairs","":"downstairs","session_id":"fe1d0cc1-d36f-497b-ae37-124d90da8f6c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:02.237359093Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1476,"":"downstairs","":"downstairs","session_id":"fe1d0cc1-d36f-497b-ae37-124d90da8f6c"} {"msg":"[1]R flush_numbers[0..12]: [112, 111, 102, 108, 109, 109, 105, 91, 100, 110, 107, 103]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:02.237373815Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1476,"":"downstairs","":"downstairs","session_id":"fe1d0cc1-d36f-497b-ae37-124d90da8f6c"} {"msg":"[1]R generation[0..12]: [73, 72, 68, 71, 72, 72, 69, 60, 66, 72, 70, 68]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:02.237388537Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1476,"":"downstairs","":"downstairs","session_id":"fe1d0cc1-d36f-497b-ae37-124d90da8f6c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:02.237405929Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1476,"":"downstairs","":"downstairs","session_id":"fe1d0cc1-d36f-497b-ae37-124d90da8f6c"} {"msg":"[2]R flush_numbers[0..12]: [112, 111, 102, 108, 109, 109, 105, 91, 100, 110, 107, 103]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:02.237421041Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1476,"":"downstairs","":"downstairs","session_id":"fe1d0cc1-d36f-497b-ae37-124d90da8f6c"} {"msg":"[2]R generation[0..12]: [73, 72, 68, 71, 72, 72, 69, 60, 66, 72, 70, 68]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:02.237434803Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1476,"":"downstairs","":"downstairs","session_id":"fe1d0cc1-d36f-497b-ae37-124d90da8f6c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:02.237448804Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1476,"":"downstairs","":"downstairs","session_id":"fe1d0cc1-d36f-497b-ae37-124d90da8f6c"} {"msg":"Max found gen is 74","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:02.237462666Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1476,"":"downstairs","":"downstairs","session_id":"fe1d0cc1-d36f-497b-ae37-124d90da8f6c"} {"msg":"Generation requested: 74 >= found:74","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:02.237476258Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1476,"":"downstairs","":"downstairs","session_id":"fe1d0cc1-d36f-497b-ae37-124d90da8f6c"} {"msg":"Next flush: 113","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:02.23749013Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1476,"":"downstairs","":"downstairs","session_id":"fe1d0cc1-d36f-497b-ae37-124d90da8f6c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:02.237504021Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1476,"":"downstairs","":"downstairs","session_id":"fe1d0cc1-d36f-497b-ae37-124d90da8f6c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:02.237517413Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1476,"session_id":"fe1d0cc1-d36f-497b-ae37-124d90da8f6c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-01T16:53:02.237529495Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1476,"client":"0","":"downstairs","session_id":"fe1d0cc1-d36f-497b-ae37-124d90da8f6c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:02.237550367Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1476,"client":"1","":"downstairs","session_id":"fe1d0cc1-d36f-497b-ae37-124d90da8f6c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:02.237563769Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1476,"client":"2","":"downstairs","session_id":"fe1d0cc1-d36f-497b-ae37-124d90da8f6c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:02.237576891Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1476,"session_id":"fe1d0cc1-d36f-497b-ae37-124d90da8f6c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:02.237589832Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1476,"session_id":"fe1d0cc1-d36f-497b-ae37-124d90da8f6c"} {"msg":"771f5e02-8a5e-4717-a602-1f94488efc94 is now active with session: fe1d0cc1-d36f-497b-ae37-124d90da8f6c","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:02.237602814Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1476,"session_id":"fe1d0cc1-d36f-497b-ae37-124d90da8f6c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:02.237617036Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1476,"session_id":"fe1d0cc1-d36f-497b-ae37-124d90da8f6c"} {"msg":"The guest has finished waiting for activation with:74","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:02.237629558Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1476} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 511 len 20480 data: 8 8 5 4 3 02/30 Write block 47 len 24576 data: 6 5 8 6 5 5 03/30 Read block 309 len 24576 04/30 Write block 465 len 20480 data: 4 6 6 6 7 05/30 Write block 239 len 4096 data: 5 06/30 Write block 237 len 24576 data: 6 6 6 5 5 4 07/30 Read block 532 len 40960 08/30 Write block 534 len 40960 data: 9 9 10 8 7 6 6 5 3 5 09/30 Read block 463 len 4096 10/30 Write block 125 len 36864 data: 7 8 9 8 9 9 10 11 11 11/30 Write block 88 len 36864 data: 9 10 13 10 9 12 11 8 9 12/30 Read block 313 len 28672 13/30 Read block 331 len 28672 14/30 Read block 592 len 28672 15/30 Flush 16/30 Write block 207 len 8192 data: 11 9 17/30 Read block 260 len 28672 18/30 Read block 301 len 28672 19/30 Read block 570 len 28672 20/30 Write block 50 len 24576 data: 7 6 6 6 5 5 21/30 Read block 363 len 16384 22/30 Read block 145 len 8192 23/30 Write block 261 len 36864 data: 11 10 9 8 8 9 7 7 8 24/30 Write block 484 len 28672 data: 8 8 8 9 10 8 8 25/30 Write block 572 len 16384 data: 6 5 5 5 26/30 Read block 568 len 4096 27/30 Read block 120 len 12288 28/30 Read block 145 len 36864 29/30 Read block 571 len 12288 30/30 Write block 318 len 4096 data: 9 ---------------------------------------------------------------- Crucible gen:74 GIO:true work queues: Upstairs:0 downstairs:15 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1015 Acked Write 2 Done Done Done false 1016 Acked Read 7 Done Done Done false 1017 Acked Read 7 Done Done Done false 1018 Acked Read 7 Done Done Done false 1019 Acked Write 6 Done Done Done false 1020 Acked Read 4 Done Done Done false 1021 Acked Read 2 Done Done Done false 1022 Acked Write 9 Done Done Done false 1023 Acked Write 7 Done Done Done false 1024 Acked Write 4 Done Done Done false 1025 Acked Read 1 Done Done Done false 1026 Acked Read 3 Done Done Done false 1027 Acked Read 9 Done Done Done false 1028 Acked Read 3 Done Done Done false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 14 14 14 42 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1014 1014 1014 Downstairs last five completed: 1014 1013 1012 1011 1010 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:53:02.260173492Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1476} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 73 73 73 112 112 112 F F F 1 020-039 72 72 72 111 111 111 F F F 2 040-059 74 74 74 113 113 113 T T T 3 060-079 71 71 71 108 108 108 F F F 4 080-099 74 74 74 113 113 113 F F F 5 100-119 72 72 72 109 109 109 F F F 6 120-139 74 74 74 113 113 113 F F F 7 140-159 60 60 60 91 91 91 F F F 8 160-179 66 66 66 100 100 100 F F F 9 180-199 72 72 72 110 110 110 F F F 10 200-219 70 70 70 107 107 107 T T T 11 220-239 74 74 74 113 113 113 F F F 12 240-259 74 74 74 113 113 113 F F F 13 260-279 73 73 73 112 112 112 T T T 14 280-299 70 70 70 107 107 107 F F F 15 300-319 71 71 71 108 108 108 T T T 16 320-339 62 62 62 95 95 95 F F F 17 340-359 70 70 70 106 106 106 F F F 18 360-379 72 72 72 111 111 111 F F F 19 380-399 72 72 72 111 111 111 F F F 20 400-419 72 72 72 110 110 110 F F F 21 420-439 68 68 68 103 103 103 F F F 22 440-459 66 66 66 100 100 100 F F F 23 460-479 74 74 74 113 113 113 F F F 24 480-499 73 73 73 112 112 112 T T T 25 500-519 74 74 74 113 113 113 F F F 26 520-539 74 74 74 113 113 113 F F F 27 540-559 74 74 74 113 113 113 F F F 28 560-579 71 71 71 108 108 108 T T T 29 580-599 70 70 70 106 106 106 F F F Max gen: 74, Max flush: 113 A Difference in extent metadata is expected here On loop 37 Verifying data now No change after verify Loop: 37 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 73 73 73 112 112 112 F F F 1 020-039 72 72 72 111 111 111 F F F 2 040-059 75 75 75 114 114 114 F F F 3 060-079 71 71 71 108 108 108 F F F 4 080-099 74 74 74 113 113 113 F F F 5 100-119 72 72 72 109 109 109 F F F 6 120-139 74 74 74 113 113 113 F F F 7 140-159 60 60 60 91 91 91 F F F 8 160-179 66 66 66 100 100 100 F F F 9 180-199 72 72 72 110 110 110 F F F 10 200-219 75 75 75 114 114 114 F F F 11 220-239 74 74 74 113 113 113 F F F 12 240-259 74 74 74 113 113 113 F F F 13 260-279 75 75 75 114 114 114 F F F 14 280-299 70 70 70 107 107 107 F F F 15 300-319 75 75 75 114 114 114 F F F 16 320-339 62 62 62 95 95 95 F F F 17 340-359 70 70 70 106 106 106 F F F 18 360-379 72 72 72 111 111 111 F F F 19 380-399 72 72 72 111 111 111 F F F 20 400-419 72 72 72 110 110 110 F F F 21 420-439 68 68 68 103 103 103 F F F 22 440-459 66 66 66 100 100 100 F F F 23 460-479 74 74 74 113 113 113 F F F 24 480-499 75 75 75 114 114 114 F F F 25 500-519 74 74 74 113 113 113 F F F 26 520-539 74 74 74 113 113 113 F F F 27 540-559 74 74 74 113 113 113 F F F 28 560-579 75 75 75 114 114 114 F F F 29 580-599 70 70 70 106 106 106 F F F Max gen: 75, Max flush: 114 Begin loop 38 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:06.5761117Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1484,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:06.619739882Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1484,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:06.620394686Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1484} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"3818f0fc76fe787c6cc412af30fd59b930203885\",\n git_commit_timestamp: \"2024-11-01T16:08:30.000000000Z\",\n git_branch: \"renovate/hyper-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:06.620435071Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1484} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:06.620453404Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1484} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:06.620468435Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1484} {"msg":"Crucible 297c7cb1-6f7d-4aeb-8662-bad188111e95 has session id: 11fcec1b-1bf8-4d47-9ad0-34beff02406b","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:06.620902331Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1484,"session_id":"11fcec1b-1bf8-4d47-9ad0-34beff02406b"} {"msg":"Upstairs opts: Upstairs UUID: 297c7cb1-6f7d-4aeb-8662-bad188111e95, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:06.621031928Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1484,"session_id":"11fcec1b-1bf8-4d47-9ad0-34beff02406b"} {"msg":"Crucible stats registered with UUID: 297c7cb1-6f7d-4aeb-8662-bad188111e95","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:06.621069203Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1484,"session_id":"11fcec1b-1bf8-4d47-9ad0-34beff02406b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:06.652216986Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1484,"task":"crutest"} {"msg":"The guest has requested activation with gen:76","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:06.65224692Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1484} {"msg":"297c7cb1-6f7d-4aeb-8662-bad188111e95 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:06.652318059Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1484,"session_id":"11fcec1b-1bf8-4d47-9ad0-34beff02406b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:06.652340992Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1484,"client":"0","":"downstairs","session_id":"11fcec1b-1bf8-4d47-9ad0-34beff02406b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:06.652371216Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1484,"client":"0","":"downstairs","session_id":"11fcec1b-1bf8-4d47-9ad0-34beff02406b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:06.652389339Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1484,"client":"1","":"downstairs","session_id":"11fcec1b-1bf8-4d47-9ad0-34beff02406b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:06.652406541Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1484,"client":"1","":"downstairs","session_id":"11fcec1b-1bf8-4d47-9ad0-34beff02406b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:06.652423033Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1484,"client":"2","":"downstairs","session_id":"11fcec1b-1bf8-4d47-9ad0-34beff02406b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:06.652443155Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1484,"client":"2","":"downstairs","session_id":"11fcec1b-1bf8-4d47-9ad0-34beff02406b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:06.652460278Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1484,"":"io task","client":"2","":"downstairs","session_id":"11fcec1b-1bf8-4d47-9ad0-34beff02406b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:06.652479Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1484,"":"io task","client":"0","":"downstairs","session_id":"11fcec1b-1bf8-4d47-9ad0-34beff02406b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:06.652496422Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1484,"":"io task","client":"1","":"downstairs","session_id":"11fcec1b-1bf8-4d47-9ad0-34beff02406b"} {"msg":"ds_connection connected from Ok(127.0.0.1:48955)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:06.652528296Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1484,"":"io task","client":"2","":"downstairs","session_id":"11fcec1b-1bf8-4d47-9ad0-34beff02406b"} {"msg":"ds_connection connected from Ok(127.0.0.1:45179)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:06.65279093Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1484,"":"io task","client":"1","":"downstairs","session_id":"11fcec1b-1bf8-4d47-9ad0-34beff02406b"} {"msg":"ds_connection connected from Ok(127.0.0.1:43948)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:06.652831885Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1484,"":"io task","client":"0","":"downstairs","session_id":"11fcec1b-1bf8-4d47-9ad0-34beff02406b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:06.653294325Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1484,"client":"0","":"downstairs","session_id":"11fcec1b-1bf8-4d47-9ad0-34beff02406b"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:06.653322669Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1484,"client":"0","":"downstairs","session_id":"11fcec1b-1bf8-4d47-9ad0-34beff02406b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:06.653477819Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1484,"client":"1","":"downstairs","session_id":"11fcec1b-1bf8-4d47-9ad0-34beff02406b"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:06.653497701Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1484,"client":"1","":"downstairs","session_id":"11fcec1b-1bf8-4d47-9ad0-34beff02406b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:06.653573711Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1484,"client":"2","":"downstairs","session_id":"11fcec1b-1bf8-4d47-9ad0-34beff02406b"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:06.653593504Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1484,"client":"2","":"downstairs","session_id":"11fcec1b-1bf8-4d47-9ad0-34beff02406b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:06.653929027Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1484,"client":"0","":"downstairs","session_id":"11fcec1b-1bf8-4d47-9ad0-34beff02406b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:06.654192791Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1484,"client":"1","":"downstairs","session_id":"11fcec1b-1bf8-4d47-9ad0-34beff02406b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:06.654225055Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1484,"client":"2","":"downstairs","session_id":"11fcec1b-1bf8-4d47-9ad0-34beff02406b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:06.654472347Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1484,"client":"0","":"downstairs","session_id":"11fcec1b-1bf8-4d47-9ad0-34beff02406b"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:06.654501991Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1484,"client":"0","":"downstairs","session_id":"11fcec1b-1bf8-4d47-9ad0-34beff02406b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:06.654518403Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1484,"session_id":"11fcec1b-1bf8-4d47-9ad0-34beff02406b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:06.654829703Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1484,"client":"2","":"downstairs","session_id":"11fcec1b-1bf8-4d47-9ad0-34beff02406b"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:06.654854436Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1484,"client":"2","":"downstairs","session_id":"11fcec1b-1bf8-4d47-9ad0-34beff02406b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:06.654868668Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1484,"session_id":"11fcec1b-1bf8-4d47-9ad0-34beff02406b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:06.654894821Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1484,"client":"1","":"downstairs","session_id":"11fcec1b-1bf8-4d47-9ad0-34beff02406b"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:06.654909223Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1484,"client":"1","":"downstairs","session_id":"11fcec1b-1bf8-4d47-9ad0-34beff02406b"} {"msg":"[0]R flush_numbers[0..12]: [112, 111, 114, 108, 113, 109, 113, 91, 100, 110, 114, 113]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:06.654931926Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1484,"":"downstairs","":"downstairs","session_id":"11fcec1b-1bf8-4d47-9ad0-34beff02406b"} {"msg":"[0]R generation[0..12]: [73, 72, 75, 71, 74, 72, 74, 60, 66, 72, 75, 74]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:06.654946478Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1484,"":"downstairs","":"downstairs","session_id":"11fcec1b-1bf8-4d47-9ad0-34beff02406b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:06.6549606Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1484,"":"downstairs","":"downstairs","session_id":"11fcec1b-1bf8-4d47-9ad0-34beff02406b"} {"msg":"[1]R flush_numbers[0..12]: [112, 111, 114, 108, 113, 109, 113, 91, 100, 110, 114, 113]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:06.654974401Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1484,"":"downstairs","":"downstairs","session_id":"11fcec1b-1bf8-4d47-9ad0-34beff02406b"} {"msg":"[1]R generation[0..12]: [73, 72, 75, 71, 74, 72, 74, 60, 66, 72, 75, 74]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:06.654987983Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1484,"":"downstairs","":"downstairs","session_id":"11fcec1b-1bf8-4d47-9ad0-34beff02406b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:06.655017477Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1484,"":"downstairs","":"downstairs","session_id":"11fcec1b-1bf8-4d47-9ad0-34beff02406b"} {"msg":"[2]R flush_numbers[0..12]: [112, 111, 114, 108, 113, 109, 113, 91, 100, 110, 114, 113]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:06.6550375Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1484,"":"downstairs","":"downstairs","session_id":"11fcec1b-1bf8-4d47-9ad0-34beff02406b"} {"msg":"[2]R generation[0..12]: [73, 72, 75, 71, 74, 72, 74, 60, 66, 72, 75, 74]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:06.655055642Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1484,"":"downstairs","":"downstairs","session_id":"11fcec1b-1bf8-4d47-9ad0-34beff02406b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:06.655081545Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1484,"":"downstairs","":"downstairs","session_id":"11fcec1b-1bf8-4d47-9ad0-34beff02406b"} {"msg":"Max found gen is 76","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:06.655107759Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1484,"":"downstairs","":"downstairs","session_id":"11fcec1b-1bf8-4d47-9ad0-34beff02406b"} {"msg":"Generation requested: 76 >= found:76","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:06.655126581Z","hostname":"Wait for a query_work_queue command to finish before sending IO w-01JBMA2T0XD4W63AXFF321JBBP","pid":1484,"":"downstairs","":"downstairs","session_id":"11fcec1b-1bf8-4d47-9ad0-34beff02406b"} {"msg":"Next flush: 115","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:06.655149374Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1484,"":"downstairs","":"downstairs","session_id":"11fcec1b-1bf8-4d47-9ad0-34beff02406b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:06.655162626Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1484,"":"downstairs","":"downstairs","session_id":"11fcec1b-1bf8-4d47-9ad0-34beff02406b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:06.655176938Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1484,"session_id":"11fcec1b-1bf8-4d47-9ad0-34beff02406b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:06.655189209Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1484,"client":"0","":"downstairs","session_id":"11fcec1b-1bf8-4d47-9ad0-34beff02406b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:06.655202851Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1484,"client":"1","":"downstairs","session_id":"11fcec1b-1bf8-4d47-9ad0-34beff02406b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:06.655216103Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1484,"client":"2","":"downstairs","session_id":"11fcec1b-1bf8-4d47-9ad0-34beff02406b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:06.655230045Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1484,"session_id":"11fcec1b-1bf8-4d47-9ad0-34beff02406b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:06.655242596Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1484,"session_id":"11fcec1b-1bf8-4d47-9ad0-34beff02406b"} {"msg":"297c7cb1-6f7d-4aeb-8662-bad188111e95 is now active with session: 11fcec1b-1bf8-4d47-9ad0-34beff02406b","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:06.655255458Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1484,"session_id":"11fcec1b-1bf8-4d47-9ad0-34beff02406b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:06.65526853Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1484,"session_id":"11fcec1b-1bf8-4d47-9ad0-34beff02406b"} {"msg":"The guest has finished waiting for activation with:76","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:06.655283511Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1484} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 315 len 24576 data: 10 11 9 10 9 8 02/30 Flush 03/30 Write block 30 len 40960 data: 5 7 7 5 5 6 7 7 8 7 04/30 Read block 382 len 24576 05/30 Write block 313 len 8192 data: 9 8 06/30 Read block 112 len 28672 07/30 Write block 118 len 8192 data: 5 6 08/30 Write block 428 len 36864 data: 9 8 7 7 3 2 2 3 4 09/30 Write block 187 len 4096 data: 9 10/30 Read block 376 len 16384 11/30 Read block 540 len 24576 12/30 Flush 13/30 Write block 358 len 8192 data: 7 6 14/30 Read block 149 len 24576 15/30 Read block 378 len 36864 16/30 Read block 515 len 28672 17/30 Write block 32 len 40960 data: 8 6 6 7 8 8 9 8 6 7 18/30 Write block 221 len 12288 data: 8 8 11 19/30 Write block 33 len 40960 data: 7 7 8 9 9 10 9 7 8 9 20/30 Read block 476 len 28672 21/30 Write block 481 len 20480 data: 8 7 6 9 9 22/30 Write block 118 len 28672 data: 6 7 7 8 9 6 7 23/30 Read block 453 len 32768 24/30 Flush 25/30 Write block 4 len 36864 data: 6 6 7 6 6 5 4 6 7 26/30 Write block 489 len 4096 data: 9 27/30 Write block 203 len 40960 data: 8 9 10 11 12 10 8 9 11 9 28/30 Read block 295 len 20480 29/30 Write block 562 len 32768 data: 6 7 7 4 5 6 5 7 30/30 Read block 262 len 24576 ---------------------------------------------------------------- Crucible gen:76 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 9 Done Done Done false 1025 Acked Write 1 Done Done Done false 1026 Acked Write 10 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Write 8 Done Done Done false 1029 Acked Read 6 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 6 6 5 17 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:53:06.68064916Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1484} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 73 73 73 112 112 112 T T T 1 020-039 76 76 76 117 117 117 F F F 2 040-059 76 76 76 117 117 117 F F F 3 060-079 71 71 71 108 108 108 F F F 4 080-099 74 74 74 113 113 113 F F F 5 100-119 76 76 76 117 117 117 F F F 6 120-139 76 76 76 117 117 117 F F F 7 140-159 60 60 60 91 91 91 F F F 8 160-179 66 66 66 100 100 100 F F F 9 180-199 76 76 76 116 116 116 F F F 10 200-219 75 75 75 114 114 114 T T T 11 220-239 76 76 76 117 117 117 F F F 12 240-259 74 74 74 113 113 113 F F F 13 260-279 75 75 75 114 114 114 F F F 14 280-299 70 70 70 107 107 107 F F F 15 300-319 76 76 76 116 116 116 F F F 16 320-339 76 76 76 115 115 115 F F F 17 340-359 76 76 76 117 117 117 F F F 18 360-379 72 72 72 111 111 111 F F F 19 380-399 72 72 72 111 111 111 F F F 20 400-419 72 72 72 110 110 110 F F F 21 420-439 76 76 76 116 116 116 F F F 22 440-459 66 66 66 100 100 100 F F F 23 460-479 74 74 74 113 113 113 F F F 24 480-499 76 76 76 117 117 117 T T T 25 500-519 74 74 74 113 113 113 F F F 26 520-539 74 74 74 113 113 113 F F F 27 540-559 74 74 74 113 113 113 F F F 28 560-579 75 75 75 114 114 114 T T T 29 580-599 70 70 70 106 106 106 F F F Max gen: 76, Max flush: 117 A Difference in extent metadata is expected here On loop 38 Verifying data now No change after verify Loop: 38 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 77 77 77 118 118 118 F F F 1 020-039 76 76 76 117 117 117 F F F 2 040-059 76 76 76 117 117 117 F F F 3 060-079 71 71 71 108 108 108 F F F 4 080-099 74 74 74 113 113 113 F F F 5 100-119 76 76 76 117 117 117 F F F 6 120-139 76 76 76 117 117 117 F F F 7 140-159 60 60 60 91 91 91 F F F 8 160-179 66 66 66 100 100 100 F F F 9 180-199 76 76 76 116 116 116 F F F 10 200-219 77 77 77 118 118 118 F F F 11 220-239 76 76 76 117 117 117 F F F 12 240-259 74 74 74 113 113 113 F F F 13 260-279 75 75 75 114 114 114 F F F 14 280-299 70 70 70 107 107 107 F F F 15 300-319 76 76 76 116 116 116 F F F 16 320-339 76 76 76 115 115 115 F F F 17 340-359 76 76 76 117 117 117 F F F 18 360-379 72 72 72 111 111 111 F F F 19 380-399 72 72 72 111 111 111 F F F 20 400-419 72 72 72 110 110 110 F F F 21 420-439 76 76 76 116 116 116 F F F 22 440-459 66 66 66 100 100 100 F F F 23 460-479 74 74 74 113 113 113 F F F 24 480-499 77 77 77 118 118 118 F F F 25 500-519 74 74 74 113 113 113 F F F 26 520-539 74 74 74 113 113 113 F F F 27 540-559 74 74 74 113 113 113 F F F 28 560-579 77 77 77 118 118 118 F F F 29 580-599 70 70 70 106 106 106 F F F Max gen: 77, Max flush: 118 Begin loop 39 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:10.972561329Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1492,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:11.018351469Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1492,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:11.019113467Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1492} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"3818f0fc76fe787c6cc412af30fd59b930203885\",\n git_commit_timestamp: \"2024-11-01T16:08:30.000000000Z\",\n git_branch: \"renovate/hyper-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:11.019164204Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1492} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:11.019182887Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1492} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:11.019201009Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1492} {"msg":"Crucible abf945ba-f948-4df9-b34a-f52a7f2b420c has session id: 184f937a-d4c6-4a51-a368-d4f164e9ded9","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:11.019629704Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1492,"session_id":"184f937a-d4c6-4a51-a368-d4f164e9ded9"} {"msg":"Upstairs opts: Upstairs UUID: abf945ba-f948-4df9-b34a-f52a7f2b420c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:11.019667069Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1492,"session_id":"184f937a-d4c6-4a51-a368-d4f164e9ded9"} {"msg":"Crucible stats registered with UUID: abf945ba-f948-4df9-b34a-f52a7f2b420c","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:11.019683951Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1492,"session_id":"184f937a-d4c6-4a51-a368-d4f164e9ded9"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:11.050437634Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1492,"task":"crutest"} {"msg":"The guest has requested activation with gen:78","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:11.050480039Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1492} {"msg":"abf945ba-f948-4df9-b34a-f52a7f2b420c active request set","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:11.050547958Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1492,"session_id":"184f937a-d4c6-4a51-a368-d4f164e9ded9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:11.050575241Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1492,"client":"0","":"downstairs","session_id":"184f937a-d4c6-4a51-a368-d4f164e9ded9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:11.050600075Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1492,"client":"0","":"downstairs","session_id":"184f937a-d4c6-4a51-a368-d4f164e9ded9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:11.050615037Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1492,"client":"1","":"downstairs","session_id":"184f937a-d4c6-4a51-a368-d4f164e9ded9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:11.050628918Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1492,"client":"1","":"downstairs","session_id":"184f937a-d4c6-4a51-a368-d4f164e9ded9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:11.05064216Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1492,"client":"2","":"downstairs","session_id":"184f937a-d4c6-4a51-a368-d4f164e9ded9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:11.050655832Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1492,"client":"2","":"downstairs","session_id":"184f937a-d4c6-4a51-a368-d4f164e9ded9"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:11.050673054Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1492,"":"io task","client":"2","":"downstairs","session_id":"184f937a-d4c6-4a51-a368-d4f164e9ded9"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:11.050690446Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1492,"":"io task","client":"1","":"downstairs","session_id":"184f937a-d4c6-4a51-a368-d4f164e9ded9"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:11.050707638Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1492,"":"io task","client":"0","":"downstairs","session_id":"184f937a-d4c6-4a51-a368-d4f164e9ded9"} {"msg":"ds_connection connected from Ok(127.0.0.1:35704)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:11.050747454Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1492,"":"io task","client":"1","":"downstairs","session_id":"184f937a-d4c6-4a51-a368-d4f164e9ded9"} {"msg":"ds_connection connected from Ok(127.0.0.1:36088)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:11.050785728Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1492,"":"io task","client":"2","":"downstairs","session_id":"184f937a-d4c6-4a51-a368-d4f164e9ded9"} {"msg":"ds_connection connected from Ok(127.0.0.1:59235)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:11.050822003Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1492,"":"io task","client":"0","":"downstairs","session_id":"184f937a-d4c6-4a51-a368-d4f164e9ded9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:11.051594513Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1492,"client":"1","":"downstairs","session_id":"184f937a-d4c6-4a51-a368-d4f164e9ded9"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:11.051629167Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1492,"client":"1","":"downstairs","session_id":"184f937a-d4c6-4a51-a368-d4f164e9ded9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:11.051648019Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1492,"client":"0","":"downstairs","session_id":"184f937a-d4c6-4a51-a368-d4f164e9ded9"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:11.051672183Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1492,"client":"0","":"downstairs","session_id":"184f937a-d4c6-4a51-a368-d4f164e9ded9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:11.051699276Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1492,"client":"2","":"downstairs","session_id":"184f937a-d4c6-4a51-a368-d4f164e9ded9"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:11.05172504Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1492,"client":"2","":"downstairs","session_id":"184f937a-d4c6-4a51-a368-d4f164e9ded9"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:11.052335318Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1492,"client":"0","":"downstairs","session_id":"184f937a-d4c6-4a51-a368-d4f164e9ded9"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:11.052365302Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1492,"client":"1","":"downstairs","session_id":"184f937a-d4c6-4a51-a368-d4f164e9ded9"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:11.052381984Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1492,"client":"2","":"downstairs","session_id":"184f937a-d4c6-4a51-a368-d4f164e9ded9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:11.052923974Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1492,"client":"1","":"downstairs","session_id":"184f937a-d4c6-4a51-a368-d4f164e9ded9"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:11.052952067Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1492,"client":"1","":"downstairs","session_id":"184f937a-d4c6-4a51-a368-d4f164e9ded9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:11.05297333Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1492,"session_id":"184f937a-d4c6-4a51-a368-d4f164e9ded9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:11.052990932Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1492,"client":"2","":"downstairs","session_id":"184f937a-d4c6-4a51-a368-d4f164e9ded9"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:11.053014626Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1492,"client":"2","":"downstairs","session_id":"184f937a-d4c6-4a51-a368-d4f164e9ded9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:11.053032178Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1492,"session_id":"184f937a-d4c6-4a51-a368-d4f164e9ded9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:11.05304475Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1492,"client":"0","":"downstairs","session_id":"184f937a-d4c6-4a51-a368-d4f164e9ded9"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:11.053058391Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1492,"client":"0","":"downstairs","session_id":"184f937a-d4c6-4a51-a368-d4f164e9ded9"} {"msg":"[0]R flush_numbers[0..12]: [118, 117, 117, 108, 113, 117, 117, 91, 100, 116, 118, 117]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:11.053072533Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1492,"":"downstairs","":"downstairs","session_id":"184f937a-d4c6-4a51-a368-d4f164e9ded9"} {"msg":"[0]R generation[0..12]: [77, 76, 76, 71, 74, 76, 76, 60, 66, 76, 77, 76]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:11.053086415Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1492,"":"downstairs","":"downstairs","session_id":"184f937a-d4c6-4a51-a368-d4f164e9ded9"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:11.053100967Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1492,"":"downstairs","":"downstairs","session_id":"184f937a-d4c6-4a51-a368-d4f164e9ded9"} {"msg":"[1]R flush_numbers[0..12]: [118, 117, 117, 108, 113, 117, 117, 91, 100, 116, 118, 117]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:11.053114959Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1492,"":"downstairs","":"downstairs","session_id":"184f937a-d4c6-4a51-a368-d4f164e9ded9"} {"msg":"[1]R generation[0..12]: [77, 76, 76, 71, 74, 76, 76, 60, 66, 76, 77, 76]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:11.05312961Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1492,"":"downstairs","":"downstairs","session_id":"184f937a-d4c6-4a51-a368-d4f164e9ded9"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:11.053143592Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1492,"":"downstairs","":"downstairs","session_id":"184f937a-d4c6-4a51-a368-d4f164e9ded9"} {"msg":"[2]R flush_numbers[0..12]: [118, 117, 117, 108, 113, 117, 117, 91, 100, 116, 118, 117]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:11.053157424Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1492,"":"downstairs","":"downstairs","session_id":"184f937a-d4c6-4a51-a368-d4f164e9ded9"} {"msg":"[2]R generation[0..12]: [77, 76, 76, 71, 74, 76, 76, 60, 66, 76, 77, 76]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:11.053171016Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1492,"":"downstairs","":"downstairs","session_id":"184f937a-d4c6-4a51-a368-d4f164e9ded9"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-01T16:53:11.053184937Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1492,"":"downstairs","":"downstairs","session_id":"184f937a-d4c6-4a51-a368-d4f164e9ded9"} {"msg":"Max found gen is 78","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:11.053212441Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1492,"":"downstairs","":"downstairs","session_id":"184f937a-d4c6-4a51-a368-d4f164e9ded9"} {"msg":"Generation requested: 78 >= found:78","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:11.053225693Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1492,"":"downstairs","":"downstairs","session_id":"184f937a-d4c6-4a51-a368-d4f164e9ded9"} {"msg":"Next flush: 119","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:11.053238804Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1492,"":"downstairs","":"downstairs","session_id":"184f937a-d4c6-4a51-a368-d4f164e9ded9"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:11.053252006Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1492,"":"downstairs","":"downstairs","session_id":"184f937a-d4c6-4a51-a368-d4f164e9ded9"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:11.053265458Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1492,"session_id":"184f937a-d4c6-4a51-a368-d4f164e9ded9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:11.0532778Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1492,"client":"0","":"downstairs","session_id":"184f937a-d4c6-4a51-a368-d4f164e9ded9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:11.053291381Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1492,"client":"1","":"downstairs","session_id":"184f937a-d4c6-4a51-a368-d4f164e9ded9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:11.053304573Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1492,"client":"2","":"downstairs","session_id":"184f937a-d4c6-4a51-a368-d4f164e9ded9"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:11.053317815Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1492,"session_id":"184f937a-d4c6-4a51-a368-d4f164e9ded9"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:11.053330176Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1492,"session_id":"184f937a-d4c6-4a51-a368-d4f164e9ded9"} {"msg":"abf945ba-f948-4df9-b34a-f52a7f2b420c is now active with session: 184f937a-d4c6-4a51-a368-d4f164e9ded9","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:11.053343098Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1492,"session_id":"184f937a-d4c6-4a51-a368-d4f164e9ded9"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:11.05335527Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1492,"session_id":"184f937a-d4c6-4a51-a368-d4f164e9ded9"} {"msg":"The guest has finished waiting for activation with:78","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:11.053367761Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1492} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 551 len 8192 data: 9 8 02/30 Read block 130 len 16384 03/30 Read block 104 len 36864 04/30 Write block 487 len 20480 data: 10 11 10 9 9 05/30 Write block 19 len 28672 data: 7 5 3 4 6 7 7 06/30 Read block 91 len 36864 07/30 Write block 535 len 20480 data: 10 11 9 8 7 08/30 Read block 508 len 36864 09/30 Read block 105 len 36864 10/30 Read block 381 len 16384 11/30 Flush 12/30 Write block 223 len 32768 data: 12 11 10 8 6 4 6 6 13/30 Write block 535 len 4096 data: 11 14/30 Read block 377 len 24576 15/30 Read block 231 len 40960 16/30 Read block 110 len 20480 17/30 Read block 407 len 12288 18/30 Write block 587 len 16384 data: 11 9 10 8 19/30 Read block 470 len 4096 20/30 Write block 52 len 4096 data: 7 21/30 Read block 102 len 40960 22/30 Read block 490 len 16384 23/30 Write block 170 len 40960 data: 5 4 4 4 4 6 7 8 7 7 24/30 Write block 309 len 32768 data: 7 7 8 8 10 9 11 12 25/30 Read block 91 len 40960 26/30 Read block 337 len 4096 27/30 Write block 309 len 4096 data: 8 28/30 Read block 577 len 36864 29/30 Read block 91 len 32768 30/30 Write block 61 len 24576 data: 5 5 5 7 8 11 ---------------------------------------------------------------- Crucible gen:78 GIO:true work queues: Upstairs:0 downstairs:19 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1011 Acked Write 8 Done Done Done false 1012 Acked Write 1 Done Done Done false 1013 Acked Read 6 Done Done Done false 1014 Acked Read 10 Done Done Done false 1015 Acked Read 5 Done Done Done false 1016 Acked Read 3 Done Done Done false 1017 Acked Write 4 Done Done Done false 1018 Acked Read 1 Done Done Done false 1019 Acked Write 1 Done Done Done false 1020 Acked Read 10 Done Done Done false 1021 Acked Read 4 Done Done Done false 1022 Acked Write 10 Done Done Done false 1023 Acked Write 8 Done Done Done false 1024 Acked Read 10 Done Done Done false 1025 Acked Read 1 Done Done Done false 1026 Acked Write 1 Done Done Done false 1027 Acked Read 9 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 18 18 18 54 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1010 1010 1010 Downstairs last five completed: 1010 1009 1008 1007 1006 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:53:11.075057756Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1492} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 78 78 78 119 119 119 F F F 1 020-039 78 78 78 119 119 119 F F F 2 040-059 76 76 76 117 117 117 T T T 3 060-079 71 71 71 108 108 108 T T T 4 080-099 74 74 74 113 113 113 F F F 5 100-119 76 76 76 117 117 117 F F F 6 120-139 76 76 76 117 117 117 F F F 7 140-159 60 60 60 91 91 91 F F F 8 160-179 66 66 66 100 100 100 T T T 9 180-199 76 76 76 116 116 116 F F F 10 200-219 77 77 77 118 118 118 F F F 11 220-239 76 76 76 117 117 117 T T T 12 240-259 74 74 74 113 113 113 F F F 13 260-279 75 75 75 114 114 114 F F F 14 280-299 70 70 70 107 107 107 F F F 15 300-319 76 76 76 116 116 116 T T T 16 320-339 76 76 76 115 115 115 F F F 17 340-359 76 76 76 117 117 117 F F F 18 360-379 72 72 72 111 111 111 F F F 19 380-399 72 72 72 111 111 111 F F F 20 400-419 72 72 72 110 110 110 F F F 21 420-439 76 76 76 116 116 116 F F F 22 440-459 66 66 66 100 100 100 F F F 23 460-479 74 74 74 113 113 113 F F F 24 480-499 78 78 78 119 119 119 F F F 25 500-519 74 74 74 113 113 113 F F F 26 520-539 78 78 78 119 119 119 T T T 27 540-559 78 78 78 119 119 119 F F F 28 560-579 77 77 77 118 118 118 F F F 29 580-599 70 70 70 106 106 106 T T T Max gen: 78, Max flush: 119 A Difference in extent metadata is expected here On loop 39 Verifying data now No change after verify Loop: 39 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 78 78 78 119 119 119 F F F 1 020-039 78 78 78 119 119 119 F F F 2 040-059 79 79 79 120 120 120 F F F 3 060-079 79 79 79 120 120 120 F F F 4 080-099 74 74 74 113 113 113 F F F 5 100-119 76 76 76 117 117 117 F F F 6 120-139 76 76 76 117 117 117 F F F 7 140-159 60 60 60 91 91 91 F F F 8 160-179 79 79 79 120 120 120 F F F 9 180-199 76 76 76 116 116 116 F F F 10 200-219 77 77 77 118 118 118 F F F 11 220-239 79 79 79 120 120 120 F F F 12 240-259 74 74 74 113 113 113 F F F 13 260-279 75 75 75 114 114 114 F F F 14 280-299 70 70 70 107 107 107 F F F 15 300-319 79 79 79 120 120 120 F F F 16 320-339 76 76 76 115 115 115 F F F 17 340-359 76 76 76 117 117 117 F F F 18 360-379 72 72 72 111 111 111 F F F 19 380-399 72 72 72 111 111 111 F F F 20 400-419 72 72 72 110 110 110 F F F 21 420-439 76 76 76 116 116 116 F F F 22 440-459 66 66 66 100 100 100 F F F 23 460-479 74 74 74 113 113 113 F F F 24 480-499 78 78 78 119 119 119 F F F 25 500-519 74 74 74 113 113 113 F F F 26 520-539 79 79 79 120 120 120 F F F 27 540-559 78 78 78 119 119 119 F F F 28 560-579 77 77 77 118 118 118 F F F 29 580-599 79 79 79 120 120 120 F F F Max gen: 79, Max flush: 120 Begin loop 40 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.40454709Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1500,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:53:15.447427895Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1500,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.447825526Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1500,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.479733967Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1500,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.480275027Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1500} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"3818f0fc76fe787c6cc412af30fd59b930203885\",\n git_commit_timestamp: \"2024-11-01T16:08:30.000000000Z\",\n git_branch: \"renovate/hyper-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.480307001Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1500} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.480320793Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1500} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.480339615Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1500} {"msg":"Crucible 7b79fa44-5505-4ab0-a51d-7e860140b3e4 has session id: 220b1843-a5f4-45e0-b3ce-212dd3803b91","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.480809686Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1500,"session_id":"220b1843-a5f4-45e0-b3ce-212dd3803b91"} {"msg":"Upstairs opts: Upstairs UUID: 7b79fa44-5505-4ab0-a51d-7e860140b3e4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.480850261Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1500,"session_id":"220b1843-a5f4-45e0-b3ce-212dd3803b91"} {"msg":"Crucible stats registered with UUID: 7b79fa44-5505-4ab0-a51d-7e860140b3e4","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.480873614Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1500,"session_id":"220b1843-a5f4-45e0-b3ce-212dd3803b91"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.511668452Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1500,"task":"crutest"} {"msg":"The guest has requested activation with gen:80","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.511701846Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1500} {"msg":"7b79fa44-5505-4ab0-a51d-7e860140b3e4 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.5117274Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1500,"session_id":"220b1843-a5f4-45e0-b3ce-212dd3803b91"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.511745152Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1500,"client":"0","":"downstairs","session_id":"220b1843-a5f4-45e0-b3ce-212dd3803b91"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.511775386Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1500,"client":"0","":"downstairs","session_id":"220b1843-a5f4-45e0-b3ce-212dd3803b91"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.511792988Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1500,"client":"1","":"downstairs","session_id":"220b1843-a5f4-45e0-b3ce-212dd3803b91"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.51181003Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1500,"client":"1","":"downstairs","session_id":"220b1843-a5f4-45e0-b3ce-212dd3803b91"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.511835033Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1500,"client":"2","":"downstairs","session_id":"220b1843-a5f4-45e0-b3ce-212dd3803b91"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.511868398Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1500,"client":"2","":"downstairs","session_id":"220b1843-a5f4-45e0-b3ce-212dd3803b91"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.511899772Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1500,"":"io task","client":"2","":"downstairs","session_id":"220b1843-a5f4-45e0-b3ce-212dd3803b91"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.511934716Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1500,"":"io task","client":"0","":"downstairs","session_id":"220b1843-a5f4-45e0-b3ce-212dd3803b91"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.511972411Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1500,"":"io task","client":"1","":"downstairs","session_id":"220b1843-a5f4-45e0-b3ce-212dd3803b91"} {"msg":"ds_connection connected from Ok(127.0.0.1:38279)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.512008496Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1500,"":"io task","client":"0","":"downstairs","session_id":"220b1843-a5f4-45e0-b3ce-212dd3803b91"} {"msg":"ds_connection connected from Ok(127.0.0.1:62364)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.512170977Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1500,"":"io task","client":"2","":"downstairs","session_id":"220b1843-a5f4-45e0-b3ce-212dd3803b91"} {"msg":"ds_connection connected from Ok(127.0.0.1:56589)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.512213532Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1500,"":"io task","client":"1","":"downstairs","session_id":"220b1843-a5f4-45e0-b3ce-212dd3803b91"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.512699255Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1500,"client":"2","":"downstairs","session_id":"220b1843-a5f4-45e0-b3ce-212dd3803b91"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.512728969Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1500,"client":"2","":"downstairs","session_id":"220b1843-a5f4-45e0-b3ce-212dd3803b91"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.51274357Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1500,"client":"0","":"downstairs","session_id":"220b1843-a5f4-45e0-b3ce-212dd3803b91"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.512758232Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1500,"client":"0","":"downstairs","session_id":"220b1843-a5f4-45e0-b3ce-212dd3803b91"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.512771754Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1500,"client":"1","":"downstairs","session_id":"220b1843-a5f4-45e0-b3ce-212dd3803b91"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.512788726Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1500,"client":"1","":"downstairs","session_id":"220b1843-a5f4-45e0-b3ce-212dd3803b91"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.513395414Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1500,"client":"1","":"downstairs","session_id":"220b1843-a5f4-45e0-b3ce-212dd3803b91"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.513427328Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1500,"client":"0","":"downstairs","session_id":"220b1843-a5f4-45e0-b3ce-212dd3803b91"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.513473725Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1500,"client":"2","":"downstairs","session_id":"220b1843-a5f4-45e0-b3ce-212dd3803b91"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.514017455Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1500,"client":"1","":"downstairs","session_id":"220b1843-a5f4-45e0-b3ce-212dd3803b91"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.51405668Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1500,"client":"1","":"downstairs","session_id":"220b1843-a5f4-45e0-b3ce-212dd3803b91"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.514089974Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1500,"session_id":"220b1843-a5f4-45e0-b3ce-212dd3803b91"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.514112847Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1500,"client":"2","":"downstairs","session_id":"220b1843-a5f4-45e0-b3ce-212dd3803b91"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.51413607Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1500,"client":"2","":"downstairs","session_id":"220b1843-a5f4-45e0-b3ce-212dd3803b91"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.514159953Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1500,"session_id":"220b1843-a5f4-45e0-b3ce-212dd3803b91"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.514181806Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1500,"client":"0","":"downstairs","session_id":"220b1843-a5f4-45e0-b3ce-212dd3803b91"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.514204219Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1500,"client":"0","":"downstairs","session_id":"220b1843-a5f4-45e0-b3ce-212dd3803b91"} {"msg":"[0]R flush_numbers[0..12]: [119, 119, 120, 120, 113, 117, 117, 91, 120, 116, 118, 120]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.514227062Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1500,"":"downstairs","":"downstairs","session_id":"220b1843-a5f4-45e0-b3ce-212dd3803b91"} {"msg":"[0]R generation[0..12]: [78, 78, 79, 79, 74, 76, 76, 60, 79, 76, 77, 79]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.514250365Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1500,"":"downstairs","":"downstairs","session_id":"220b1843-a5f4-45e0-b3ce-212dd3803b91"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.514276528Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1500,"":"downstairs","":"downstairs","session_id":"220b1843-a5f4-45e0-b3ce-212dd3803b91"} {"msg":"[1]R flush_numbers[0..12]: [119, 119, 120, 120, 113, 117, 117, 91, 120, 116, 118, 120]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.514303631Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1500,"":"downstairs","":"downstairs","session_id":"220b1843-a5f4-45e0-b3ce-212dd3803b91"} {"msg":"[1]R generation[0..12]: [78, 78, 79, 79, 74, 76, 76, 60, 79, 76, 77, 79]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.514329095Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1500,"":"downstairs","":"downstairs","session_id":"220b1843-a5f4-45e0-b3ce-212dd3803b91"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.514357148Z","hostname":"Wait for a query_work_queue command to finish before sending IO w-01JBMA2T0XD4W63AXFF321JBBP","pid":1500,"":"downstairs","":"downstairs","session_id":"220b1843-a5f4-45e0-b3ce-212dd3803b91"} {"msg":"[2]R flush_numbers[0..12]: [119, 119, 120, 120, 113, 117, 117, 91, 120, 116, 118, 120]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.514395153Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1500,"":"downstairs","":"downstairs","session_id":"220b1843-a5f4-45e0-b3ce-212dd3803b91"} {"msg":"[2]R generation[0..12]: [78, 78, 79, 79, 74, 76, 76, 60, 79, 76, 77, 79]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.514424447Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1500,"":"downstairs","":"downstairs","session_id":"220b1843-a5f4-45e0-b3ce-212dd3803b91"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.514457131Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1500,"":"downstairs","":"downstairs","session_id":"220b1843-a5f4-45e0-b3ce-212dd3803b91"} {"msg":"Max found gen is 80","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.514485415Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1500,"":"downstairs","":"downstairs","session_id":"220b1843-a5f4-45e0-b3ce-212dd3803b91"} {"msg":"Generation requested: 80 >= found:80","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.514513819Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1500,"":"downstairs","":"downstairs","session_id":"220b1843-a5f4-45e0-b3ce-212dd3803b91"} {"msg":"Next flush: 121","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.514540572Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1500,"":"downstairs","":"downstairs","session_id":"220b1843-a5f4-45e0-b3ce-212dd3803b91"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.514566975Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1500,"":"downstairs","":"downstairs","session_id":"220b1843-a5f4-45e0-b3ce-212dd3803b91"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2024-11-01T16:53:15.514594409Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1500,"session_id":"220b1843-a5f4-45e0-b3ce-212dd3803b91"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.514624573Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1500,"client":"0","":"downstairs","session_id":"220b1843-a5f4-45e0-b3ce-212dd3803b91"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.514642685Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1500,"client":"1","":"downstairs","session_id":"220b1843-a5f4-45e0-b3ce-212dd3803b91"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.514656017Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1500,"client":"2","":"downstairs","session_id":"220b1843-a5f4-45e0-b3ce-212dd3803b91"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.514669479Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1500,"session_id":"220b1843-a5f4-45e0-b3ce-212dd3803b91"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.51468188Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1500,"session_id":"220b1843-a5f4-45e0-b3ce-212dd3803b91"} {"msg":"7b79fa44-5505-4ab0-a51d-7e860140b3e4 is now active with session: 220b1843-a5f4-45e0-b3ce-212dd3803b91","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.514694992Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1500,"session_id":"220b1843-a5f4-45e0-b3ce-212dd3803b91"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.514707163Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1500,"session_id":"220b1843-a5f4-45e0-b3ce-212dd3803b91"} {"msg":"The guest has finished waiting for activation with:80","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.514719415Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1500} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 430 len 16384 data: 8 8 4 3 02/30 Read block 290 len 40960 03/30 Read block 72 len 8192 04/30 Read block 270 len 16384 05/30 Read block 535 len 8192 06/30 Flush 07/30 Write block 97 len 40960 data: 10 10 10 9 8 8 6 6 6 7 08/30 Write block 276 len 32768 data: 10 7 6 7 6 7 7 4 09/30 Read block 106 len 12288 10/30 Write block 525 len 32768 data: 4 4 4 5 6 8 8 8 11/30 Write block 478 len 12288 data: 12 10 10 12/30 Flush 13/30 Flush 14/30 Flush 15/30 Write block 446 len 24576 data: 7 9 8 8 9 9 16/30 Write block 9 len 28672 data: 6 5 7 8 7 8 8 17/30 Write block 591 len 8192 data: 7 6 18/30 Write block 130 len 12288 data: 10 11 12 19/30 Flush 20/30 Write block 7 len 8192 data: 7 7 21/30 Write block 305 len 20480 data: 9 9 10 10 9 22/30 Read block 524 len 24576 23/30 Flush 24/30 Write block 466 len 12288 data: 7 7 7 25/30 Write block 598 len 8192 data: 4 2 26/30 Read block 280 len 32768 27/30 Write block 96 len 8192 data: 10 11 28/30 Write block 125 len 36864 data: 8 9 10 9 10 11 12 13 12 29/30 Read block 531 len 24576 30/30 Read block 420 len 32768 ---------------------------------------------------------------- Crucible gen:80 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 3 Done Done Done false 1024 Acked Write 2 Done Done Done false 1025 Acked Read 8 Done Done Done false 1026 Acked Write 2 Done Done Done false 1027 Acked Write 9 Done Done Done false 1028 Acked Read 6 Done Done Done false 1029 Acked Read 8 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 0 1 Done 6 7 7 20 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:53:15.542443557Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1500} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 80 80 80 126 126 126 F F F 1 020-039 78 78 78 119 119 119 F F F 2 040-059 79 79 79 120 120 120 F F F 3 060-079 79 79 79 120 120 120 F F F 4 080-099 80 80 80 122 122 122 T T T 5 100-119 80 80 80 122 122 122 F F F 6 120-139 80 80 80 125 125 125 T T T 7 140-159 60 60 60 91 91 91 F F F 8 160-179 79 79 79 120 120 120 F F F 9 180-199 76 76 76 116 116 116 F F F 10 200-219 77 77 77 118 118 118 F F F 11 220-239 79 79 79 120 120 120 F F F 12 240-259 74 74 74 113 113 113 F F F 13 260-279 80 80 80 122 122 122 F F F 14 280-299 80 80 80 122 122 122 F F F 15 300-319 80 80 80 126 126 126 F F F 16 320-339 76 76 76 115 115 115 F F F 17 340-359 76 76 76 117 117 117 F F F 18 360-379 72 72 72 111 111 111 F F F 19 380-399 72 72 72 111 111 111 F F F 20 400-419 72 72 72 110 110 110 F F F 21 420-439 80 80 80 121 121 121 F F F 22 440-459 80 80 80 125 125 125 F F F 23 460-479 80 80 80 122 122 122 T T T 24 480-499 80 80 80 122 122 122 F F F 25 500-519 74 74 74 113 113 113 F F F 26 520-539 80 80 80 122 122 122 F F F 27 540-559 78 78 78 119 119 119 F F F 28 560-579 77 77 77 118 118 118 F F F 29 580-599 80 80 80 125 125 125 T T T Max gen: 80, Max flush: 126 A Difference in extent metadata is expected here On loop 40 Verifying data now No change after verify Loop: 40 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 80 80 80 126 126 126 F F F 1 020-039 78 78 78 119 119 119 F F F 2 040-059 79 79 79 120 120 120 F F F 3 060-079 79 79 79 120 120 120 F F F 4 080-099 81 81 81 127 127 127 F F F 5 100-119 80 80 80 122 122 122 F F F 6 120-139 81 81 81 127 127 127 F F F 7 140-159 60 60 60 91 91 91 F F F 8 160-179 79 79 79 120 120 120 F F F 9 180-199 76 76 76 116 116 116 F F F 10 200-219 77 77 77 118 118 118 F F F 11 220-239 79 79 79 120 120 120 F F F 12 240-259 74 74 74 113 113 113 F F F 13 260-279 80 80 80 122 122 122 F F F 14 280-299 80 80 80 122 122 122 F F F 15 300-319 80 80 80 126 126 126 F F F 16 320-339 76 76 76 115 115 115 F F F 17 340-359 76 76 76 117 117 117 F F F 18 360-379 72 72 72 111 111 111 F F F 19 380-399 72 72 72 111 111 111 F F F 20 400-419 72 72 72 110 110 110 F F F 21 420-439 80 80 80 121 121 121 F F F 22 440-459 80 80 80 125 125 125 F F F 23 460-479 81 81 81 127 127 127 F F F 24 480-499 80 80 80 122 122 122 F F F 25 500-519 74 74 74 113 113 113 F F F 26 520-539 80 80 80 122 122 122 F F F 27 540-559 78 78 78 119 119 119 F F F 28 560-579 77 77 77 118 118 118 F F F 29 580-599 81 81 81 127 127 127 F F F Max gen: 81, Max flush: 127 Begin loop 41 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:19.859734337Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1508,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:19.903572555Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1508,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:19.904142919Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1508} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"3818f0fc76fe787c6cc412af30fd59b930203885\",\n git_commit_timestamp: \"2024-11-01T16:08:30.000000000Z\",\n git_branch: \"renovate/hyper-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:19.904183934Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1508} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:19.904198936Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1508} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:19.904213508Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1508} {"msg":"Crucible 2a99682a-ccee-45a4-9a5f-943b92716542 has session id: 687d8786-b17c-4f94-afac-c4a9df48b661","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:19.904612959Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1508,"session_id":"687d8786-b17c-4f94-afac-c4a9df48b661"} {"msg":"Upstairs opts: Upstairs UUID: 2a99682a-ccee-45a4-9a5f-943b92716542, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:19.904644533Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1508,"session_id":"687d8786-b17c-4f94-afac-c4a9df48b661"} {"msg":"Crucible stats registered with UUID: 2a99682a-ccee-45a4-9a5f-943b92716542","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:19.904663346Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1508,"session_id":"687d8786-b17c-4f94-afac-c4a9df48b661"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:19.935983671Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1508,"task":"crutest"} {"msg":"The guest has requested activation with gen:82","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:19.936010735Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1508} {"msg":"2a99682a-ccee-45a4-9a5f-943b92716542 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:19.936084074Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1508,"session_id":"687d8786-b17c-4f94-afac-c4a9df48b661"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:19.936105147Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1508,"client":"0","":"downstairs","session_id":"687d8786-b17c-4f94-afac-c4a9df48b661"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:19.93613302Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1508,"client":"0","":"downstairs","session_id":"687d8786-b17c-4f94-afac-c4a9df48b661"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:19.936150663Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1508,"client":"1","":"downstairs","session_id":"687d8786-b17c-4f94-afac-c4a9df48b661"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:19.936167345Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1508,"client":"1","":"downstairs","session_id":"687d8786-b17c-4f94-afac-c4a9df48b661"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:19.936185097Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1508,"client":"2","":"downstairs","session_id":"687d8786-b17c-4f94-afac-c4a9df48b661"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:19.936214701Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1508,"client":"2","":"downstairs","session_id":"687d8786-b17c-4f94-afac-c4a9df48b661"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:19.936244255Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1508,"":"io task","client":"2","":"downstairs","session_id":"687d8786-b17c-4f94-afac-c4a9df48b661"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:19.936274099Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1508,"":"io task","client":"0","":"downstairs","session_id":"687d8786-b17c-4f94-afac-c4a9df48b661"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:19.936305693Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1508,"":"io task","client":"1","":"downstairs","session_id":"687d8786-b17c-4f94-afac-c4a9df48b661"} {"msg":"ds_connection connected from Ok(127.0.0.1:42534)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:19.936337107Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1508,"":"io task","client":"2","":"downstairs","session_id":"687d8786-b17c-4f94-afac-c4a9df48b661"} {"msg":"ds_connection connected from Ok(127.0.0.1:64783)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:19.936368961Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1508,"":"io task","client":"1","":"downstairs","session_id":"687d8786-b17c-4f94-afac-c4a9df48b661"} {"msg":"ds_connection connected from Ok(127.0.0.1:59980)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:19.936577848Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1508,"":"io task","client":"0","":"downstairs","session_id":"687d8786-b17c-4f94-afac-c4a9df48b661"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:19.937095404Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1508,"client":"2","":"downstairs","session_id":"687d8786-b17c-4f94-afac-c4a9df48b661"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:19.937122018Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1508,"client":"2","":"downstairs","session_id":"687d8786-b17c-4f94-afac-c4a9df48b661"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:19.937255105Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1508,"client":"1","":"downstairs","session_id":"687d8786-b17c-4f94-afac-c4a9df48b661"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:19.937304201Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1508,"client":"1","":"downstairs","session_id":"687d8786-b17c-4f94-afac-c4a9df48b661"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:19.937321053Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1508,"client":"0","":"downstairs","session_id":"687d8786-b17c-4f94-afac-c4a9df48b661"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:19.937338096Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1508,"client":"0","":"downstairs","session_id":"687d8786-b17c-4f94-afac-c4a9df48b661"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:19.937773752Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1508,"client":"2","":"downstairs","session_id":"687d8786-b17c-4f94-afac-c4a9df48b661"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:19.93784084Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1508,"client":"0","":"downstairs","session_id":"687d8786-b17c-4f94-afac-c4a9df48b661"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:19.937883456Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1508,"client":"1","":"downstairs","session_id":"687d8786-b17c-4f94-afac-c4a9df48b661"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:19.938389851Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1508,"client":"0","":"downstairs","session_id":"687d8786-b17c-4f94-afac-c4a9df48b661"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:19.938415975Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1508,"client":"0","":"downstairs","session_id":"687d8786-b17c-4f94-afac-c4a9df48b661"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:19.938431067Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1508,"session_id":"687d8786-b17c-4f94-afac-c4a9df48b661"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:19.938497805Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1508,"client":"1","":"downstairs","session_id":"687d8786-b17c-4f94-afac-c4a9df48b661"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:19.938517008Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1508,"client":"1","":"downstairs","session_id":"687d8786-b17c-4f94-afac-c4a9df48b661"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:19.938530629Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1508,"session_id":"687d8786-b17c-4f94-afac-c4a9df48b661"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:19.938546261Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1508,"client":"2","":"downstairs","session_id":"687d8786-b17c-4f94-afac-c4a9df48b661"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:19.938559903Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1508,"client":"2","":"downstairs","session_id":"687d8786-b17c-4f94-afac-c4a9df48b661"} {"msg":"[0]R flush_numbers[0..12]: [126, 119, 120, 120, 127, 122, 127, 91, 120, 116, 118, 120]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:19.938573925Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1508,"":"downstairs","":"downstairs","session_id":"687d8786-b17c-4f94-afac-c4a9df48b661"} {"msg":"[0]R generation[0..12]: [80, 78, 79, 79, 81, 80, 81, 60, 79, 76, 77, 79]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:19.938588077Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1508,"":"downstairs","":"downstairs","session_id":"687d8786-b17c-4f94-afac-c4a9df48b661"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:19.938602169Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1508,"":"downstairs","":"downstairs","session_id":"687d8786-b17c-4f94-afac-c4a9df48b661"} {"msg":"[1]R flush_numbers[0..12]: [126, 119, 120, 120, 127, 122, 127, 91, 120, 116, 118, 120]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:19.93861616Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1508,"":"downstairs","":"downstairs","session_id":"687d8786-b17c-4f94-afac-c4a9df48b661"} {"msg":"[1]R generation[0..12]: [80, 78, 79, 79, 81, 80, 81, 60, 79, 76, 77, 79]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:19.938630092Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1508,"":"downstairs","":"downstairs","session_id":"687d8786-b17c-4f94-afac-c4a9df48b661"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:19.938644244Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1508,"":"downstairs","":"downstairs","session_id":"687d8786-b17c-4f94-afac-c4a9df48b661"} {"msg":"[2]R flush_numbers[0..12]: [126, 119, 120, 120, 127, 122, 127, 91, 120, 116, 118, 120]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:19.938658016Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1508,"":"downstairs","":"downstairs","session_id":"687d8786-b17c-4f94-afac-c4a9df48b661"} {"msg":"[2]R generation[0..12]: [80, 78, 79, 79, 81, 80, 81, 60, 79, 76, 77, 79]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:19.938671758Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1508,"":"downstairs","":"downstairs","session_id":"687d8786-b17c-4f94-afac-c4a9df48b661"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:19.938685699Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1508,"":"downstairs","":"downstairs","session_id":"687d8786-b17c-4f94-afac-c4a9df48b661"} {"msg":"Max found gen is 82","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:19.938698801Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1508,"":"downstairs","":"downstairs","session_id":"687d8786-b17c-4f94-afac-c4a9df48b661"} {"msg":"Generation requested: 82 >= found:82","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-01T16:53:19.938712233Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1508,"":"downstairs","":"downstairs","session_id":"687d8786-b17c-4f94-afac-c4a9df48b661"} {"msg":"Next flush: 128","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:19.938783042Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1508,"":"downstairs","":"downstairs","session_id":"687d8786-b17c-4f94-afac-c4a9df48b661"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:19.938801074Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1508,"":"downstairs","":"downstairs","session_id":"687d8786-b17c-4f94-afac-c4a9df48b661"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:19.938815086Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1508,"session_id":"687d8786-b17c-4f94-afac-c4a9df48b661"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:19.938827598Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1508,"client":"0","":"downstairs","session_id":"687d8786-b17c-4f94-afac-c4a9df48b661"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:19.938841409Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1508,"client":"1","":"downstairs","session_id":"687d8786-b17c-4f94-afac-c4a9df48b661"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:19.938854901Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1508,"client":"2","":"downstairs","session_id":"687d8786-b17c-4f94-afac-c4a9df48b661"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:19.938868393Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1508,"session_id":"687d8786-b17c-4f94-afac-c4a9df48b661"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:19.938880645Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1508,"session_id":"687d8786-b17c-4f94-afac-c4a9df48b661"} {"msg":"2a99682a-ccee-45a4-9a5f-943b92716542 is now active with session: 687d8786-b17c-4f94-afac-c4a9df48b661","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:19.938894166Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1508,"session_id":"687d8786-b17c-4f94-afac-c4a9df48b661"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:19.938906198Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1508,"session_id":"687d8786-b17c-4f94-afac-c4a9df48b661"} {"msg":"The guest has finished waiting for activation with:82","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:19.938918269Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1508} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 262 len 40960 data: 11 10 9 9 10 8 8 9 8 10 02/30 Write block 348 len 28672 data: 6 6 5 6 8 9 6 03/30 Flush 04/30 Write block 150 len 20480 data: 9 7 8 7 9 05/30 Flush 06/30 Write block 559 len 24576 data: 13 8 7 7 8 8 07/30 Read block 99 len 28672 08/30 Read block 426 len 24576 09/30 Read block 5 len 8192 10/30 Write block 218 len 28672 data: 12 8 8 9 9 13 12 11/30 Flush 12/30 Write block 578 len 8192 data: 8 8 13/30 Write block 10 len 24576 data: 6 8 9 8 9 9 14/30 Read block 432 len 16384 15/30 Read block 442 len 36864 16/30 Write block 246 len 36864 data: 9 11 10 11 12 10 9 9 7 17/30 Write block 133 len 8192 data: 13 9 18/30 Read block 326 len 12288 19/30 Write block 507 len 16384 data: 7 8 7 7 20/30 Read block 86 len 4096 21/30 Read block 580 len 4096 22/30 Write block 444 len 12288 data: 8 8 8 23/30 Write block 378 len 8192 data: 6 6 24/30 Read block 151 len 36864 25/30 Read block 102 len 40960 26/30 Write block 551 len 16384 data: 10 9 10 10 27/30 Read block 226 len 4096 28/30 Read block 509 len 12288 29/30 Write block 415 len 40960 data: 4 3 5 7 7 7 8 8 8 8 30/30 Write block 587 len 40960 data: 12 10 11 9 8 7 8 6 6 5 ---------------------------------------------------------------- Crucible gen:82 GIO:true work queues: Upstairs:0 downstairs:19 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1011 Acked Write 2 Done Done Done false 1012 Acked Write 6 Done Done Done false 1013 Acked Read 4 Done Done Done false 1014 Acked Read 9 Done Done Done false 1015 Acked Write 9 Done Done Done false 1016 Acked Write 2 Done Done Done false 1017 Acked Read 3 Done Done Done false 1018 Acked Write 4 Done Done Done false 1019 Acked Read 1 Done Done Done false 1020 Acked Read 1 Done Done Done false 1021 Acked Write 3 Done Done Done false 1022 Acked Write 2 Done Done Done false 1023 Acked Read 9 Done Done Done false 1024 Acked Read 10 Done Done Done false 1025 Acked Write 4 Done Done Done false 1026 Acked Read 1 Done Done Done false 1027 Acked Read 3 Done Done Done false 1028 Acked Write 10 Sent Sent Sent false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 17 17 17 51 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1010 1010 1010 Downstairs last five completed: 1010 1009 1008 1007 1006 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:53:19.963092894Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1508} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 80 80 80 126 126 126 T T T 1 020-039 78 78 78 119 119 119 F F F 2 040-059 79 79 79 120 120 120 F F F 3 060-079 79 79 79 120 120 120 F F F 4 080-099 81 81 81 127 127 127 F F F 5 100-119 80 80 80 122 122 122 F F F 6 120-139 81 81 81 127 127 127 T T T 7 140-159 82 82 82 129 129 129 F F F 8 160-179 79 79 79 120 120 120 F F F 9 180-199 76 76 76 116 116 116 F F F 10 200-219 82 82 82 130 130 130 F F F 11 220-239 82 82 82 130 130 130 F F F 12 240-259 74 74 74 113 113 113 T T T 13 260-279 82 82 82 128 128 128 F F F 14 280-299 80 80 80 122 122 122 F F F 15 300-319 80 80 80 126 126 126 F F F 16 320-339 76 76 76 115 115 115 F F F 17 340-359 82 82 82 128 128 128 F F F 18 360-379 72 72 72 111 111 111 T T T 19 380-399 72 72 72 111 111 111 F F F 20 400-419 72 72 72 110 110 110 T T T 21 420-439 80 80 80 121 121 121 T T T 22 440-459 80 80 80 125 125 125 T T T 23 460-479 81 81 81 127 127 127 F F F 24 480-499 80 80 80 122 122 122 F F F 25 500-519 74 74 74 113 113 113 T T T 26 520-539 80 80 80 122 122 122 F F F 27 540-559 82 82 82 130 130 130 T T T 28 560-579 82 82 82 130 130 130 T T T 29 580-599 81 81 81 127 127 127 T T T Max gen: 82, Max flush: 130 A Difference in extent metadata is expected here On loop 41 Verifying data now No change after verify Loop: 41 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 83 83 83 131 131 131 F F F 1 020-039 78 78 78 119 119 119 F F F 2 040-059 79 79 79 120 120 120 F F F 3 060-079 79 79 79 120 120 120 F F F 4 080-099 81 81 81 127 127 127 F F F 5 100-119 80 80 80 122 122 122 F F F 6 120-139 83 83 83 131 131 131 F F F 7 140-159 82 82 82 129 129 129 F F F 8 160-179 79 79 79 120 120 120 F F F 9 180-199 76 76 76 116 116 116 F F F 10 200-219 82 82 82 130 130 130 F F F 11 220-239 82 82 82 130 130 130 F F F 12 240-259 83 83 83 131 131 131 F F F 13 260-279 82 82 82 128 128 128 F F F 14 280-299 80 80 80 122 122 122 F F F 15 300-319 80 80 80 126 126 126 F F F 16 320-339 76 76 76 115 115 115 F F F 17 340-359 82 82 82 128 128 128 F F F 18 360-379 83 83 83 131 131 131 F F F 19 380-399 72 72 72 111 111 111 F F F 20 400-419 83 83 83 131 131 131 F F F 21 420-439 83 83 83 131 131 131 F F F 22 440-459 83 83 83 131 131 131 F F F 23 460-479 81 81 81 127 127 127 F F F 24 480-499 80 80 80 122 122 122 F F F 25 500-519 83 83 83 131 131 131 F F F 26 520-539 80 80 80 122 122 122 F F F 27 540-559 83 83 83 131 131 131 F F F 28 560-579 83 83 83 131 131 131 F F F 29 580-599 83 83 83 131 131 131 F F F Max gen: 83, Max flush: 131 Begin loop 42 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:24.35237186Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1516,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:53:24.39513494Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1516,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:24.395500037Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1516,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:24.427921204Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1516,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:24.428450673Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1516} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"3818f0fc76fe787c6cc412af30fd59b930203885\",\n git_commit_timestamp: \"2024-11-01T16:08:30.000000000Z\",\n git_branch: \"renovate/hyper-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:24.428496078Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1516} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:24.428513991Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1516} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:24.428532413Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1516} {"msg":"Crucible 817f5211-0ec0-4bb1-823c-c8e7400ab955 has session id: 8612057d-690d-4e8c-8040-7d3f6cb2eb38","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:24.428915192Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1516,"session_id":"8612057d-690d-4e8c-8040-7d3f6cb2eb38"} {"msg":"Upstairs opts: Upstairs UUID: 817f5211-0ec0-4bb1-823c-c8e7400ab955, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:24.428966559Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1516,"session_id":"8612057d-690d-4e8c-8040-7d3f6cb2eb38"} {"msg":"Crucible stats registered with UUID: 817f5211-0ec0-4bb1-823c-c8e7400ab955","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:24.428995493Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1516,"session_id":"8612057d-690d-4e8c-8040-7d3f6cb2eb38"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:24.45970793Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1516,"task":"crutest"} {"msg":"The guest has requested activation with gen:84","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:24.459738594Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1516} {"msg":"817f5211-0ec0-4bb1-823c-c8e7400ab955 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:24.459803792Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1516,"session_id":"8612057d-690d-4e8c-8040-7d3f6cb2eb38"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:24.459826185Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1516,"client":"0","":"downstairs","session_id":"8612057d-690d-4e8c-8040-7d3f6cb2eb38"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:24.459854019Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1516,"client":"0","":"downstairs","session_id":"8612057d-690d-4e8c-8040-7d3f6cb2eb38"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:24.459871351Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1516,"client":"1","":"downstairs","session_id":"8612057d-690d-4e8c-8040-7d3f6cb2eb38"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:24.459888173Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1516,"client":"1","":"downstairs","session_id":"8612057d-690d-4e8c-8040-7d3f6cb2eb38"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:24.459904945Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1516,"client":"2","":"downstairs","session_id":"8612057d-690d-4e8c-8040-7d3f6cb2eb38"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:24.459921628Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1516,"client":"2","":"downstairs","session_id":"8612057d-690d-4e8c-8040-7d3f6cb2eb38"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:24.459938079Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1516,"":"io task","client":"2","":"downstairs","session_id":"8612057d-690d-4e8c-8040-7d3f6cb2eb38"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:24.459955262Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1516,"":"io task","client":"0","":"downstairs","session_id":"8612057d-690d-4e8c-8040-7d3f6cb2eb38"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:24.459972344Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1516,"":"io task","client":"1","":"downstairs","session_id":"8612057d-690d-4e8c-8040-7d3f6cb2eb38"} {"msg":"ds_connection connected from Ok(127.0.0.1:35187)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:24.460082838Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1516,"":"io task","client":"0","":"downstairs","session_id":"8612057d-690d-4e8c-8040-7d3f6cb2eb38"} {"msg":"ds_connection connected from Ok(127.0.0.1:43317)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:24.460123194Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1516,"":"io task","client":"1","":"downstairs","session_id":"8612057d-690d-4e8c-8040-7d3f6cb2eb38"} {"msg":"ds_connection connected from Ok(127.0.0.1:63725)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:24.460443855Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1516,"":"io task","client":"2","":"downstairs","session_id":"8612057d-690d-4e8c-8040-7d3f6cb2eb38"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:24.460877321Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1516,"client":"1","":"downstairs","session_id":"8612057d-690d-4e8c-8040-7d3f6cb2eb38"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:24.460903924Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1516,"client":"1","":"downstairs","session_id":"8612057d-690d-4e8c-8040-7d3f6cb2eb38"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:24.461037711Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1516,"client":"0","":"downstairs","session_id":"8612057d-690d-4e8c-8040-7d3f6cb2eb38"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:24.461057084Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1516,"client":"0","":"downstairs","session_id":"8612057d-690d-4e8c-8040-7d3f6cb2eb38"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:24.461070716Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1516,"client":"2","":"downstairs","session_id":"8612057d-690d-4e8c-8040-7d3f6cb2eb38"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:24.461084037Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1516,"client":"2","":"downstairs","session_id":"8612057d-690d-4e8c-8040-7d3f6cb2eb38"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:24.461434122Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1516,"client":"1","":"downstairs","session_id":"8612057d-690d-4e8c-8040-7d3f6cb2eb38"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:24.461745592Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1516,"client":"0","":"downstairs","session_id":"8612057d-690d-4e8c-8040-7d3f6cb2eb38"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:24.461786738Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1516,"client":"2","":"downstairs","session_id":"8612057d-690d-4e8c-8040-7d3f6cb2eb38"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:24.461970881Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1516,"client":"1","":"downstairs","session_id":"8612057d-690d-4e8c-8040-7d3f6cb2eb38"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:24.461997065Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1516,"client":"1","":"downstairs","session_id":"8612057d-690d-4e8c-8040-7d3f6cb2eb38"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:24.462011947Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1516,"session_id":"8612057d-690d-4e8c-8040-7d3f6cb2eb38"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:24.462300964Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1516,"client":"0","":"downstairs","session_id":"8612057d-690d-4e8c-8040-7d3f6cb2eb38"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:24.462339899Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1516,"client":"0","":"downstairs","session_id":"8612057d-690d-4e8c-8040-7d3f6cb2eb38"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:24.462380794Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1516,"session_id":"8612057d-690d-4e8c-8040-7d3f6cb2eb38"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:24.462409998Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1516,"client":"2","":"downstairs","session_id":"8612057d-690d-4e8c-8040-7d3f6cb2eb38"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:24.462434981Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1516,"client":"2","":"downstairs","session_id":"8612057d-690d-4e8c-8040-7d3f6cb2eb38"} {"msg":"[0]R flush_numbers[0..12]: [131, 119, 120, 120, 127, 122, 131, 129, 120, 116, 130, 130]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:24.462459604Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1516,"":"downstairs","":"downstairs","session_id":"8612057d-690d-4e8c-8040-7d3f6cb2eb38"} {"msg":"[0]R generation[0..12]: [83, 78, 79, 79, 81, 80, 83, 82, 79, 76, 82, 82]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:24.462483707Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1516,"":"downstairs","":"downstairs","session_id":"8612057d-690d-4e8c-8040-7d3f6cb2eb38"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:24.462510191Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1516,"":"downstairs","":"downstairs","session_id":"8612057d-690d-4e8c-8040-7d3f6cb2eb38"} {"msg":"[1]R flush_numbers[0..12]: [131, 119, 120, 120, 127, 122, 131, 129, 120, 116, 130, 130]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:24.462535454Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1516,"":"downstairs","":"downstairs","session_id":"8612057d-690d-4e8c-8040-7d3f6cb2eb38"} {"msg":"[1]R generation[0..12]: [83, 78, 79, 79, 81, 80, 83, 82, 79, 76, 82, 82]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:24.462558177Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1516,"":"downstairs","":"downstairs","session_id":"8612057d-690d-4e8c-8040-7d3f6cb2eb38"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:24.46258182Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1516,"":"downstairs","":"downstairs","session_id":"8612057d-690d-4e8c-8040-7d3f6cb2eb38"} {"msg":"[2]R flush_numbers[0..12]: [131, 119, 120, 120, 127, 122, 131, 129, 120, 116, 130, 130]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:24.462606093Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1516,"":"downstairs","":"downstairs","session_id":"8612057d-690d-4e8c-8040-7d3f6cb2eb38"} {"msg":"[2]R generation[0..12]: [83, 78, 79, 79, 81, 80, 83, 82, 79, 76, 82, 82]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:24.462632597Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1516,"":"downstairs","":"downstairs","session_id":"8612057d-690d-4e8c-8040-7d3f6cb2eb38"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:24.46265982Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1516,"":"downstairs","":"downstairs","session_id":"8612057d-690d-4e8c-8040-7d3f6cb2eb38"} {"msg":"Max found gen is 84","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:24.462684653Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1516,"":"downstairs","":"downstairs","session_id":"8612057d-690d-4e8c-8040-7d3f6cb2eb38"} {"msg":"Generation requested: 84 >= found:84","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-01T16:53:24.462712217Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1516,"":"downstairs","":"downstairs","session_id":"8612057d-690d-4e8c-8040-7d3f6cb2eb38"} {"msg":"Next flush: 132","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:24.462743871Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1516,"":"downstairs","":"downstairs","session_id":"8612057d-690d-4e8c-8040-7d3f6cb2eb38"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:24.462771475Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1516,"":"downstairs","":"downstairs","session_id":"8612057d-690d-4e8c-8040-7d3f6cb2eb38"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:24.462799088Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1516,"session_id":"8612057d-690d-4e8c-8040-7d3f6cb2eb38"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:24.462823822Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1516,"client":"0","":"downstairs","session_id":"8612057d-690d-4e8c-8040-7d3f6cb2eb38"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:24.462850965Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1516,"client":"1","":"downstairs","session_id":"8612057d-690d-4e8c-8040-7d3f6cb2eb38"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:24.462878958Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1516,"client":"2","":"downstairs","session_id":"8612057d-690d-4e8c-8040-7d3f6cb2eb38"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:24.462903522Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1516,"session_id":"8612057d-690d-4e8c-8040-7d3f6cb2eb38"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:24.462917003Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1516,"session_id":"8612057d-690d-4e8c-8040-7d3f6cb2eb38"} {"msg":"817f5211-0ec0-4bb1-823c-c8e7400ab955 is now active with session: 8612057d-690d-4e8c-8040-7d3f6cb2eb38","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:24.462929945Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1516,"session_id":"8612057d-690d-4e8c-8040-7d3f6cb2eb38"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 30,"time":"2024-11-01T16:53:24.462946097Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1516,"session_id":"8612057d-690d-4e8c-8040-7d3f6cb2eb38"} {"msg":"The guest has finished waiting for activation with:84","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:24.462958849Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1516} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 121 len 24576 data: 9 10 7 8 9 10 02/30 Write block 292 len 32768 data: 7 7 9 7 8 7 6 6 03/30 Read block 484 len 24576 04/30 Read block 150 len 40960 05/30 Write block 581 len 16384 data: 7 6 7 8 06/30 Flush 07/30 Write block 106 len 24576 data: 8 6 5 6 6 6 08/30 Read block 185 len 8192 09/30 Write block 242 len 36864 data: 5 4 5 8 10 12 11 12 13 10/30 Read block 510 len 8192 11/30 Write block 269 len 12288 data: 10 9 11 12/30 Read block 6 len 36864 13/30 Write block 50 len 8192 data: 8 7 14/30 Read block 335 len 40960 15/30 Write block 344 len 36864 data: 7 6 6 6 7 7 6 7 9 16/30 Read block 56 len 8192 17/30 Read block 429 len 16384 18/30 Read block 309 len 28672 19/30 Read block 493 len 28672 20/30 Read block 495 len 40960 21/30 Read block 320 len 4096 22/30 Flush 23/30 Write block 251 len 32768 data: 11 10 10 8 5 5 4 6 24/30 Flush 25/30 Write block 575 len 20480 data: 6 5 7 9 9 26/30 Write block 438 len 12288 data: 5 6 6 27/30 Read block 456 len 4096 28/30 Write block 276 len 28672 data: 11 8 7 8 7 8 8 29/30 Read block 85 len 32768 30/30 Read block 576 len 40960 ---------------------------------------------------------------- Crucible gen:84 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 5 Done Done Done false 1025 Acked Write 3 Done Done Done false 1026 Acked Read 1 Done Done Done false 1027 Acked Write 7 Done Done Sent false 1028 Acked Read 8 Done Done Sent false 1029 Acked Read 10 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 3 3 Done 6 6 3 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:53:24.487517283Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1516} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 83 83 83 131 131 131 F F F 1 020-039 78 78 78 119 119 119 F F F 2 040-059 84 84 84 133 133 133 F F F 3 060-079 79 79 79 120 120 120 F F F 4 080-099 81 81 81 127 127 127 F F F 5 100-119 84 84 84 133 133 133 F F F 6 120-139 84 84 84 132 132 132 F F F 7 140-159 82 82 82 129 129 129 F F F 8 160-179 79 79 79 120 120 120 F F F 9 180-199 76 76 76 116 116 116 F F F 10 200-219 82 82 82 130 130 130 F F F 11 220-239 82 82 82 130 130 130 F F F 12 240-259 84 84 84 134 134 134 F F F 13 260-279 84 84 84 133 133 133 T T T 14 280-299 84 84 84 132 132 132 T T T 15 300-319 80 80 80 126 126 126 F F F 16 320-339 76 76 76 115 115 115 F F F 17 340-359 84 84 84 133 133 133 F F F 18 360-379 83 83 83 131 131 131 F F F 19 380-399 72 72 72 111 111 111 F F F 20 400-419 83 83 83 131 131 131 F F F 21 420-439 83 83 83 131 131 131 T T T 22 440-459 83 83 83 131 131 131 T T T 23 460-479 81 81 81 127 127 127 F F F 24 480-499 80 80 80 122 122 122 F F F 25 500-519 83 83 83 131 131 131 F F F 26 520-539 80 80 80 122 122 122 F F F 27 540-559 83 83 83 131 131 131 F F F 28 560-579 83 83 83 131 131 131 T T T 29 580-599 84 84 84 132 132 132 F F F Max gen: 84, Max flush: 134 A Difference in extent metadata is expected here On loop 42 Verifying data now No change after verify Loop: 42 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 83 83 83 131 131 131 F F F 1 020-039 78 78 78 119 119 119 F F F 2 040-059 84 84 84 133 133 133 F F F 3 060-079 79 79 79 120 120 120 F F F 4 080-099 81 81 81 127 127 127 F F F 5 100-119 84 84 84 133 133 133 F F F 6 120-139 84 84 84 132 132 132 F F F 7 140-159 82 82 82 129 129 129 F F F 8 160-179 79 79 79 120 120 120 F F F 9 180-199 76 76 76 116 116 116 F F F 10 200-219 82 82 82 130 130 130 F F F 11 220-239 82 82 82 130 130 130 F F F 12 240-259 84 84 84 134 134 134 F F F 13 260-279 85 85 85 135 135 135 F F F 14 280-299 85 85 85 135 135 135 F F F 15 300-319 80 80 80 126 126 126 F F F 16 320-339 76 76 76 115 115 115 F F F 17 340-359 84 84 84 133 133 133 F F F 18 360-379 83 83 83 131 131 131 F F F 19 380-399 72 72 72 111 111 111 F F F 20 400-419 83 83 83 131 131 131 F F F 21 420-439 85 85 85 135 135 135 F F F 22 440-459 85 85 85 135 135 135 F F F 23 460-479 81 81 81 127 127 127 F F F 24 480-499 80 80 80 122 122 122 F F F 25 500-519 83 83 83 131 131 131 F F F 26 520-539 80 80 80 122 122 122 F F F 27 540-559 83 83 83 131 131 131 F F F 28 560-579 85 85 85 135 135 135 F F F 29 580-599 84 84 84 132 132 132 F F F Max gen: 85, Max flush: 135 Begin loop 43 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:28.826202231Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1524,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:28.869649019Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1524,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:28.870213882Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1524} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"3818f0fc76fe787c6cc412af30fd59b930203885\",\n git_commit_timestamp: \"2024-11-01T16:08:30.000000000Z\",\n git_branch: \"renovate/hyper-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:28.870250407Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1524} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:28.870264809Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1524} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:28.87027885Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1524} {"msg":"Crucible 1a76058e-39a5-48f4-abbd-0bb8b763204c has session id: 91766f62-7e18-44c2-ba4d-c683f8929093","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:28.870706065Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1524,"session_id":"91766f62-7e18-44c2-ba4d-c683f8929093"} {"msg":"Upstairs opts: Upstairs UUID: 1a76058e-39a5-48f4-abbd-0bb8b763204c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:28.87074181Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1524,"session_id":"91766f62-7e18-44c2-ba4d-c683f8929093"} {"msg":"Crucible stats registered with UUID: 1a76058e-39a5-48f4-abbd-0bb8b763204c","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:28.870758462Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1524,"session_id":"91766f62-7e18-44c2-ba4d-c683f8929093"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:28.901666795Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1524,"task":"crutest"} {"msg":"The guest has requested activation with gen:86","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:28.901700019Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1524} {"msg":"1a76058e-39a5-48f4-abbd-0bb8b763204c active request set","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:28.901737664Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1524,"session_id":"91766f62-7e18-44c2-ba4d-c683f8929093"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:28.901754996Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1524,"client":"0","":"downstairs","session_id":"91766f62-7e18-44c2-ba4d-c683f8929093"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:28.90178514Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1524,"client":"0","":"downstairs","session_id":"91766f62-7e18-44c2-ba4d-c683f8929093"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:28.901803102Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1524,"client":"1","":"downstairs","session_id":"91766f62-7e18-44c2-ba4d-c683f8929093"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:28.901820234Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1524,"client":"1","":"downstairs","session_id":"91766f62-7e18-44c2-ba4d-c683f8929093"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:28.901836966Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1524,"client":"2","":"downstairs","session_id":"91766f62-7e18-44c2-ba4d-c683f8929093"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:28.90186048Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1524,"client":"2","":"downstairs","session_id":"91766f62-7e18-44c2-ba4d-c683f8929093"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:28.901891724Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1524,"":"io task","client":"2","":"downstairs","session_id":"91766f62-7e18-44c2-ba4d-c683f8929093"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:28.901924098Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1524,"":"io task","client":"0","":"downstairs","session_id":"91766f62-7e18-44c2-ba4d-c683f8929093"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:28.901958162Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1524,"":"io task","client":"1","":"downstairs","session_id":"91766f62-7e18-44c2-ba4d-c683f8929093"} {"msg":"ds_connection connected from Ok(127.0.0.1:36961)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:28.901990616Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1524,"":"io task","client":"2","":"downstairs","session_id":"91766f62-7e18-44c2-ba4d-c683f8929093"} {"msg":"ds_connection connected from Ok(127.0.0.1:59828)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:28.902023641Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1524,"":"io task","client":"1","":"downstairs","session_id":"91766f62-7e18-44c2-ba4d-c683f8929093"} {"msg":"ds_connection connected from Ok(127.0.0.1:36674)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:28.902153847Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1524,"":"io task","client":"0","":"downstairs","session_id":"91766f62-7e18-44c2-ba4d-c683f8929093"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:28.902692117Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1524,"client":"0","":"downstairs","session_id":"91766f62-7e18-44c2-ba4d-c683f8929093"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:28.90272081Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1524,"client":"0","":"downstairs","session_id":"91766f62-7e18-44c2-ba4d-c683f8929093"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:28.902755345Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1524,"client":"2","":"downstairs","session_id":"91766f62-7e18-44c2-ba4d-c683f8929093"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:28.902777868Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1524,"client":"2","":"downstairs","session_id":"91766f62-7e18-44c2-ba4d-c683f8929093"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:28.902800671Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1524,"client":"1","":"downstairs","session_id":"91766f62-7e18-44c2-ba4d-c683f8929093"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:28.902817993Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1524,"client":"1","":"downstairs","session_id":"91766f62-7e18-44c2-ba4d-c683f8929093"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:28.903351832Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1524,"client":"2","":"downstairs","session_id":"91766f62-7e18-44c2-ba4d-c683f8929093"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:28.903479198Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1524,"client":"1","":"downstairs","session_id":"91766f62-7e18-44c2-ba4d-c683f8929093"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:28.903553578Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1524,"client":"0","":"downstairs","session_id":"91766f62-7e18-44c2-ba4d-c683f8929093"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:28.903915034Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1524,"client":"2","":"downstairs","session_id":"91766f62-7e18-44c2-ba4d-c683f8929093"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:28.903944258Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1524,"client":"2","":"downstairs","session_id":"91766f62-7e18-44c2-ba4d-c683f8929093"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:28.90396115Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1524,"session_id":"91766f62-7e18-44c2-ba4d-c683f8929093"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:28.904029289Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1524,"client":"1","":"downstairs","session_id":"91766f62-7e18-44c2-ba4d-c683f8929093"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:28.904050842Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1524,"client":"1","":"downstairs","session_id":"91766f62-7e18-44c2-ba4d-c683f8929093"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:28.904064713Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1524,"session_id":"91766f62-7e18-44c2-ba4d-c683f8929093"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:28.904090217Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1524,"client":"0","":"downstairs","session_id":"91766f62-7e18-44c2-ba4d-c683f8929093"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:28.904105309Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1524,"client":"0","":"downstairs","session_id":"91766f62-7e18-44c2-ba4d-c683f8929093"} {"msg":"[0]R flush_numbers[0..12]: [131, 119, 133, 120, 127, 133, 132, 129, 120, 116, 130, 130]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:28.904126952Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1524,"":"downstairs","":"downstairs","session_id":"91766f62-7e18-44c2-ba4d-c683f8929093"} {"msg":"[0]R generation[0..12]: [83, 78, 84, 79, 81, 84, 84, 82, 79, 76, 82, 82]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:28.904145074Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1524,"":"downstairs","":"downstairs","session_id":"91766f62-7e18-44c2-ba4d-c683f8929093"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:28.904160586Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1524,"":"downstairs","":"downstairs","session_id":"91766f62-7e18-44c2-ba4d-c683f8929093"} {"msg":"[1]R flush_numbers[0..12]: [131, 119, 133, 120, 127, 133, 132, 129, 120, 116, 130, 130]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:28.904175348Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1524,"":"downstairs","":"downstairs","session_id":"91766f62-7e18-44c2-ba4d-c683f8929093"} {"msg":"[1]R generation[0..12]: [83, 78, 84, 79, 81, 84, 84, 82, 79, 76, 82, 82]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:28.904189069Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1524,"":"downstairs","":"downstairs","session_id":"91766f62-7e18-44c2-ba4d-c683f8929093"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:28.904203722Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1524,"":"downstairs","":"downstairs","session_id":"91766f62-7e18-44c2-ba4d-c683f8929093"} {"msg":"[2]R flush_numbers[0..12]: [131, 119, 133, 120, 127, 133, 132, 129, 120, 116, 130, 130]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:28.904217593Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1524,"":"downstairs","":"downstairs","session_id":"91766f62-7e18-44c2-ba4d-c683f8929093"} {"msg":"[2]R generation[0..12]: [83, 78, 84, 79, 81, 84, 84, 82, 79, 76, 82, 82]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:28.904231115Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1524,"":"downstairs","":"downstairs","session_id":"91766f62-7e18-44c2-ba4d-c683f8929093"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:28.904245017Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1524,"":"downstairs","":"downstairs","session_id":"91766f62-7e18-44c2-ba4d-c683f8929093"} {"msg":"Max found gen is 86","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:28.904258069Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1524,"":"downstairs","":"downstairs","session_id":"91766f62-7e18-44c2-ba4d-c683f8929093"} {"msg":"Generation requested: 86 >= found:86","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:28.90427142Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1524,"":"downstairs","":"downstairs","session_id":"91766f62-7e18-44c2-ba4d-c683f8929093"} {"msg":"Next flush: 136","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:28.904285262Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1524,"":"downstairs","":"downstairs","session_id":"91766f62-7e18-44c2-ba4d-c683f8929093"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:28.904298414Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1524,"":"downstairs","":"downstairs","session_id":"91766f62-7e18-44c2-ba4d-c683f8929093"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:28.904312216Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1524,"session_id":"91766f62-7e18-44c2-ba4d-c683f8929093"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:28.904324377Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1524,"client":"0","":"downstairs","session_id":"91766f62-7e18-44c2-ba4d-c683f8929093"} {"msg":"Transition from WaitQuorum to Active","Wait for a query_work_queue command to finish before sending IO v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:28.90434689Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1524,"client":"1","":"downstairs","session_id":"91766f62-7e18-44c2-ba4d-c683f8929093"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:28.904361712Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1524,"client":"2","":"downstairs","session_id":"91766f62-7e18-44c2-ba4d-c683f8929093"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:28.904375473Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1524,"session_id":"91766f62-7e18-44c2-ba4d-c683f8929093"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:28.904387845Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1524,"session_id":"91766f62-7e18-44c2-ba4d-c683f8929093"} {"msg":"1a76058e-39a5-48f4-abbd-0bb8b763204c is now active with session: 91766f62-7e18-44c2-ba4d-c683f8929093","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:28.904400537Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1524,"session_id":"91766f62-7e18-44c2-ba4d-c683f8929093"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:28.904412498Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1524,"session_id":"91766f62-7e18-44c2-ba4d-c683f8929093"} {"msg":"The guest has finished waiting for activation with:86","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:28.90442464Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1524} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 54 len 16384 data: 6 6 6 6 02/30 Write block 351 len 8192 data: 8 10 03/30 Read block 242 len 16384 04/30 Read block 118 len 16384 05/30 Read block 98 len 20480 06/30 Read block 424 len 24576 07/30 Read block 372 len 16384 08/30 Read block 260 len 40960 09/30 Write block 150 len 12288 data: 10 8 9 10/30 Flush 11/30 Write block 95 len 24576 data: 9 11 12 11 11 10 12/30 Read block 309 len 16384 13/30 Read block 312 len 28672 14/30 Read block 253 len 8192 15/30 Read block 273 len 8192 16/30 Read block 360 len 12288 17/30 Write block 345 len 36864 data: 7 7 7 8 8 7 9 11 10 18/30 Read block 455 len 12288 19/30 Write block 108 len 40960 data: 6 7 7 7 6 5 5 4 4 5 20/30 Read block 537 len 28672 21/30 Write block 97 len 40960 data: 13 12 12 11 9 9 7 7 7 9 22/30 Read block 451 len 40960 23/30 Write block 110 len 24576 data: 8 8 7 6 6 5 24/30 Write block 332 len 36864 data: 5 6 6 8 6 7 6 4 5 25/30 Write block 391 len 8192 data: 5 6 26/30 Read block 88 len 24576 27/30 Read block 95 len 4096 28/30 Write block 406 len 12288 data: 5 5 6 29/30 Read block 95 len 36864 30/30 Write block 534 len 4096 data: 10 ---------------------------------------------------------------- Crucible gen:86 GIO:true work queues: Upstairs:0 downstairs:20 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1010 Acked Write 6 Done Done Done false 1011 Acked Read 4 Done Done Done false 1012 Acked Read 7 Done Done Done false 1013 Acked Read 2 Done Done Done false 1014 Acked Read 2 Done Done Done false 1015 Acked Read 3 Done Done Done false 1016 Acked Write 9 Done Done Done false 1017 Acked Read 3 Done Done Done false 1018 Acked Write 10 Done Done Done false 1019 Acked Read 7 Done Done Done false 1020 Acked Write 10 Done Done Done false 1021 Acked Read 10 Done Done Done false 1022 Acked Write 6 Done Done Done false 1023 Acked Write 9 Done Done Done false 1024 Acked Write 2 Done Done Sent false 1025 Acked Read 6 Done Done Sent false 1026 Acked Read 1 Done Done Sent false 1027 Acked Write 3 Done Done Sent false 1028 Acked Read 9 Done Done Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 6 8 Done 19 19 14 52 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1009 1009 1009 Downstairs last five completed: 1009 1008 1007 1006 1005 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:53:28.927128625Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1524} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 83 83 83 131 131 131 F F F 1 020-039 78 78 78 119 119 119 F F F 2 040-059 86 86 86 136 136 136 F F F 3 060-079 79 79 79 120 120 120 F F F 4 080-099 81 81 81 127 127 127 T T T 5 100-119 84 84 84 133 133 133 T T T 6 120-139 84 84 84 132 132 132 F F F 7 140-159 86 86 86 136 136 136 F F F 8 160-179 79 79 79 120 120 120 F F F 9 180-199 76 76 76 116 116 116 F F F 10 200-219 82 82 82 130 130 130 F F F 11 220-239 82 82 82 130 130 130 F F F 12 240-259 84 84 84 134 134 134 F F F 13 260-279 85 85 85 135 135 135 F F F 14 280-299 85 85 85 135 135 135 F F F 15 300-319 80 80 80 126 126 126 F F F 16 320-339 76 76 76 115 115 115 T T T 17 340-359 86 86 86 136 136 136 T T T 18 360-379 83 83 83 131 131 131 F F F 19 380-399 72 72 72 111 111 111 T T T 20 400-419 83 83 83 131 131 131 T T T 21 420-439 85 85 85 135 135 135 F F F 22 440-459 85 85 85 135 135 135 F F F 23 460-479 81 81 81 127 127 127 F F F 24 480-499 80 80 80 122 122 122 F F F 25 500-519 83 83 83 131 131 131 F F F 26 520-539 80 80 80 122 122 122 T T T 27 540-559 83 83 83 131 131 131 F F F 28 560-579 85 85 85 135 135 135 F F F 29 580-599 84 84 84 132 132 132 F F F Max gen: 86, Max flush: 136 A Difference in extent metadata is expected here On loop 43 Verifying data now No change after verify Loop: 43 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 83 83 83 131 131 131 F F F 1 020-039 78 78 78 119 119 119 F F F 2 040-059 86 86 86 136 136 136 F F F 3 060-079 79 79 79 120 120 120 F F F 4 080-099 87 87 87 137 137 137 F F F 5 100-119 87 87 87 137 137 137 F F F 6 120-139 84 84 84 132 132 132 F F F 7 140-159 86 86 86 136 136 136 F F F 8 160-179 79 79 79 120 120 120 F F F 9 180-199 76 76 76 116 116 116 F F F 10 200-219 82 82 82 130 130 130 F F F 11 220-239 82 82 82 130 130 130 F F F 12 240-259 84 84 84 134 134 134 F F F 13 260-279 85 85 85 135 135 135 F F F 14 280-299 85 85 85 135 135 135 F F F 15 300-319 80 80 80 126 126 126 F F F 16 320-339 87 87 87 137 137 137 F F F 17 340-359 87 87 87 137 137 137 F F F 18 360-379 83 83 83 131 131 131 F F F 19 380-399 87 87 87 137 137 137 F F F 20 400-419 87 87 87 137 137 137 F F F 21 420-439 85 85 85 135 135 135 F F F 22 440-459 85 85 85 135 135 135 F F F 23 460-479 81 81 81 127 127 127 F F F 24 480-499 80 80 80 122 122 122 F F F 25 500-519 83 83 83 131 131 131 F F F 26 520-539 87 87 87 137 137 137 F F F 27 540-559 83 83 83 131 131 131 F F F 28 560-579 85 85 85 135 135 135 F F F 29 580-599 84 84 84 132 132 132 F F F Max gen: 87, Max flush: 137 Begin loop 44 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:33.269039659Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1532,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:53:33.312003674Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1532,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:33.312397915Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1532,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:33.344380406Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1532,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:33.344901233Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1532} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"3818f0fc76fe787c6cc412af30fd59b930203885\",\n git_commit_timestamp: \"2024-11-01T16:08:30.000000000Z\",\n git_branch: \"renovate/hyper-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:33.344946359Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1532} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:33.344964351Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1532} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:33.344982254Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1532} {"msg":"Crucible 4042319a-57a0-4413-9b99-829898a5e63e has session id: ae515acc-5cd8-4c4d-a8f9-bc00f103f984","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:33.345464246Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1532,"session_id":"ae515acc-5cd8-4c4d-a8f9-bc00f103f984"} {"msg":"Upstairs opts: Upstairs UUID: 4042319a-57a0-4413-9b99-829898a5e63e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:33.345503551Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1532,"session_id":"ae515acc-5cd8-4c4d-a8f9-bc00f103f984"} {"msg":"Crucible stats registered with UUID: 4042319a-57a0-4413-9b99-829898a5e63e","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:33.345522593Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1532,"session_id":"ae515acc-5cd8-4c4d-a8f9-bc00f103f984"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:33.376500885Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1532,"task":"crutest"} {"msg":"The guest has requested activation with gen:88","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:33.376529669Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1532} {"msg":"4042319a-57a0-4413-9b99-829898a5e63e active request set","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:33.376587056Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1532,"session_id":"ae515acc-5cd8-4c4d-a8f9-bc00f103f984"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:33.37661987Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1532,"client":"0","":"downstairs","session_id":"ae515acc-5cd8-4c4d-a8f9-bc00f103f984"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:33.376650844Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1532,"client":"0","":"downstairs","session_id":"ae515acc-5cd8-4c4d-a8f9-bc00f103f984"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:33.376670036Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1532,"client":"1","":"downstairs","session_id":"ae515acc-5cd8-4c4d-a8f9-bc00f103f984"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:33.376690909Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1532,"client":"1","":"downstairs","session_id":"ae515acc-5cd8-4c4d-a8f9-bc00f103f984"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:33.376720623Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1532,"client":"2","":"downstairs","session_id":"ae515acc-5cd8-4c4d-a8f9-bc00f103f984"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:33.376752177Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1532,"client":"2","":"downstairs","session_id":"ae515acc-5cd8-4c4d-a8f9-bc00f103f984"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:33.376781281Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1532,"":"io task","client":"2","":"downstairs","session_id":"ae515acc-5cd8-4c4d-a8f9-bc00f103f984"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:33.376808134Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1532,"":"io task","client":"0","":"downstairs","session_id":"ae515acc-5cd8-4c4d-a8f9-bc00f103f984"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:33.376833388Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1532,"":"io task","client":"1","":"downstairs","session_id":"ae515acc-5cd8-4c4d-a8f9-bc00f103f984"} {"msg":"ds_connection connected from Ok(127.0.0.1:46851)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:33.376859481Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1532,"":"io task","client":"2","":"downstairs","session_id":"ae515acc-5cd8-4c4d-a8f9-bc00f103f984"} {"msg":"ds_connection connected from Ok(127.0.0.1:56842)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:33.376886294Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1532,"":"io task","client":"0","":"downstairs","session_id":"ae515acc-5cd8-4c4d-a8f9-bc00f103f984"} {"msg":"ds_connection connected from Ok(127.0.0.1:60421)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:33.377075799Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1532,"":"io task","client":"1","":"downstairs","session_id":"ae515acc-5cd8-4c4d-a8f9-bc00f103f984"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:33.377579954Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1532,"client":"2","":"downstairs","session_id":"ae515acc-5cd8-4c4d-a8f9-bc00f103f984"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:33.377615148Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1532,"client":"2","":"downstairs","session_id":"ae515acc-5cd8-4c4d-a8f9-bc00f103f984"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:33.377734864Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1532,"client":"1","":"downstairs","session_id":"ae515acc-5cd8-4c4d-a8f9-bc00f103f984"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:33.377779819Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1532,"client":"1","":"downstairs","session_id":"ae515acc-5cd8-4c4d-a8f9-bc00f103f984"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:33.37785823Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1532,"client":"0","":"downstairs","session_id":"ae515acc-5cd8-4c4d-a8f9-bc00f103f984"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:33.377883553Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1532,"client":"0","":"downstairs","session_id":"ae515acc-5cd8-4c4d-a8f9-bc00f103f984"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:33.378203384Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1532,"client":"2","":"downstairs","session_id":"ae515acc-5cd8-4c4d-a8f9-bc00f103f984"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:33.378349623Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1532,"client":"1","":"downstairs","session_id":"ae515acc-5cd8-4c4d-a8f9-bc00f103f984"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:33.378727602Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1532,"client":"0","":"downstairs","session_id":"ae515acc-5cd8-4c4d-a8f9-bc00f103f984"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:33.378855898Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1532,"client":"2","":"downstairs","session_id":"ae515acc-5cd8-4c4d-a8f9-bc00f103f984"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:33.378883982Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1532,"client":"2","":"downstairs","session_id":"ae515acc-5cd8-4c4d-a8f9-bc00f103f984"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:33.378899714Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1532,"session_id":"ae515acc-5cd8-4c4d-a8f9-bc00f103f984"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:33.378919306Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1532,"client":"1","":"downstairs","session_id":"ae515acc-5cd8-4c4d-a8f9-bc00f103f984"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:33.378933348Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1532,"client":"1","":"downstairs","session_id":"ae515acc-5cd8-4c4d-a8f9-bc00f103f984"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:33.37894705Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1532,"session_id":"ae515acc-5cd8-4c4d-a8f9-bc00f103f984"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:33.379160187Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1532,"client":"0","":"downstairs","session_id":"ae515acc-5cd8-4c4d-a8f9-bc00f103f984"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:33.3791845Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1532,"client":"0","":"downstairs","session_id":"ae515acc-5cd8-4c4d-a8f9-bc00f103f984"} {"msg":"[0]R flush_numbers[0..12]: [131, 119, 136, 120, 137, 137, 132, 136, 120, 116, 130, 130]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:33.379202733Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1532,"":"downstairs","":"downstairs","session_id":"ae515acc-5cd8-4c4d-a8f9-bc00f103f984"} {"msg":"[0]R generation[0..12]: [83, 78, 86, 79, 87, 87, 84, 86, 79, 76, 82, 82]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:33.379217435Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1532,"":"downstairs","":"downstairs","session_id":"ae515acc-5cd8-4c4d-a8f9-bc00f103f984"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:33.379231697Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1532,"":"downstairs","":"downstairs","session_id":"ae515acc-5cd8-4c4d-a8f9-bc00f103f984"} {"msg":"[1]R flush_numbers[0..12]: [131, 119, 136, 120, 137, 137, 132, 136, 120, 116, 130, 130]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:33.379245908Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1532,"":"downstairs","":"downstairs","session_id":"ae515acc-5cd8-4c4d-a8f9-bc00f103f984"} {"msg":"[1]R generation[0..12]: [83, 78, 86, 79, 87, 87, 84, 86, 79, 76, 82, 82]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:33.37925981Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1532,"":"downstairs","":"downstairs","session_id":"ae515acc-5cd8-4c4d-a8f9-bc00f103f984"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:33.379274112Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1532,"":"downstairs","":"downstairs","session_id":"ae515acc-5cd8-4c4d-a8f9-bc00f103f984"} {"msg":"[2]R flush_numbers[0..12]: [131, 119, 136, 120, 137, 137, 132, 136, 120, 116, 130, 130]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:33.379287934Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1532,"":"downstairs","":"downstairs","session_id":"ae515acc-5cd8-4c4d-a8f9-bc00f103f984"} {"msg":"[2]R generation[0..12]: [83, 78, 86, 79, 87, 87, 84, 86, 79, 76, 82, 82]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:33.379313867Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1532,"":"downstairs","":"downstairs","session_id":"ae515acc-5cd8-4c4d-a8f9-bc00f103f984"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:33.379328319Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1532,"":"downstairs","":"downstairs","session_id":"ae515acc-5cd8-4c4d-a8f9-bc00f103f984"} {"msg":"Max found gen is 88","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:33.379341561Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1532,"":"downstairs","":"downstairs","session_id":"ae515acc-5cd8-4c4d-a8f9-bc00f103f984"} {"msg":"Generation requested: 88 >= found:88","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:33.379354882Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1532,"":"downstairs","":"downstairs","session_id":"ae515acc-5cd8-4c4d-a8f9-bc00f103f984"} {"msg":"Next flush: 138","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:33.379367924Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1532,"":"downstairs","":"downstairs","session_id":"ae515acc-5cd8-4c4d-a8f9-bc00f103f984"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-01T16:53:33.379381046Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1532,"":"downstairs","":"downstairs","session_id":"ae515acc-5cd8-4c4d-a8f9-bc00f103f984"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:33.379398098Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1532,"session_id":"ae515acc-5cd8-4c4d-a8f9-bc00f103f984"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:33.37941258Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1532,"client":"0","":"downstairs","session_id":"ae515acc-5cd8-4c4d-a8f9-bc00f103f984"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:33.379426202Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1532,"client":"1","":"downstairs","session_id":"ae515acc-5cd8-4c4d-a8f9-bc00f103f984"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:33.379439513Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1532,"client":"2","":"downstairs","session_id":"ae515acc-5cd8-4c4d-a8f9-bc00f103f984"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:33.379452795Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1532,"session_id":"ae515acc-5cd8-4c4d-a8f9-bc00f103f984"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:33.379465097Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1532,"session_id":"ae515acc-5cd8-4c4d-a8f9-bc00f103f984"} {"msg":"4042319a-57a0-4413-9b99-829898a5e63e is now active with session: ae515acc-5cd8-4c4d-a8f9-bc00f103f984","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:33.379477998Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1532,"session_id":"ae515acc-5cd8-4c4d-a8f9-bc00f103f984"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:33.37948995Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1532,"session_id":"ae515acc-5cd8-4c4d-a8f9-bc00f103f984"} {"msg":"The guest has finished waiting for activation with:88","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:33.379502181Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1532} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 506 len 4096 data: 7 02/30 Write block 26 len 16384 data: 6 6 6 5 03/30 Read block 550 len 36864 04/30 Write block 346 len 20480 data: 8 8 9 9 8 05/30 Write block 28 len 20480 data: 7 6 6 8 9 06/30 Write block 12 len 20480 data: 10 9 10 10 8 07/30 Write block 552 len 4096 data: 10 08/30 Write block 113 len 28672 data: 7 7 6 5 6 7 8 09/30 Read block 586 len 8192 10/30 Write block 576 len 4096 data: 6 11/30 Read block 529 len 16384 12/30 Write block 275 len 16384 data: 11 12 9 8 13/30 Write block 83 len 36864 data: 10 9 10 8 9 10 11 14 11 14/30 Flush 15/30 Flush 16/30 Write block 2 len 36864 data: 6 6 7 7 8 8 8 7 7 17/30 Flush 18/30 Flush 19/30 Flush 20/30 Write block 138 len 40960 data: 7 5 5 5 5 3 4 5 4 5 21/30 Write block 558 len 32768 data: 12 14 9 8 8 9 9 5 22/30 Read block 105 len 24576 23/30 Write block 24 len 8192 data: 8 8 24/30 Write block 104 len 36864 data: 8 8 10 7 7 8 9 9 8 25/30 Read block 359 len 32768 26/30 Read block 261 len 28672 27/30 Read block 89 len 24576 28/30 Write block 575 len 4096 data: 7 29/30 Read block 150 len 12288 30/30 Write block 528 len 24576 data: 6 7 9 9 9 9 ---------------------------------------------------------------- Crucible gen:88 GIO:true work queues: Upstairs:0 downstairs:11 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1019 Acked Write 10 Done Done Done false 1020 Acked Write 8 Done Done Done false 1021 Acked Read 6 Done Done Done false 1022 Acked Write 2 Done Done Done false 1023 Acked Write 9 Done Done Done false 1024 Acked Read 8 Done Done Done false 1025 Acked Read 7 Done Done Done false 1026 Acked Read 6 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Read 3 Done Done Done false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 10 10 10 30 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1018 1018 1018 Downstairs last five completed: 1018 1017 1016 1015 1014 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:53:33.40230834Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1532} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 88 88 88 140 140 140 F F F 1 020-039 88 88 88 138 138 138 T T T 2 040-059 86 86 86 136 136 136 F F F 3 060-079 79 79 79 120 120 120 F F F 4 080-099 88 88 88 138 138 138 F F F 5 100-119 88 88 88 138 138 138 T T T 6 120-139 84 84 84 132 132 132 T T T 7 140-159 86 86 86 136 136 136 T T T 8 160-179 79 79 79 120 120 120 F F F 9 180-199 76 76 76 116 116 116 F F F 10 200-219 82 82 82 130 130 130 F F F 11 220-239 82 82 82 130 130 130 F F F 12 240-259 84 84 84 134 134 134 F F F 13 260-279 88 88 88 138 138 138 F F F 14 280-299 85 85 85 135 135 135 F F F 15 300-319 80 80 80 126 126 126 F F F 16 320-339 87 87 87 137 137 137 F F F 17 340-359 88 88 88 138 138 138 F F F 18 360-379 83 83 83 131 131 131 F F F 19 380-399 87 87 87 137 137 137 F F F 20 400-419 87 87 87 137 137 137 F F F 21 420-439 85 85 85 135 135 135 F F F 22 440-459 85 85 85 135 135 135 F F F 23 460-479 81 81 81 127 127 127 F F F 24 480-499 80 80 80 122 122 122 F F F 25 500-519 88 88 88 138 138 138 F F F 26 520-539 87 87 87 137 137 137 T T T 27 540-559 88 88 88 138 138 138 T T T 28 560-579 88 88 88 138 138 138 T T T 29 580-599 84 84 84 132 132 132 F F F Max gen: 88, Max flush: 140 A Difference in extent metadata is expected here On loop 44 Verifying data now No change after verify Loop: 44 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 88 88 88 140 140 140 F F F 1 020-039 89 89 89 141 141 141 F F F 2 040-059 86 86 86 136 136 136 F F F 3 060-079 79 79 79 120 120 120 F F F 4 080-099 88 88 88 138 138 138 F F F 5 100-119 89 89 89 141 141 141 F F F 6 120-139 89 89 89 141 141 141 F F F 7 140-159 89 89 89 141 141 141 F F F 8 160-179 79 79 79 120 120 120 F F F 9 180-199 76 76 76 116 116 116 F F F 10 200-219 82 82 82 130 130 130 F F F 11 220-239 82 82 82 130 130 130 F F F 12 240-259 84 84 84 134 134 134 F F F 13 260-279 88 88 88 138 138 138 F F F 14 280-299 85 85 85 135 135 135 F F F 15 300-319 80 80 80 126 126 126 F F F 16 320-339 87 87 87 137 137 137 F F F 17 340-359 88 88 88 138 138 138 F F F 18 360-379 83 83 83 131 131 131 F F F 19 380-399 87 87 87 137 137 137 F F F 20 400-419 87 87 87 137 137 137 F F F 21 420-439 85 85 85 135 135 135 F F F 22 440-459 85 85 85 135 135 135 F F F 23 460-479 81 81 81 127 127 127 F F F 24 480-499 80 80 80 122 122 122 F F F 25 500-519 88 88 88 138 138 138 F F F 26 520-539 89 89 89 141 141 141 F F F 27 540-559 89 89 89 141 141 141 F F F 28 560-579 89 89 89 141 141 141 F F F 29 580-599 84 84 84 132 132 132 F F F Max gen: 89, Max flush: 141 Begin loop 45 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:37.770720409Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1540,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:37.818043426Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1540,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:37.818631061Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1540} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"3818f0fc76fe787c6cc412af30fd59b930203885\",\n git_commit_timestamp: \"2024-11-01T16:08:30.000000000Z\",\n git_branch: \"renovate/hyper-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:37.818671787Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1540} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:37.818686718Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1540} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:37.81870116Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1540} {"msg":"Crucible cef5fdde-2232-4860-abc6-fa3a9fb81b12 has session id: 0b381dfe-dac2-41cf-ba17-d8f1fbdaa149","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:37.819140317Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1540,"session_id":"0b381dfe-dac2-41cf-ba17-d8f1fbdaa149"} {"msg":"Upstairs opts: Upstairs UUID: cef5fdde-2232-4860-abc6-fa3a9fb81b12, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:37.819180642Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1540,"session_id":"0b381dfe-dac2-41cf-ba17-d8f1fbdaa149"} {"msg":"Crucible stats registered with UUID: cef5fdde-2232-4860-abc6-fa3a9fb81b12","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:37.819204275Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1540,"session_id":"0b381dfe-dac2-41cf-ba17-d8f1fbdaa149"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:37.850742439Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1540,"task":"crutest"} {"msg":"The guest has requested activation with gen:90","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:37.850777073Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1540} {"msg":"cef5fdde-2232-4860-abc6-fa3a9fb81b12 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:37.85082951Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1540,"session_id":"0b381dfe-dac2-41cf-ba17-d8f1fbdaa149"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:37.850851223Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1540,"client":"0","":"downstairs","session_id":"0b381dfe-dac2-41cf-ba17-d8f1fbdaa149"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:37.850885647Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1540,"client":"0","":"downstairs","session_id":"0b381dfe-dac2-41cf-ba17-d8f1fbdaa149"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:37.850904269Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1540,"client":"1","":"downstairs","session_id":"0b381dfe-dac2-41cf-ba17-d8f1fbdaa149"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:37.850932173Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1540,"client":"1","":"downstairs","session_id":"0b381dfe-dac2-41cf-ba17-d8f1fbdaa149"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:37.850964457Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1540,"client":"2","":"downstairs","session_id":"0b381dfe-dac2-41cf-ba17-d8f1fbdaa149"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:37.850995281Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1540,"client":"2","":"downstairs","session_id":"0b381dfe-dac2-41cf-ba17-d8f1fbdaa149"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:37.851027385Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1540,"":"io task","client":"2","":"downstairs","session_id":"0b381dfe-dac2-41cf-ba17-d8f1fbdaa149"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:37.85106182Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1540,"":"io task","client":"0","":"downstairs","session_id":"0b381dfe-dac2-41cf-ba17-d8f1fbdaa149"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:37.851094924Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1540,"":"io task","client":"1","":"downstairs","session_id":"0b381dfe-dac2-41cf-ba17-d8f1fbdaa149"} {"msg":"ds_connection connected from Ok(127.0.0.1:64604)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:37.851127958Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1540,"":"io task","client":"0","":"downstairs","session_id":"0b381dfe-dac2-41cf-ba17-d8f1fbdaa149"} {"msg":"ds_connection connected from Ok(127.0.0.1:60095)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:37.851162603Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1540,"":"io task","client":"2","":"downstairs","session_id":"0b381dfe-dac2-41cf-ba17-d8f1fbdaa149"} {"msg":"ds_connection connected from Ok(127.0.0.1:50812)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:37.851200188Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1540,"":"io task","client":"1","":"downstairs","session_id":"0b381dfe-dac2-41cf-ba17-d8f1fbdaa149"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:37.851787733Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1540,"client":"1","":"downstairs","session_id":"0b381dfe-dac2-41cf-ba17-d8f1fbdaa149"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:37.851816157Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1540,"client":"1","":"downstairs","session_id":"0b381dfe-dac2-41cf-ba17-d8f1fbdaa149"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:37.851944754Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1540,"client":"0","":"downstairs","session_id":"0b381dfe-dac2-41cf-ba17-d8f1fbdaa149"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:37.851974147Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1540,"client":"0","":"downstairs","session_id":"0b381dfe-dac2-41cf-ba17-d8f1fbdaa149"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:37.85199407Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1540,"client":"2","":"downstairs","session_id":"0b381dfe-dac2-41cf-ba17-d8f1fbdaa149"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:37.852008572Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1540,"client":"2","":"downstairs","session_id":"0b381dfe-dac2-41cf-ba17-d8f1fbdaa149"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:37.852439297Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1540,"client":"1","":"downstairs","session_id":"0b381dfe-dac2-41cf-ba17-d8f1fbdaa149"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:37.852484473Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1540,"client":"0","":"downstairs","session_id":"0b381dfe-dac2-41cf-ba17-d8f1fbdaa149"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:37.852821407Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1540,"client":"2","":"downstairs","session_id":"0b381dfe-dac2-41cf-ba17-d8f1fbdaa149"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:37.85308445Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1540,"client":"1","":"downstairs","session_id":"0b381dfe-dac2-41cf-ba17-d8f1fbdaa149"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:37.853113104Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1540,"client":"1","":"downstairs","session_id":"0b381dfe-dac2-41cf-ba17-d8f1fbdaa149"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:37.853128606Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1540,"session_id":"0b381dfe-dac2-41cf-ba17-d8f1fbdaa149"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:37.853141318Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1540,"client":"0","":"downstairs","session_id":"0b381dfe-dac2-41cf-ba17-d8f1fbdaa149"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:37.853155019Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1540,"client":"0","":"downstairs","session_id":"0b381dfe-dac2-41cf-ba17-d8f1fbdaa149"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:37.853168241Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1540,"session_id":"0b381dfe-dac2-41cf-ba17-d8f1fbdaa149"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:37.853340343Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1540,"client":"2","":"downstairs","session_id":"0b381dfe-dac2-41cf-ba17-d8f1fbdaa149"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:37.853359686Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1540,"client":"2","":"downstairs","session_id":"0b381dfe-dac2-41cf-ba17-d8f1fbdaa149"} {"msg":"[0]R flush_numbers[0..12]: [140, 141, 136, 120, 138, 141, 141, 141, 120, 116, 130, 130]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:37.853373758Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1540,"":"downstairs","":"downstairs","session_id":"0b381dfe-dac2-41cf-ba17-d8f1fbdaa149"} {"msg":"[0]R generation[0..12]: [88, 89, 86, 79, 88, 89, 89, 89, 79, 76, 82, 82]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:37.853388269Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1540,"":"downstairs","":"downstairs","session_id":"0b381dfe-dac2-41cf-ba17-d8f1fbdaa149"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:37.853406132Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1540,"":"downstairs","":"downstairs","session_id":"0b381dfe-dac2-41cf-ba17-d8f1fbdaa149"} {"msg":"[1]R flush_numbers[0..12]: [140, 141, 136, 120, 138, 141, 141, 141, 120, 116, 130, 130]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:37.853420544Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1540,"":"downstairs","":"downstairs","session_id":"0b381dfe-dac2-41cf-ba17-d8f1fbdaa149"} {"msg":"[1]R generation[0..12]: [88, 89, 86, 79, 88, 89, 89, 89, 79, 76, 82, 82]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:37.853434526Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1540,"":"downstairs","":"downstairs","session_id":"0b381dfe-dac2-41cf-ba17-d8f1fbdaa149"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:37.853448657Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1540,"":"downstairs","":"downstairs","session_id":"0b381dfe-dac2-41cf-ba17-d8f1fbdaa149"} {"msg":"[2]R flush_numbers[0..12]: [140, 141, 136, 120, 138, 141, 141, 141, 120, 116, 130, 130]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:37.853462499Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1540,"":"downstairs","":"downstairs","session_id":"0b381dfe-dac2-41cf-ba17-d8f1fbdaa149"} {"msg":"[2]R generation[0..12]: [88, 89, 86, 79, 88, 89, 89, 89, 79, 76, 82, 82]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:37.853476121Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1540,"":"downstairs","":"downstairs","session_id":"0b381dfe-dac2-41cf-ba17-d8f1fbdaa149"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:37.853490023Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1540,"":"downstairs","":"downstairs","session_id":"0b381dfe-dac2-41cf-ba17-d8f1fbdaa149"} {"msg":"Max found gen is 90","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:37.853503134Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1540,"":"downstairs","":"downstairs","session_id":"0b381dfe-dac2-41cf-ba17-d8f1fbdaa149"} {"msg":"Generation requested: 90 >= found:90","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:37.853516336Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1540,"":"downstairs","":"downstairs","session_id":"0b381dfe-dac2-41cf-ba17-d8f1fbdaa149"} {"msg":"Next flush: 142","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:37.853529178Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1540,"":"downstairs","":"downstairs","session_id":"0b381dfe-dac2-41cf-ba17-d8f1fbdaa149"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:37.853542159Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1540,"":"downstairs","":"downstairs","session_id":"0b381dfe-dac2-41cf-ba17-d8f1fbdaa149"} Wait for a query_work_queue command to finish before sending IO {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:37.853563172Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1540,"session_id":"0b381dfe-dac2-41cf-ba17-d8f1fbdaa149"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:37.853575404Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1540,"client":"0","":"downstairs","session_id":"0b381dfe-dac2-41cf-ba17-d8f1fbdaa149"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:37.853588725Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1540,"client":"1","":"downstairs","session_id":"0b381dfe-dac2-41cf-ba17-d8f1fbdaa149"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:37.853602087Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1540,"client":"2","":"downstairs","session_id":"0b381dfe-dac2-41cf-ba17-d8f1fbdaa149"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:37.853617049Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1540,"session_id":"0b381dfe-dac2-41cf-ba17-d8f1fbdaa149"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:37.853629601Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1540,"session_id":"0b381dfe-dac2-41cf-ba17-d8f1fbdaa149"} {"msg":"cef5fdde-2232-4860-abc6-fa3a9fb81b12 is now active with session: 0b381dfe-dac2-41cf-ba17-d8f1fbdaa149","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:37.853642162Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1540,"session_id":"0b381dfe-dac2-41cf-ba17-d8f1fbdaa149"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:37.853654144Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1540,"session_id":"0b381dfe-dac2-41cf-ba17-d8f1fbdaa149"} {"msg":"The guest has finished waiting for activation with:90","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:37.853666425Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1540} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 534 len 4096 data: 11 02/30 Write block 545 len 40960 data: 5 7 7 7 8 6 11 11 11 11 03/30 Read block 334 len 40960 04/30 Write block 290 len 20480 data: 3 5 8 8 10 05/30 Write block 497 len 20480 data: 5 7 8 8 7 06/30 Write block 138 len 8192 data: 8 6 07/30 Read block 182 len 12288 08/30 Read block 539 len 40960 09/30 Read block 160 len 36864 10/30 Read block 481 len 8192 11/30 Write block 424 len 32768 data: 9 9 9 9 10 9 9 9 12/30 Write block 527 len 4096 data: 5 13/30 Read block 335 len 16384 14/30 Read block 531 len 16384 15/30 Write block 495 len 36864 data: 5 5 6 8 9 9 8 6 4 16/30 Read block 1 len 4096 17/30 Write block 313 len 12288 data: 11 10 12 18/30 Read block 164 len 28672 19/30 Write block 254 len 8192 data: 9 6 20/30 Read block 11 len 16384 21/30 Write block 527 len 36864 data: 6 7 8 10 10 10 10 12 12 22/30 Read block 581 len 8192 23/30 Flush 24/30 Write block 17 len 8192 data: 8 7 25/30 Write block 387 len 20480 data: 6 6 7 6 6 26/30 Write block 251 len 32768 data: 12 11 11 10 7 6 5 7 27/30 Write block 27 len 28672 data: 7 8 7 7 9 10 8 28/30 Write block 8 len 40960 data: 9 8 8 9 11 10 11 11 9 9 29/30 Write block 64 len 36864 data: 8 9 12 10 10 11 14 12 12 30/30 Read block 451 len 8192 ---------------------------------------------------------------- Crucible gen:90 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 2 Done Done Done false 1024 Acked Write 5 Done Done Done false 1025 Acked Write 8 Done Done Done false 1026 Acked Write 7 Done Done Done false 1027 Acked Write 10 Done Sent Done false 1028 Acked Write 9 Done Sent Sent false 1029 Acked Read 2 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 3 2 5 Done 7 4 5 16 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:53:37.874553076Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1540} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 88 88 88 140 140 140 T T T 1 020-039 89 89 89 141 141 141 T T T 2 040-059 86 86 86 136 136 136 F F F 3 060-079 79 79 79 120 120 120 T T T 4 080-099 88 88 88 138 138 138 F F F 5 100-119 89 89 89 141 141 141 F F F 6 120-139 90 90 90 142 142 142 F F F 7 140-159 89 89 89 141 141 141 F F F 8 160-179 79 79 79 120 120 120 F F F 9 180-199 76 76 76 116 116 116 F F F 10 200-219 82 82 82 130 130 130 F F F 11 220-239 82 82 82 130 130 130 F F F 12 240-259 90 90 90 142 142 142 T T T 13 260-279 88 88 88 138 138 138 F F F 14 280-299 90 90 90 142 142 142 F F F 15 300-319 90 90 90 142 142 142 F F F 16 320-339 87 87 87 137 137 137 F F F 17 340-359 88 88 88 138 138 138 F F F 18 360-379 83 83 83 131 131 131 F F F 19 380-399 87 87 87 137 137 137 T T T 20 400-419 87 87 87 137 137 137 F F F 21 420-439 90 90 90 142 142 142 F F F 22 440-459 85 85 85 135 135 135 F F F 23 460-479 81 81 81 127 127 127 F F F 24 480-499 90 90 90 142 142 142 F F F 25 500-519 90 90 90 142 142 142 F F F 26 520-539 90 90 90 142 142 142 F F F 27 540-559 90 90 90 142 142 142 F F F 28 560-579 89 89 89 141 141 141 F F F 29 580-599 84 84 84 132 132 132 F F F Max gen: 90, Max flush: 142 A Difference in extent metadata is expected here On loop 45 Verifying data now No change after verify Loop: 45 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 91 91 91 143 143 143 F F F 1 020-039 91 91 91 143 143 143 F F F 2 040-059 86 86 86 136 136 136 F F F 3 060-079 91 91 91 143 143 143 F F F 4 080-099 88 88 88 138 138 138 F F F 5 100-119 89 89 89 141 141 141 F F F 6 120-139 90 90 90 142 142 142 F F F 7 140-159 89 89 89 141 141 141 F F F 8 160-179 79 79 79 120 120 120 F F F 9 180-199 76 76 76 116 116 116 F F F 10 200-219 82 82 82 130 130 130 F F F 11 220-239 82 82 82 130 130 130 F F F 12 240-259 91 91 91 143 143 143 F F F 13 260-279 88 88 88 138 138 138 F F F 14 280-299 90 90 90 142 142 142 F F F 15 300-319 90 90 90 142 142 142 F F F 16 320-339 87 87 87 137 137 137 F F F 17 340-359 88 88 88 138 138 138 F F F 18 360-379 83 83 83 131 131 131 F F F 19 380-399 91 91 91 143 143 143 F F F 20 400-419 87 87 87 137 137 137 F F F 21 420-439 90 90 90 142 142 142 F F F 22 440-459 85 85 85 135 135 135 F F F 23 460-479 81 81 81 127 127 127 F F F 24 480-499 90 90 90 142 142 142 F F F 25 500-519 90 90 90 142 142 142 F F F 26 520-539 90 90 90 142 142 142 F F F 27 540-559 90 90 90 142 142 142 F F F 28 560-579 89 89 89 141 141 141 F F F 29 580-599 84 84 84 132 132 132 F F F Max gen: 91, Max flush: 143 Begin loop 46 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:42.181357417Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1548,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:42.225189124Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1548,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:42.225923379Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1548} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"3818f0fc76fe787c6cc412af30fd59b930203885\",\n git_commit_timestamp: \"2024-11-01T16:08:30.000000000Z\",\n git_branch: \"renovate/hyper-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:42.225961444Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1548} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:42.225975666Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1548} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:42.225989838Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1548} {"msg":"Crucible 659eac65-de62-443a-a9ed-7aa4440cfa29 has session id: d73713fb-d14a-4f47-a54f-3239dc5145d1","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:42.226463219Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1548,"session_id":"d73713fb-d14a-4f47-a54f-3239dc5145d1"} {"msg":"Upstairs opts: Upstairs UUID: 659eac65-de62-443a-a9ed-7aa4440cfa29, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:42.226504774Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1548,"session_id":"d73713fb-d14a-4f47-a54f-3239dc5145d1"} {"msg":"Crucible stats registered with UUID: 659eac65-de62-443a-a9ed-7aa4440cfa29","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:42.226529377Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1548,"session_id":"d73713fb-d14a-4f47-a54f-3239dc5145d1"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:42.257040019Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1548,"task":"crutest"} {"msg":"The guest has requested activation with gen:92","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:42.257076283Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1548} {"msg":"659eac65-de62-443a-a9ed-7aa4440cfa29 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:42.257144792Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1548,"session_id":"d73713fb-d14a-4f47-a54f-3239dc5145d1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:42.257164095Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1548,"client":"0","":"downstairs","session_id":"d73713fb-d14a-4f47-a54f-3239dc5145d1"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:42.257187387Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1548,"client":"0","":"downstairs","session_id":"d73713fb-d14a-4f47-a54f-3239dc5145d1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:42.257201669Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1548,"client":"1","":"downstairs","session_id":"d73713fb-d14a-4f47-a54f-3239dc5145d1"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:42.257215321Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1548,"client":"1","":"downstairs","session_id":"d73713fb-d14a-4f47-a54f-3239dc5145d1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:42.257232263Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1548,"client":"2","":"downstairs","session_id":"d73713fb-d14a-4f47-a54f-3239dc5145d1"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:42.257246275Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1548,"client":"2","":"downstairs","session_id":"d73713fb-d14a-4f47-a54f-3239dc5145d1"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:42.257259567Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1548,"":"io task","client":"2","":"downstairs","session_id":"d73713fb-d14a-4f47-a54f-3239dc5145d1"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:42.257273469Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1548,"":"io task","client":"0","":"downstairs","session_id":"d73713fb-d14a-4f47-a54f-3239dc5145d1"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:42.25728694Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1548,"":"io task","client":"1","":"downstairs","session_id":"d73713fb-d14a-4f47-a54f-3239dc5145d1"} {"msg":"ds_connection connected from Ok(127.0.0.1:48409)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:42.257346368Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1548,"":"io task","client":"2","":"downstairs","session_id":"d73713fb-d14a-4f47-a54f-3239dc5145d1"} {"msg":"ds_connection connected from Ok(127.0.0.1:50425)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:42.257494717Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1548,"":"io task","client":"1","":"downstairs","session_id":"d73713fb-d14a-4f47-a54f-3239dc5145d1"} {"msg":"ds_connection connected from Ok(127.0.0.1:58195)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:42.257628474Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1548,"":"io task","client":"0","":"downstairs","session_id":"d73713fb-d14a-4f47-a54f-3239dc5145d1"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:42.258168424Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1548,"client":"0","":"downstairs","session_id":"d73713fb-d14a-4f47-a54f-3239dc5145d1"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:42.258206209Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1548,"client":"0","":"downstairs","session_id":"d73713fb-d14a-4f47-a54f-3239dc5145d1"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:42.258236213Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1548,"client":"2","":"downstairs","session_id":"d73713fb-d14a-4f47-a54f-3239dc5145d1"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:42.258302451Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1548,"client":"2","":"downstairs","session_id":"d73713fb-d14a-4f47-a54f-3239dc5145d1"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:42.258331375Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1548,"client":"1","":"downstairs","session_id":"d73713fb-d14a-4f47-a54f-3239dc5145d1"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:42.258346107Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1548,"client":"1","":"downstairs","session_id":"d73713fb-d14a-4f47-a54f-3239dc5145d1"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:42.258771842Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1548,"client":"0","":"downstairs","session_id":"d73713fb-d14a-4f47-a54f-3239dc5145d1"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:42.258892227Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1548,"client":"1","":"downstairs","session_id":"d73713fb-d14a-4f47-a54f-3239dc5145d1"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:42.25891401Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1548,"client":"2","":"downstairs","session_id":"d73713fb-d14a-4f47-a54f-3239dc5145d1"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:42.259390131Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1548,"client":"0","":"downstairs","session_id":"d73713fb-d14a-4f47-a54f-3239dc5145d1"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:42.259419165Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1548,"client":"0","":"downstairs","session_id":"d73713fb-d14a-4f47-a54f-3239dc5145d1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:42.259435287Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1548,"session_id":"d73713fb-d14a-4f47-a54f-3239dc5145d1"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:42.259502346Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1548,"client":"1","":"downstairs","session_id":"d73713fb-d14a-4f47-a54f-3239dc5145d1"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:42.25953426Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1548,"client":"1","":"downstairs","session_id":"d73713fb-d14a-4f47-a54f-3239dc5145d1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:42.259550002Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1548,"session_id":"d73713fb-d14a-4f47-a54f-3239dc5145d1"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:42.259562544Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1548,"client":"2","":"downstairs","session_id":"d73713fb-d14a-4f47-a54f-3239dc5145d1"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:42.259576215Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1548,"client":"2","":"downstairs","session_id":"d73713fb-d14a-4f47-a54f-3239dc5145d1"} {"msg":"[0]R flush_numbers[0..12]: [143, 143, 136, 143, 138, 141, 142, 141, 120, 116, 130, 130]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:42.259590097Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1548,"":"downstairs","":"downstairs","session_id":"d73713fb-d14a-4f47-a54f-3239dc5145d1"} {"msg":"[0]R generation[0..12]: [91, 91, 86, 91, 88, 89, 90, 89, 79, 76, 82, 82]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:42.259604579Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1548,"":"downstairs","":"downstairs","session_id":"d73713fb-d14a-4f47-a54f-3239dc5145d1"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:42.259618581Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1548,"":"downstairs","":"downstairs","session_id":"d73713fb-d14a-4f47-a54f-3239dc5145d1"} {"msg":"[1]R flush_numbers[0..12]: [143, 143, 136, 143, 138, 141, 142, 141, 120, 116, 130, 130]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:42.259632233Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1548,"":"downstairs","":"downstairs","session_id":"d73713fb-d14a-4f47-a54f-3239dc5145d1"} {"msg":"[1]R generation[0..12]: [91, 91, 86, 91, 88, 89, 90, 89, 79, 76, 82, 82]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:42.259645794Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1548,"":"downstairs","":"downstairs","session_id":"d73713fb-d14a-4f47-a54f-3239dc5145d1"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:42.259663426Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1548,"":"downstairs","":"downstairs","session_id":"d73713fb-d14a-4f47-a54f-3239dc5145d1"} {"msg":"[2]R flush_numbers[0..12]: [143, 143, 136, 143, 138, 141, 142, 141, 120, 116, 130, 130]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:42.259677658Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1548,"":"downstairs","":"downstairs","session_id":"d73713fb-d14a-4f47-a54f-3239dc5145d1"} {"msg":"[2]R generation[0..12]: [91, 91, 86, 91, 88, 89, 90, 89, 79, 76, 82, 82]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:42.25969191Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1548,"":"downstairs","":"downstairs","session_id":"d73713fb-d14a-4f47-a54f-3239dc5145d1"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:42.259707042Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1548,"":"downstairs","":"downstairs","session_id":"d73713fb-d14a-4f47-a54f-3239dc5145d1"} {"msg":"Max found gen is 92","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:42.259730365Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBPWait for a query_work_queue command to finish before sending IO ","pid":1548,"":"downstairs","":"downstairs","session_id":"d73713fb-d14a-4f47-a54f-3239dc5145d1"} {"msg":"Generation requested: 92 >= found:92","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:42.259763919Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1548,"":"downstairs","":"downstairs","session_id":"d73713fb-d14a-4f47-a54f-3239dc5145d1"} {"msg":"Next flush: 144","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:42.259782462Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1548,"":"downstairs","":"downstairs","session_id":"d73713fb-d14a-4f47-a54f-3239dc5145d1"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:42.259796003Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1548,"":"downstairs","":"downstairs","session_id":"d73713fb-d14a-4f47-a54f-3239dc5145d1"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:42.259809325Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1548,"session_id":"d73713fb-d14a-4f47-a54f-3239dc5145d1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:42.259821337Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1548,"client":"0","":"downstairs","session_id":"d73713fb-d14a-4f47-a54f-3239dc5145d1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:42.259834599Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1548,"client":"1","":"downstairs","session_id":"d73713fb-d14a-4f47-a54f-3239dc5145d1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:42.25984783Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1548,"client":"2","":"downstairs","session_id":"d73713fb-d14a-4f47-a54f-3239dc5145d1"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:42.259862242Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1548,"session_id":"d73713fb-d14a-4f47-a54f-3239dc5145d1"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:42.259874744Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1548,"session_id":"d73713fb-d14a-4f47-a54f-3239dc5145d1"} {"msg":"659eac65-de62-443a-a9ed-7aa4440cfa29 is now active with session: d73713fb-d14a-4f47-a54f-3239dc5145d1","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:42.259887575Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1548,"session_id":"d73713fb-d14a-4f47-a54f-3239dc5145d1"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:42.259902037Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1548,"session_id":"d73713fb-d14a-4f47-a54f-3239dc5145d1"} {"msg":"The guest has finished waiting for activation with:92","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:42.259914409Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1548} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 36 len 4096 data: 10 03/30 Write block 147 len 24576 data: 6 6 8 11 9 10 04/30 Read block 523 len 36864 05/30 Write block 214 len 12288 data: 7 5 9 06/30 Flush 07/30 Write block 169 len 40960 data: 5 6 5 5 5 5 7 8 9 8 08/30 Flush 09/30 Write block 63 len 40960 data: 6 9 10 13 11 11 12 15 13 13 10/30 Read block 60 len 8192 11/30 Write block 155 len 28672 data: 10 9 7 7 5 7 7 12/30 Read block 529 len 24576 13/30 Read block 466 len 40960 14/30 Read block 550 len 40960 15/30 Read block 498 len 8192 16/30 Write block 469 len 24576 data: 8 9 11 11 11 10 17/30 Write block 276 len 4096 data: 13 18/30 Read block 342 len 28672 19/30 Write block 68 len 8192 data: 12 13 20/30 Write block 366 len 4096 data: 5 21/30 Read block 141 len 16384 22/30 Write block 485 len 36864 data: 10 9 11 12 11 10 10 7 4 23/30 Read block 464 len 12288 24/30 Read block 108 len 28672 25/30 Write block 156 len 40960 data: 10 8 8 6 8 8 7 7 6 6 26/30 Write block 70 len 16384 data: 16 14 14 9 27/30 Read block 142 len 16384 28/30 Write block 210 len 32768 data: 10 12 10 8 8 6 10 11 29/30 Write block 229 len 8192 data: 7 7 30/30 Read block 119 len 4096 ---------------------------------------------------------------- Crucible gen:92 GIO:true work queues: Upstairs:0 downstairs:22 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1008 Acked Write 10 Done Done Done false 1009 Acked Read 2 Done Done Done false 1010 Acked Write 7 Done Done Done false 1011 Acked Read 6 Done Done Done false 1012 Acked Read 10 Done Done Done false 1013 Acked Read 10 Done Done Done false 1014 Acked Read 2 Done Done Done false 1015 Acked Write 6 Done Done Done false 1016 Acked Write 1 Done Done Done false 1017 Acked Read 7 Done Done Done false 1018 Acked Write 2 Done Done Done false 1019 Acked Write 1 Done Done Done false 1020 Acked Read 4 Done Done Done false 1021 Acked Write 9 Done Done Done false 1022 Acked Read 3 Done Done Done false 1023 Acked Read 7 Done Done Done false 1024 Acked Write 10 Done Done Done false 1025 Acked Write 4 Done Done Done false 1026 Acked Read 4 Done Done Done false 1027 Acked Write 8 Done Done Done false 1028 Acked Write 2 Done Done Done false 1029 Acked Read 1 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 22 22 22 66 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1007 1007 1007 Downstairs last five completed: 1007 1006 1005 1004 1003 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:53:42.281624876Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1548} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 91 91 91 143 143 143 F F F 1 020-039 92 92 92 145 145 145 F F F 2 040-059 86 86 86 136 136 136 F F F 3 060-079 91 91 91 143 143 143 T T T 4 080-099 88 88 88 138 138 138 F F F 5 100-119 89 89 89 141 141 141 F F F 6 120-139 90 90 90 142 142 142 F F F 7 140-159 92 92 92 145 145 145 T T T 8 160-179 92 92 92 146 146 146 T T T 9 180-199 76 76 76 116 116 116 F F F 10 200-219 92 92 92 145 145 145 T T T 11 220-239 82 82 82 130 130 130 T T T 12 240-259 91 91 91 143 143 143 F F F 13 260-279 88 88 88 138 138 138 T T T 14 280-299 90 90 90 142 142 142 F F F 15 300-319 90 90 90 142 142 142 F F F 16 320-339 87 87 87 137 137 137 F F F 17 340-359 88 88 88 138 138 138 F F F 18 360-379 83 83 83 131 131 131 T T T 19 380-399 91 91 91 143 143 143 F F F 20 400-419 87 87 87 137 137 137 F F F 21 420-439 90 90 90 142 142 142 F F F 22 440-459 85 85 85 135 135 135 F F F 23 460-479 81 81 81 127 127 127 T T T 24 480-499 90 90 90 142 142 142 T T T 25 500-519 90 90 90 142 142 142 F F F 26 520-539 90 90 90 142 142 142 F F F 27 540-559 90 90 90 142 142 142 F F F 28 560-579 89 89 89 141 141 141 F F F 29 580-599 84 84 84 132 132 132 F F F Max gen: 92, Max flush: 146 A Difference in extent metadata is expected here On loop 46 Verifying data now No change after verify Loop: 46 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 91 91 91 143 143 143 F F F 1 020-039 92 92 92 145 145 145 F F F 2 040-059 86 86 86 136 136 136 F F F 3 060-079 93 93 93 147 147 147 F F F 4 080-099 88 88 88 138 138 138 F F F 5 100-119 89 89 89 141 141 141 F F F 6 120-139 90 90 90 142 142 142 F F F 7 140-159 93 93 93 147 147 147 F F F 8 160-179 93 93 93 147 147 147 F F F 9 180-199 76 76 76 116 116 116 F F F 10 200-219 93 93 93 147 147 147 F F F 11 220-239 93 93 93 147 147 147 F F F 12 240-259 91 91 91 143 143 143 F F F 13 260-279 93 93 93 147 147 147 F F F 14 280-299 90 90 90 142 142 142 F F F 15 300-319 90 90 90 142 142 142 F F F 16 320-339 87 87 87 137 137 137 F F F 17 340-359 88 88 88 138 138 138 F F F 18 360-379 93 93 93 147 147 147 F F F 19 380-399 91 91 91 143 143 143 F F F 20 400-419 87 87 87 137 137 137 F F F 21 420-439 90 90 90 142 142 142 F F F 22 440-459 85 85 85 135 135 135 F F F 23 460-479 93 93 93 147 147 147 F F F 24 480-499 93 93 93 147 147 147 F F F 25 500-519 90 90 90 142 142 142 F F F 26 520-539 90 90 90 142 142 142 F F F 27 540-559 90 90 90 142 142 142 F F F 28 560-579 89 89 89 141 141 141 F F F 29 580-599 84 84 84 132 132 132 F F F Max gen: 93, Max flush: 147 Begin loop 47 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:46.625563242Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1556,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:46.669548569Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1556,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:46.670123283Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1556} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"3818f0fc76fe787c6cc412af30fd59b930203885\",\n git_commit_timestamp: \"2024-11-01T16:08:30.000000000Z\",\n git_branch: \"renovate/hyper-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:46.670166149Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1556} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:46.670181081Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1556} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:46.670195383Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1556} {"msg":"Crucible 8aee2b9b-4861-4e92-9ccc-3fa7b8b3fc64 has session id: 107a8127-1e2f-4911-b0d7-0fa74c56972f","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:46.670662093Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1556,"session_id":"107a8127-1e2f-4911-b0d7-0fa74c56972f"} {"msg":"Upstairs opts: Upstairs UUID: 8aee2b9b-4861-4e92-9ccc-3fa7b8b3fc64, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:46.670694027Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1556,"session_id":"107a8127-1e2f-4911-b0d7-0fa74c56972f"} {"msg":"Crucible stats registered with UUID: 8aee2b9b-4861-4e92-9ccc-3fa7b8b3fc64","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:46.670709949Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1556,"session_id":"107a8127-1e2f-4911-b0d7-0fa74c56972f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:46.701620132Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1556,"task":"crutest"} {"msg":"The guest has requested activation with gen:94","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:46.701665338Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1556} {"msg":"8aee2b9b-4861-4e92-9ccc-3fa7b8b3fc64 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:46.701734066Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1556,"session_id":"107a8127-1e2f-4911-b0d7-0fa74c56972f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:46.701765711Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1556,"client":"0","":"downstairs","session_id":"107a8127-1e2f-4911-b0d7-0fa74c56972f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:46.701811836Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1556,"client":"0","":"downstairs","session_id":"107a8127-1e2f-4911-b0d7-0fa74c56972f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:46.701849101Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1556,"client":"1","":"downstairs","session_id":"107a8127-1e2f-4911-b0d7-0fa74c56972f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:46.7019191Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1556,"client":"1","":"downstairs","session_id":"107a8127-1e2f-4911-b0d7-0fa74c56972f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:46.701968097Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1556,"client":"2","":"downstairs","session_id":"107a8127-1e2f-4911-b0d7-0fa74c56972f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:46.702005851Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1556,"client":"2","":"downstairs","session_id":"107a8127-1e2f-4911-b0d7-0fa74c56972f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:46.702037246Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1556,"":"io task","client":"2","":"downstairs","session_id":"107a8127-1e2f-4911-b0d7-0fa74c56972f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:46.70207032Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1556,"":"io task","client":"0","":"downstairs","session_id":"107a8127-1e2f-4911-b0d7-0fa74c56972f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:46.702101424Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1556,"":"io task","client":"1","":"downstairs","session_id":"107a8127-1e2f-4911-b0d7-0fa74c56972f"} {"msg":"ds_connection connected from Ok(127.0.0.1:56050)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:46.702128547Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1556,"":"io task","client":"2","":"downstairs","session_id":"107a8127-1e2f-4911-b0d7-0fa74c56972f"} {"msg":"ds_connection connected from Ok(127.0.0.1:55577)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:46.702160271Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1556,"":"io task","client":"1","":"downstairs","session_id":"107a8127-1e2f-4911-b0d7-0fa74c56972f"} {"msg":"ds_connection connected from Ok(127.0.0.1:33340)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:46.702194666Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1556,"":"io task","client":"0","":"downstairs","session_id":"107a8127-1e2f-4911-b0d7-0fa74c56972f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:46.702758698Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1556,"client":"1","":"downstairs","session_id":"107a8127-1e2f-4911-b0d7-0fa74c56972f"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:46.702787212Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1556,"client":"1","":"downstairs","session_id":"107a8127-1e2f-4911-b0d7-0fa74c56972f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:46.702802324Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1556,"client":"2","":"downstairs","session_id":"107a8127-1e2f-4911-b0d7-0fa74c56972f"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:46.702825917Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1556,"client":"2","":"downstairs","session_id":"107a8127-1e2f-4911-b0d7-0fa74c56972f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:46.70285181Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1556,"client":"0","":"downstairs","session_id":"107a8127-1e2f-4911-b0d7-0fa74c56972f"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:46.702879764Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1556,"client":"0","":"downstairs","session_id":"107a8127-1e2f-4911-b0d7-0fa74c56972f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:46.703357856Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1556,"client":"0","":"downstairs","session_id":"107a8127-1e2f-4911-b0d7-0fa74c56972f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:46.703455728Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1556,"client":"1","":"downstairs","session_id":"107a8127-1e2f-4911-b0d7-0fa74c56972f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:46.703478501Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1556,"client":"2","":"downstairs","session_id":"107a8127-1e2f-4911-b0d7-0fa74c56972f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:46.703918658Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1556,"client":"0","":"downstairs","session_id":"107a8127-1e2f-4911-b0d7-0fa74c56972f"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:46.703943291Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1556,"client":"0","":"downstairs","session_id":"107a8127-1e2f-4911-b0d7-0fa74c56972f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:46.703959523Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1556,"session_id":"107a8127-1e2f-4911-b0d7-0fa74c56972f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:46.704050575Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1556,"client":"1","":"downstairs","session_id":"107a8127-1e2f-4911-b0d7-0fa74c56972f"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:46.704070587Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1556,"client":"1","":"downstairs","session_id":"107a8127-1e2f-4911-b0d7-0fa74c56972f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:46.704084869Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1556,"session_id":"107a8127-1e2f-4911-b0d7-0fa74c56972f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:46.704097651Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1556,"client":"2","":"downstairs","session_id":"107a8127-1e2f-4911-b0d7-0fa74c56972f"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:46.704111073Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1556,"client":"2","":"downstairs","session_id":"107a8127-1e2f-4911-b0d7-0fa74c56972f"} {"msg":"[0]R flush_numbers[0..12]: [143, 145, 136, 147, 138, 141, 142, 147, 147, 116, 147, 147]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:46.704124844Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1556,"":"downstairs","":"downstairs","session_id":"107a8127-1e2f-4911-b0d7-0fa74c56972f"} {"msg":"[0]R generation[0..12]: [91, 92, 86, 93, 88, 89, 90, 93, 93, 76, 93, 93]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:46.704138536Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1556,"":"downstairs","":"downstairs","session_id":"107a8127-1e2f-4911-b0d7-0fa74c56972f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:46.704152638Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1556,"":"downstairs","":"downstairs","session_id":"107a8127-1e2f-4911-b0d7-0fa74c56972f"} {"msg":"[1]R flush_numbers[0..12]: [143, 145, 136, 147, 138, 141, 142, 147, 147, 116, 147, 147]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:46.70416642Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1556,"":"downstairs","":"downstairs","session_id":"107a8127-1e2f-4911-b0d7-0fa74c56972f"} {"msg":"[1]R generation[0..12]: [91, 92, 86, 93, 88, 89, 90, 93, 93, 76, 93, 93]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:46.704180792Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1556,"":"downstairs","":"downstairs","session_id":"107a8127-1e2f-4911-b0d7-0fa74c56972f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:46.704198024Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1556,"":"downstairs","":"downstairs","session_id":"107a8127-1e2f-4911-b0d7-0fa74c56972f"} {"msg":"[2]R flush_numbers[0..12]: [143, 145, 136, 147, 138, 141, 142, 147, 147, 116, 147, 147]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:46.704211996Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1556,"":"downstairs","":"downstairs","session_id":"107a8127-1e2f-4911-b0d7-0fa74c56972f"} {"msg":"[2]R generation[0..12]: [91, 92, 86, 93, 88, 89, 90, 93, 93, 76, 93, 93]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:46.704225767Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1556,"":"downstairs","":"downstairs","session_id":"107a8127-1e2f-4911-b0d7-0fa74c56972f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:46.704239629Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1556,"":"downstairs","":"downstairs","session_id":"107a8127-1e2f-4911-b0d7-0fa74c56972f"} {"msg":"Max found gen is 94","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:46.704252661Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1556,"":"downstairs","":"downstairs","session_id":"107a8127-1e2f-4911-b0d7-0fa74c56972f"} {"msg":"Generation requested: 94 >= found:94","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:46.704265942Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1556,"":"downstairs","":"downstairs","session_id":"107a8127-1e2f-4911-b0d7-0fa74c56972f"} {"msg":"Next flush: 148","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:46.704278994Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1556,"":"downstairs","":"downstairs","session_id":"107a8127-1e2f-4911-b0d7-0fa74c56972f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:46.704291806Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1556,"":"downstairs","":"downstairs","session_id":"107a8127-1e2f-4911-b0d7-0fa74c56972f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:46.704305138Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1556,"session_id":"107a8127-1e2f-4911-b0d7-0fa74c56972f"} {Wait for a query_work_queue command to finish before sending IO "msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:46.704328281Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1556,"client":"0","":"downstairs","session_id":"107a8127-1e2f-4911-b0d7-0fa74c56972f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:46.704342472Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1556,"client":"1","":"downstairs","session_id":"107a8127-1e2f-4911-b0d7-0fa74c56972f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:46.704355894Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1556,"client":"2","":"downstairs","session_id":"107a8127-1e2f-4911-b0d7-0fa74c56972f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:46.704369166Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1556,"session_id":"107a8127-1e2f-4911-b0d7-0fa74c56972f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:46.704381697Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1556,"session_id":"107a8127-1e2f-4911-b0d7-0fa74c56972f"} {"msg":"8aee2b9b-4861-4e92-9ccc-3fa7b8b3fc64 is now active with session: 107a8127-1e2f-4911-b0d7-0fa74c56972f","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:46.704394589Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1556,"session_id":"107a8127-1e2f-4911-b0d7-0fa74c56972f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:46.704409091Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1556,"session_id":"107a8127-1e2f-4911-b0d7-0fa74c56972f"} {"msg":"The guest has finished waiting for activation with:94","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:46.704422343Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1556} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 241 len 12288 data: 6 6 5 03/30 Read block 233 len 40960 04/30 Read block 347 len 4096 05/30 Write block 243 len 24576 data: 6 6 9 11 13 12 06/30 Write block 6 len 4096 data: 9 07/30 Write block 274 len 8192 data: 13 12 08/30 Read block 204 len 32768 09/30 Read block 430 len 28672 10/30 Read block 79 len 40960 11/30 Read block 577 len 8192 12/30 Read block 4 len 40960 13/30 Write block 9 len 24576 data: 9 9 10 12 11 12 14/30 Read block 338 len 24576 15/30 Flush 16/30 Write block 189 len 32768 data: 9 10 8 8 9 10 10 10 17/30 Write block 163 len 4096 data: 8 18/30 Write block 202 len 8192 data: 7 9 19/30 Write block 422 len 32768 data: 9 9 10 10 10 10 11 10 20/30 Write block 46 len 20480 data: 11 7 6 9 9 21/30 Write block 354 len 8192 data: 7 6 22/30 Read block 329 len 36864 23/30 Flush 24/30 Write block 420 len 16384 data: 8 9 10 10 25/30 Write block 393 len 32768 data: 6 5 7 9 10 11 10 9 26/30 Read block 28 len 32768 27/30 Write block 165 len 8192 data: 7 5 28/30 Write block 571 len 20480 data: 6 7 6 6 8 29/30 Read block 112 len 16384 30/30 Write block 34 len 8192 data: 8 9 ---------------------------------------------------------------- Crucible gen:94 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 4 Done Done Done false 1024 Acked Write 8 Done Done Done false 1025 Acked Read 8 Done Done Done false 1026 Acked Write 2 Done Done Done false 1027 Acked Write 5 Done Done Done false 1028 Acked Read 4 Done Done Done false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 6 6 6 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:53:46.726406965Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1556} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 94 94 94 149 149 149 F F F 1 020-039 92 92 92 145 145 145 T T T 2 040-059 94 94 94 150 150 150 F F F 3 060-079 93 93 93 147 147 147 F F F 4 080-099 88 88 88 138 138 138 F F F 5 100-119 89 89 89 141 141 141 F F F 6 120-139 90 90 90 142 142 142 F F F 7 140-159 93 93 93 147 147 147 F F F 8 160-179 94 94 94 150 150 150 T T T 9 180-199 94 94 94 150 150 150 F F F 10 200-219 94 94 94 150 150 150 F F F 11 220-239 93 93 93 147 147 147 F F F 12 240-259 94 94 94 149 149 149 F F F 13 260-279 94 94 94 149 149 149 F F F 14 280-299 90 90 90 142 142 142 F F F 15 300-319 90 90 90 142 142 142 F F F 16 320-339 87 87 87 137 137 137 F F F 17 340-359 94 94 94 150 150 150 F F F 18 360-379 93 93 93 147 147 147 F F F 19 380-399 91 91 91 143 143 143 T T T 20 400-419 87 87 87 137 137 137 T T T 21 420-439 94 94 94 150 150 150 T T T 22 440-459 85 85 85 135 135 135 F F F 23 460-479 93 93 93 147 147 147 F F F 24 480-499 93 93 93 147 147 147 F F F 25 500-519 90 90 90 142 142 142 F F F 26 520-539 90 90 90 142 142 142 F F F 27 540-559 90 90 90 142 142 142 F F F 28 560-579 89 89 89 141 141 141 T T T 29 580-599 84 84 84 132 132 132 F F F Max gen: 94, Max flush: 150 A Difference in extent metadata is expected here On loop 47 Verifying data now No change after verify Loop: 47 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 94 94 94 149 149 149 F F F 1 020-039 95 95 95 151 151 151 F F F 2 040-059 94 94 94 150 150 150 F F F 3 060-079 93 93 93 147 147 147 F F F 4 080-099 88 88 88 138 138 138 F F F 5 100-119 89 89 89 141 141 141 F F F 6 120-139 90 90 90 142 142 142 F F F 7 140-159 93 93 93 147 147 147 F F F 8 160-179 95 95 95 151 151 151 F F F 9 180-199 94 94 94 150 150 150 F F F 10 200-219 94 94 94 150 150 150 F F F 11 220-239 93 93 93 147 147 147 F F F 12 240-259 94 94 94 149 149 149 F F F 13 260-279 94 94 94 149 149 149 F F F 14 280-299 90 90 90 142 142 142 F F F 15 300-319 90 90 90 142 142 142 F F F 16 320-339 87 87 87 137 137 137 F F F 17 340-359 94 94 94 150 150 150 F F F 18 360-379 93 93 93 147 147 147 F F F 19 380-399 95 95 95 151 151 151 F F F 20 400-419 95 95 95 151 151 151 F F F 21 420-439 95 95 95 151 151 151 F F F 22 440-459 85 85 85 135 135 135 F F F 23 460-479 93 93 93 147 147 147 F F F 24 480-499 93 93 93 147 147 147 F F F 25 500-519 90 90 90 142 142 142 F F F 26 520-539 90 90 90 142 142 142 F F F 27 540-559 90 90 90 142 142 142 F F F 28 560-579 95 95 95 151 151 151 F F F 29 580-599 84 84 84 132 132 132 F F F Max gen: 95, Max flush: 151 Begin loop 48 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:51.045548555Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1564,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:51.089455143Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1564,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:51.089985551Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1564} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"3818f0fc76fe787c6cc412af30fd59b930203885\",\n git_commit_timestamp: \"2024-11-01T16:08:30.000000000Z\",\n git_branch: \"renovate/hyper-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:51.090025336Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1564} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:51.090044138Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1564} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:51.090059Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1564} {"msg":"Crucible 47aced27-9504-4448-9f82-19ca52b2da51 has session id: 59a45119-5de7-4a57-bd3c-92722fc94654","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:51.090427048Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1564,"session_id":"59a45119-5de7-4a57-bd3c-92722fc94654"} {"msg":"Upstairs opts: Upstairs UUID: 47aced27-9504-4448-9f82-19ca52b2da51, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:51.090468003Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1564,"session_id":"59a45119-5de7-4a57-bd3c-92722fc94654"} {"msg":"Crucible stats registered with UUID: 47aced27-9504-4448-9f82-19ca52b2da51","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:51.090485275Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1564,"session_id":"59a45119-5de7-4a57-bd3c-92722fc94654"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:51.121430773Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1564,"task":"crutest"} {"msg":"The guest has requested activation with gen:96","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:51.121464147Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1564} {"msg":"47aced27-9504-4448-9f82-19ca52b2da51 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:51.121514383Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1564,"session_id":"59a45119-5de7-4a57-bd3c-92722fc94654"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:51.121535216Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1564,"client":"0","":"downstairs","session_id":"59a45119-5de7-4a57-bd3c-92722fc94654"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:51.12156464Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1564,"client":"0","":"downstairs","session_id":"59a45119-5de7-4a57-bd3c-92722fc94654"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:51.121594674Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1564,"client":"1","":"downstairs","session_id":"59a45119-5de7-4a57-bd3c-92722fc94654"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:51.121626968Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1564,"client":"1","":"downstairs","session_id":"59a45119-5de7-4a57-bd3c-92722fc94654"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:51.121659242Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1564,"client":"2","":"downstairs","session_id":"59a45119-5de7-4a57-bd3c-92722fc94654"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:51.121693056Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1564,"client":"2","":"downstairs","session_id":"59a45119-5de7-4a57-bd3c-92722fc94654"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:51.121725681Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1564,"":"io task","client":"2","":"downstairs","session_id":"59a45119-5de7-4a57-bd3c-92722fc94654"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:51.121764936Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1564,"":"io task","client":"0","":"downstairs","session_id":"59a45119-5de7-4a57-bd3c-92722fc94654"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:51.12179919Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1564,"":"io task","client":"1","":"downstairs","session_id":"59a45119-5de7-4a57-bd3c-92722fc94654"} {"msg":"ds_connection connected from Ok(127.0.0.1:39839)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:51.121834775Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1564,"":"io task","client":"2","":"downstairs","session_id":"59a45119-5de7-4a57-bd3c-92722fc94654"} {"msg":"ds_connection connected from Ok(127.0.0.1:54907)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:51.121867209Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1564,"":"io task","client":"0","":"downstairs","session_id":"59a45119-5de7-4a57-bd3c-92722fc94654"} {"msg":"ds_connection connected from Ok(127.0.0.1:59972)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:51.121988245Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1564,"":"io task","client":"1","":"downstairs","session_id":"59a45119-5de7-4a57-bd3c-92722fc94654"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:51.122543696Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1564,"client":"1","":"downstairs","session_id":"59a45119-5de7-4a57-bd3c-92722fc94654"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:51.12257114Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1564,"client":"1","":"downstairs","session_id":"59a45119-5de7-4a57-bd3c-92722fc94654"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:51.122585842Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1564,"client":"2","":"downstairs","session_id":"59a45119-5de7-4a57-bd3c-92722fc94654"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:51.122599543Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1564,"client":"2","":"downstairs","session_id":"59a45119-5de7-4a57-bd3c-92722fc94654"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:51.122616195Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1564,"client":"0","":"downstairs","session_id":"59a45119-5de7-4a57-bd3c-92722fc94654"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:51.122641629Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1564,"client":"0","":"downstairs","session_id":"59a45119-5de7-4a57-bd3c-92722fc94654"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:51.123216403Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1564,"client":"1","":"downstairs","session_id":"59a45119-5de7-4a57-bd3c-92722fc94654"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:51.123245927Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1564,"client":"2","":"downstairs","session_id":"59a45119-5de7-4a57-bd3c-92722fc94654"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:51.123263499Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1564,"client":"0","":"downstairs","session_id":"59a45119-5de7-4a57-bd3c-92722fc94654"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:51.123809269Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1564,"client":"1","":"downstairs","session_id":"59a45119-5de7-4a57-bd3c-92722fc94654"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:51.123840623Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1564,"client":"1","":"downstairs","session_id":"59a45119-5de7-4a57-bd3c-92722fc94654"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:51.123856805Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1564,"session_id":"59a45119-5de7-4a57-bd3c-92722fc94654"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:51.123899321Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1564,"client":"0","":"downstairs","session_id":"59a45119-5de7-4a57-bd3c-92722fc94654"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:51.123917543Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1564,"client":"0","":"downstairs","session_id":"59a45119-5de7-4a57-bd3c-92722fc94654"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:51.123931885Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1564,"session_id":"59a45119-5de7-4a57-bd3c-92722fc94654"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:51.123944357Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1564,"client":"2","":"downstairs","session_id":"59a45119-5de7-4a57-bd3c-92722fc94654"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:51.123957738Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1564,"client":"2","":"downstairs","session_id":"59a45119-5de7-4a57-bd3c-92722fc94654"} {"msg":"[0]R flush_numbers[0..12]: [149, 151, 150, 147, 138, 141, 142, 147, 151, 150, 150, 147]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:51.1239726Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1564,"":"downstairs","":"downstairs","session_id":"59a45119-5de7-4a57-bd3c-92722fc94654"} {"msg":"[0]R generation[0..12]: [94, 95, 94, 93, 88, 89, 90, 93, 95, 94, 94, 93]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:51.123987572Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1564,"":"downstairs","":"downstairs","session_id":"59a45119-5de7-4a57-bd3c-92722fc94654"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:51.124017526Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1564,"":"downstairs","":"downstairs","session_id":"59a45119-5de7-4a57-bd3c-92722fc94654"} {"msg":"[1]R flush_numbers[0..12]: [149, 151, 150, 147, 138, 141, 142, 147, 151, 150, 150, 147]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:51.124033568Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1564,"":"downstairs","":"downstairs","session_id":"59a45119-5de7-4a57-bd3c-92722fc94654"} {"msg":"[1]R generation[0..12]: [94, 95, 94, 93, 88, 89, 90, 93, 95, 94, 94, 93]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:51.12404792Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1564,"":"downstairs","":"downstairs","session_id":"59a45119-5de7-4a57-bd3c-92722fc94654"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:51.124062162Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1564,"":"downstairs","":"downstairs","session_id":"59a45119-5de7-4a57-bd3c-92722fc94654"} {"msg":"[2]R flush_numbers[0..12]: [149, 151, 150, 147, 138, 141, 142, 147, 151, 150, 150, 147]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:51.124075794Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1564,"":"downstairs","":"downstairs","session_id":"59a45119-5de7-4a57-bd3c-92722fc94654"} {"msg":"[2]R generation[0..12]: [94, 95, 94, 93, 88, 89, 90, 93, 95, 94, 94, 93]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:51.124090505Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1564,"":"downstairs","":"downstairs","session_id":"59a45119-5de7-4a57-bd3c-92722fc94654"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:51.124107978Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1564,"":"downstairs","":"downstairs","session_id":"59a45119-5de7-4a57-bd3c-92722fc94654"} {"msg":"Max found gen is 96","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:51.12412234Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1564,"":"downstairs","":"downstairs","session_id":"59a45119-5de7-4a57-bd3c-92722fc94654"} {"msg":"Generation requested: 96 >= found:96","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:51.124135691Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1564,"":"downstairs","":"downstairs","session_id":"59a45119-5de7-4a57-bd3c-92722fc94654"} {"msg":"Next flush: 152","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:51.124148663Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1564,"":"downstairs","":"downstairs","session_id":"59a45119-5de7-4a57-bd3c-92722fc94654"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:51.124163125Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1564,"":"downstairs","":"downstairs","session_id":"59a45119-5de7-4a57-bd3c-92722fc94654"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:51.124176426Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1564,"session_id":"59a45119-5de7-4a57-bd3c-92722fc94654"} Wait for a query_work_queue command to finish before sending IO {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:51.124195669Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1564,"client":"0","":"downstairs","session_id":"59a45119-5de7-4a57-bd3c-92722fc94654"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:51.124209101Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1564,"client":"1","":"downstairs","session_id":"59a45119-5de7-4a57-bd3c-92722fc94654"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:51.124224233Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1564,"client":"2","":"downstairs","session_id":"59a45119-5de7-4a57-bd3c-92722fc94654"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:51.124237714Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1564,"session_id":"59a45119-5de7-4a57-bd3c-92722fc94654"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:51.124249896Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1564,"session_id":"59a45119-5de7-4a57-bd3c-92722fc94654"} {"msg":"47aced27-9504-4448-9f82-19ca52b2da51 is now active with session: 59a45119-5de7-4a57-bd3c-92722fc94654","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:51.124262687Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1564,"session_id":"59a45119-5de7-4a57-bd3c-92722fc94654"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:51.124274739Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1564,"session_id":"59a45119-5de7-4a57-bd3c-92722fc94654"} {"msg":"The guest has finished waiting for activation with:96","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:51.124286761Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1564} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 24 len 40960 data: 9 9 7 8 9 8 8 10 11 9 02/30 Read block 465 len 32768 03/30 Write block 18 len 12288 data: 8 8 6 04/30 Read block 313 len 28672 05/30 Write block 581 len 32768 data: 8 7 8 9 11 10 13 11 06/30 Write block 456 len 16384 data: 4 3 3 3 07/30 Write block 23 len 28672 data: 7 10 10 8 9 10 9 08/30 Write block 384 len 4096 data: 7 09/30 Read block 497 len 20480 10/30 Write block 20 len 40960 data: 7 4 5 8 11 11 9 10 11 10 11/30 Write block 319 len 12288 data: 10 9 7 12/30 Write block 267 len 24576 data: 9 9 11 10 12 10 13/30 Write block 4 len 20480 data: 8 8 10 9 10 14/30 Flush 15/30 Flush 16/30 Write block 486 len 12288 data: 10 12 13 17/30 Read block 480 len 36864 18/30 Read block 368 len 20480 19/30 Write block 496 len 12288 data: 6 7 9 20/30 Read block 356 len 32768 21/30 Write block 538 len 8192 data: 9 8 22/30 Read block 71 len 8192 23/30 Read block 442 len 32768 24/30 Read block 304 len 40960 25/30 Read block 343 len 16384 26/30 Flush 27/30 Write block 248 len 16384 data: 13 13 14 13 28/30 Write block 67 len 4096 data: 12 29/30 Read block 483 len 16384 30/30 Read block 445 len 4096 ---------------------------------------------------------------- Crucible gen:96 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 4 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Read 4 Done Done Done false 1029 Acked Read 1 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 0 1 Done 3 4 4 11 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:53:51.147979243Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1564} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 96 96 96 152 152 152 F F F 1 020-039 96 96 96 152 152 152 F F F 2 040-059 94 94 94 150 150 150 F F F 3 060-079 93 93 93 147 147 147 T T T 4 080-099 88 88 88 138 138 138 F F F 5 100-119 89 89 89 141 141 141 F F F 6 120-139 90 90 90 142 142 142 F F F 7 140-159 93 93 93 147 147 147 F F F 8 160-179 95 95 95 151 151 151 F F F 9 180-199 94 94 94 150 150 150 F F F 10 200-219 94 94 94 150 150 150 F F F 11 220-239 93 93 93 147 147 147 F F F 12 240-259 94 94 94 149 149 149 T T T 13 260-279 96 96 96 152 152 152 F F F 14 280-299 90 90 90 142 142 142 F F F 15 300-319 96 96 96 152 152 152 F F F 16 320-339 96 96 96 152 152 152 F F F 17 340-359 94 94 94 150 150 150 F F F 18 360-379 93 93 93 147 147 147 F F F 19 380-399 96 96 96 152 152 152 F F F 20 400-419 95 95 95 151 151 151 F F F 21 420-439 95 95 95 151 151 151 F F F 22 440-459 96 96 96 152 152 152 F F F 23 460-479 93 93 93 147 147 147 F F F 24 480-499 96 96 96 154 154 154 F F F 25 500-519 90 90 90 142 142 142 F F F 26 520-539 96 96 96 154 154 154 F F F 27 540-559 90 90 90 142 142 142 F F F 28 560-579 95 95 95 151 151 151 F F F 29 580-599 96 96 96 152 152 152 F F F Max gen: 96, Max flush: 154 A Difference in extent metadata is expected here On loop 48 Verifying data now No change after verify Loop: 48 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 96 96 96 152 152 152 F F F 1 020-039 96 96 96 152 152 152 F F F 2 040-059 94 94 94 150 150 150 F F F 3 060-079 97 97 97 155 155 155 F F F 4 080-099 88 88 88 138 138 138 F F F 5 100-119 89 89 89 141 141 141 F F F 6 120-139 90 90 90 142 142 142 F F F 7 140-159 93 93 93 147 147 147 F F F 8 160-179 95 95 95 151 151 151 F F F 9 180-199 94 94 94 150 150 150 F F F 10 200-219 94 94 94 150 150 150 F F F 11 220-239 93 93 93 147 147 147 F F F 12 240-259 97 97 97 155 155 155 F F F 13 260-279 96 96 96 152 152 152 F F F 14 280-299 90 90 90 142 142 142 F F F 15 300-319 96 96 96 152 152 152 F F F 16 320-339 96 96 96 152 152 152 F F F 17 340-359 94 94 94 150 150 150 F F F 18 360-379 93 93 93 147 147 147 F F F 19 380-399 96 96 96 152 152 152 F F F 20 400-419 95 95 95 151 151 151 F F F 21 420-439 95 95 95 151 151 151 F F F 22 440-459 96 96 96 152 152 152 F F F 23 460-479 93 93 93 147 147 147 F F F 24 480-499 96 96 96 154 154 154 F F F 25 500-519 90 90 90 142 142 142 F F F 26 520-539 96 96 96 154 154 154 F F F 27 540-559 90 90 90 142 142 142 F F F 28 560-579 95 95 95 151 151 151 F F F 29 580-599 96 96 96 152 152 152 F F F Max gen: 97, Max flush: 155 Begin loop 49 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:55.411686029Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1572,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:53:55.454419606Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1572,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:55.454825018Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1572,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:55.486797438Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1572,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:55.487321045Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1572} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"3818f0fc76fe787c6cc412af30fd59b930203885\",\n git_commit_timestamp: \"2024-11-01T16:08:30.000000000Z\",\n git_branch: \"renovate/hyper-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:55.487366801Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1572} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:55.487385173Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1572} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:55.487403236Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1572} {"msg":"Crucible 607ea399-10f4-4d00-a690-b50c3242c0a8 has session id: f6b6e6a7-ea39-459e-a230-7b58ec5ff1eb","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:55.487843102Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1572,"session_id":"f6b6e6a7-ea39-459e-a230-7b58ec5ff1eb"} {"msg":"Upstairs opts: Upstairs UUID: 607ea399-10f4-4d00-a690-b50c3242c0a8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:55.487885838Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1572,"session_id":"f6b6e6a7-ea39-459e-a230-7b58ec5ff1eb"} {"msg":"Crucible stats registered with UUID: 607ea399-10f4-4d00-a690-b50c3242c0a8","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:55.487909831Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1572,"session_id":"f6b6e6a7-ea39-459e-a230-7b58ec5ff1eb"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:55.519147056Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1572,"task":"crutest"} {"msg":"The guest has requested activation with gen:98","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:55.5191793Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1572} {"msg":"607ea399-10f4-4d00-a690-b50c3242c0a8 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:55.519248119Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1572,"session_id":"f6b6e6a7-ea39-459e-a230-7b58ec5ff1eb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:55.519271532Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1572,"client":"0","":"downstairs","session_id":"f6b6e6a7-ea39-459e-a230-7b58ec5ff1eb"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:55.519301015Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1572,"client":"0","":"downstairs","session_id":"f6b6e6a7-ea39-459e-a230-7b58ec5ff1eb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:55.519318698Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1572,"client":"1","":"downstairs","session_id":"f6b6e6a7-ea39-459e-a230-7b58ec5ff1eb"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:55.51933601Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1572,"client":"1","":"downstairs","session_id":"f6b6e6a7-ea39-459e-a230-7b58ec5ff1eb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:55.519353812Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1572,"client":"2","":"downstairs","session_id":"f6b6e6a7-ea39-459e-a230-7b58ec5ff1eb"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:55.519370564Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1572,"client":"2","":"downstairs","session_id":"f6b6e6a7-ea39-459e-a230-7b58ec5ff1eb"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:55.519386997Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1572,"":"io task","client":"2","":"downstairs","session_id":"f6b6e6a7-ea39-459e-a230-7b58ec5ff1eb"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:55.519404379Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1572,"":"io task","client":"0","":"downstairs","session_id":"f6b6e6a7-ea39-459e-a230-7b58ec5ff1eb"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:55.519421581Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1572,"":"io task","client":"1","":"downstairs","session_id":"f6b6e6a7-ea39-459e-a230-7b58ec5ff1eb"} {"msg":"ds_connection connected from Ok(127.0.0.1:64160)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:55.519578681Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1572,"":"io task","client":"0","":"downstairs","session_id":"f6b6e6a7-ea39-459e-a230-7b58ec5ff1eb"} {"msg":"ds_connection connected from Ok(127.0.0.1:58266)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:55.519746853Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1572,"":"io task","client":"2","":"downstairs","session_id":"f6b6e6a7-ea39-459e-a230-7b58ec5ff1eb"} {"msg":"ds_connection connected from Ok(127.0.0.1:57791)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:55.519794519Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1572,"":"io task","client":"1","":"downstairs","session_id":"f6b6e6a7-ea39-459e-a230-7b58ec5ff1eb"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:55.520490999Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1572,"client":"2","":"downstairs","session_id":"f6b6e6a7-ea39-459e-a230-7b58ec5ff1eb"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:55.520518662Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1572,"client":"2","":"downstairs","session_id":"f6b6e6a7-ea39-459e-a230-7b58ec5ff1eb"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:55.520534284Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1572,"client":"0","":"downstairs","session_id":"f6b6e6a7-ea39-459e-a230-7b58ec5ff1eb"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:55.520548426Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1572,"client":"0","":"downstairs","session_id":"f6b6e6a7-ea39-459e-a230-7b58ec5ff1eb"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:55.520562248Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1572,"client":"1","":"downstairs","session_id":"f6b6e6a7-ea39-459e-a230-7b58ec5ff1eb"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:55.520584071Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1572,"client":"1","":"downstairs","session_id":"f6b6e6a7-ea39-459e-a230-7b58ec5ff1eb"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:55.521221553Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1572,"client":"1","":"downstairs","session_id":"f6b6e6a7-ea39-459e-a230-7b58ec5ff1eb"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:55.521253137Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1572,"client":"0","":"downstairs","session_id":"f6b6e6a7-ea39-459e-a230-7b58ec5ff1eb"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:55.521271539Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1572,"client":"2","":"downstairs","session_id":"f6b6e6a7-ea39-459e-a230-7b58ec5ff1eb"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:55.521795057Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1572,"client":"0","":"downstairs","session_id":"f6b6e6a7-ea39-459e-a230-7b58ec5ff1eb"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:55.521825561Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1572,"client":"0","":"downstairs","session_id":"f6b6e6a7-ea39-459e-a230-7b58ec5ff1eb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:55.521843443Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1572,"session_id":"f6b6e6a7-ea39-459e-a230-7b58ec5ff1eb"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:55.521907651Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1572,"client":"1","":"downstairs","session_id":"f6b6e6a7-ea39-459e-a230-7b58ec5ff1eb"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:55.521940046Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1572,"client":"1","":"downstairs","session_id":"f6b6e6a7-ea39-459e-a230-7b58ec5ff1eb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:55.521955668Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1572,"session_id":"f6b6e6a7-ea39-459e-a230-7b58ec5ff1eb"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:55.521968619Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1572,"client":"2","":"downstairs","session_id":"f6b6e6a7-ea39-459e-a230-7b58ec5ff1eb"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:55.521982161Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1572,"client":"2","":"downstairs","session_id":"f6b6e6a7-ea39-459e-a230-7b58ec5ff1eb"} {"msg":"[0]R flush_numbers[0..12]: [152, 152, 150, 155, 138, 141, 142, 147, 151, 150, 150, 147]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:55.521997103Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1572,"":"downstairs","":"downstairs","session_id":"f6b6e6a7-ea39-459e-a230-7b58ec5ff1eb"} {"msg":"[0]R generation[0..12]: [96, 96, 94, 97, 88, 89, 90, 93, 95, 94, 94, 93]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:55.522011085Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1572,"":"downstairs","":"downstairs","session_id":"f6b6e6a7-ea39-459e-a230-7b58ec5ff1eb"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:55.522025326Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1572,"":"downstairs","":"downstairs","session_id":"f6b6e6a7-ea39-459e-a230-7b58ec5ff1eb"} {"msg":"[1]R flush_numbers[0..12]: [152, 152, 150, 155, 138, 141, 142, 147, 151, 150, 150, 147]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:55.522039218Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1572,"":"downstairs","":"downstairs","session_id":"f6b6e6a7-ea39-459e-a230-7b58ec5ff1eb"} {"msg":"[1]R generation[0..12]: [96, 96, 94, 97, 88, 89, 90, 93, 95, 94, 94, 93]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:55.52205272Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1572,"":"downstairs","":"downstairs","session_id":"f6b6e6a7-ea39-459e-a230-7b58ec5ff1eb"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:55.522066872Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1572,"":"downstairs","":"downstairs","session_id":"f6b6e6a7-ea39-459e-a230-7b58ec5ff1eb"} {"msg":"[2]R flush_numbers[0..12]: [152, 152, 150, 155, 138, 141, 142, 147, 151, 150, 150, 147]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:55.522080894Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1572,"":"downstairs","":"downstairs","session_id":"f6b6e6a7-ea39-459e-a230-7b58ec5ff1eb"} {"msg":"[2]R generation[0..12]: [96, 96, 94, 97, 88, 89, 90, 93, 95, 94, 94, 93]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:55.522094755Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1572,"":"downstairs","":"downstairs","session_id":"f6b6e6a7-ea39-459e-a230-7b58ec5ff1eb"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:55.522108707Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1572,"":"downstairs","":"downstairs","session_id":"f6b6e6a7-ea39-459e-a230-7b58ec5ff1eb"} {"msg":"Max found gen is 98","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:55.522121979Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1572,"":"downstairs","":"downstairs","session_id":"f6b6e6a7-ea39-459e-a230-7b58ec5ff1eb"} {"msg":"Generation requested: 98 >= found:98","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:55.522137411Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1572,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO f6b6e6a7-ea39-459e-a230-7b58ec5ff1eb"} {"msg":"Next flush: 156","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:55.522158604Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1572,"":"downstairs","":"downstairs","session_id":"f6b6e6a7-ea39-459e-a230-7b58ec5ff1eb"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:55.522172005Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1572,"":"downstairs","":"downstairs","session_id":"f6b6e6a7-ea39-459e-a230-7b58ec5ff1eb"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:55.522185117Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1572,"session_id":"f6b6e6a7-ea39-459e-a230-7b58ec5ff1eb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:55.522197179Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1572,"client":"0","":"downstairs","session_id":"f6b6e6a7-ea39-459e-a230-7b58ec5ff1eb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:55.5222104Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1572,"client":"1","":"downstairs","session_id":"f6b6e6a7-ea39-459e-a230-7b58ec5ff1eb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:55.522223782Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1572,"client":"2","":"downstairs","session_id":"f6b6e6a7-ea39-459e-a230-7b58ec5ff1eb"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:55.522237464Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1572,"session_id":"f6b6e6a7-ea39-459e-a230-7b58ec5ff1eb"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:55.522249965Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1572,"session_id":"f6b6e6a7-ea39-459e-a230-7b58ec5ff1eb"} {"msg":"607ea399-10f4-4d00-a690-b50c3242c0a8 is now active with session: f6b6e6a7-ea39-459e-a230-7b58ec5ff1eb","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:55.522262767Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1572,"session_id":"f6b6e6a7-ea39-459e-a230-7b58ec5ff1eb"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:55.522274789Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1572,"session_id":"f6b6e6a7-ea39-459e-a230-7b58ec5ff1eb"} {"msg":"The guest has finished waiting for activation with:98","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:55.52228698Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1572} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 349 len 40960 data: 10 9 10 12 11 8 7 6 8 8 02/30 Read block 274 len 20480 03/30 Read block 282 len 36864 04/30 Write block 164 len 28672 data: 7 8 6 5 6 6 7 05/30 Write block 113 len 20480 data: 8 8 7 6 7 06/30 Read block 456 len 24576 07/30 Read block 315 len 16384 08/30 Write block 6 len 36864 data: 11 10 11 10 10 11 13 12 13 09/30 Write block 378 len 12288 data: 7 7 6 10/30 Write block 273 len 12288 data: 11 14 13 11/30 Flush 12/30 Write block 483 len 32768 data: 7 10 11 11 13 14 12 11 13/30 Read block 290 len 40960 14/30 Write block 290 len 40960 data: 4 6 9 9 11 8 9 8 7 7 15/30 Write block 39 len 36864 data: 10 8 9 10 13 12 10 12 8 16/30 Read block 361 len 16384 17/30 Write block 487 len 4096 data: 14 18/30 Read block 233 len 24576 19/30 Read block 225 len 16384 20/30 Read block 571 len 36864 21/30 Read block 180 len 16384 22/30 Read block 219 len 28672 23/30 Write block 587 len 4096 data: 14 24/30 Write block 138 len 20480 data: 9 7 6 6 6 25/30 Write block 481 len 20480 data: 9 8 8 11 12 26/30 Read block 430 len 32768 27/30 Read block 135 len 24576 28/30 Write block 426 len 12288 data: 11 11 12 29/30 Read block 98 len 12288 30/30 Write block 109 len 28672 data: 9 10 10 9 9 9 8 ---------------------------------------------------------------- Crucible gen:98 GIO:true work queues: Upstairs:0 downstairs:19 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1011 Acked Write 8 Done Done Done false 1012 Acked Read 10 Done Done Done false 1013 Acked Write 10 Done Done Done false 1014 Acked Write 9 Done Done Done false 1015 Acked Read 4 Done Done Done false 1016 Acked Write 1 Done Done Done false 1017 Acked Read 6 Done Done Done false 1018 Acked Read 4 Done Done Done false 1019 Acked Read 9 Done Done Done false 1020 Acked Read 4 Done Done Done false 1021 Acked Read 7 Done Done Done false 1022 Acked Write 1 Done Done Done false 1023 Acked Write 5 Done Done Done false 1024 Acked Write 5 Done Done Done false 1025 Acked Read 8 Done Done Done false 1026 Acked Read 6 Done Done Done false 1027 Acked Write 3 Done Done Done false 1028 Acked Read 3 Done Done Done false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 18 18 18 54 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1010 1010 1010 Downstairs last five completed: 1010 1009 1008 1007 1006 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:53:55.544592814Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1572} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 98 98 98 156 156 156 F F F 1 020-039 96 96 96 152 152 152 T T T 2 040-059 94 94 94 150 150 150 T T T 3 060-079 97 97 97 155 155 155 F F F 4 080-099 88 88 88 138 138 138 F F F 5 100-119 98 98 98 156 156 156 T T T 6 120-139 90 90 90 142 142 142 T T T 7 140-159 93 93 93 147 147 147 T T T 8 160-179 98 98 98 156 156 156 F F F 9 180-199 94 94 94 150 150 150 F F F 10 200-219 94 94 94 150 150 150 F F F 11 220-239 93 93 93 147 147 147 F F F 12 240-259 97 97 97 155 155 155 F F F 13 260-279 98 98 98 156 156 156 F F F 14 280-299 90 90 90 142 142 142 T T T 15 300-319 96 96 96 152 152 152 F F F 16 320-339 96 96 96 152 152 152 F F F 17 340-359 98 98 98 156 156 156 F F F 18 360-379 98 98 98 156 156 156 F F F 19 380-399 98 98 98 156 156 156 F F F 20 400-419 95 95 95 151 151 151 F F F 21 420-439 95 95 95 151 151 151 T T T 22 440-459 96 96 96 152 152 152 F F F 23 460-479 93 93 93 147 147 147 F F F 24 480-499 96 96 96 154 154 154 T T T 25 500-519 90 90 90 142 142 142 F F F 26 520-539 96 96 96 154 154 154 F F F 27 540-559 90 90 90 142 142 142 F F F 28 560-579 95 95 95 151 151 151 F F F 29 580-599 96 96 96 152 152 152 T T T Max gen: 98, Max flush: 156 A Difference in extent metadata is expected here On loop 49 Verifying data now No change after verify Loop: 49 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 98 98 98 156 156 156 F F F 1 020-039 99 99 99 157 157 157 F F F 2 040-059 99 99 99 157 157 157 F F F 3 060-079 97 97 97 155 155 155 F F F 4 080-099 88 88 88 138 138 138 F F F 5 100-119 99 99 99 157 157 157 F F F 6 120-139 99 99 99 157 157 157 F F F 7 140-159 99 99 99 157 157 157 F F F 8 160-179 98 98 98 156 156 156 F F F 9 180-199 94 94 94 150 150 150 F F F 10 200-219 94 94 94 150 150 150 F F F 11 220-239 93 93 93 147 147 147 F F F 12 240-259 97 97 97 155 155 155 F F F 13 260-279 98 98 98 156 156 156 F F F 14 280-299 99 99 99 157 157 157 F F F 15 300-319 96 96 96 152 152 152 F F F 16 320-339 96 96 96 152 152 152 F F F 17 340-359 98 98 98 156 156 156 F F F 18 360-379 98 98 98 156 156 156 F F F 19 380-399 98 98 98 156 156 156 F F F 20 400-419 95 95 95 151 151 151 F F F 21 420-439 99 99 99 157 157 157 F F F 22 440-459 96 96 96 152 152 152 F F F 23 460-479 93 93 93 147 147 147 F F F 24 480-499 99 99 99 157 157 157 F F F 25 500-519 90 90 90 142 142 142 F F F 26 520-539 96 96 96 154 154 154 F F F 27 540-559 90 90 90 142 142 142 F F F 28 560-579 95 95 95 151 151 151 F F F 29 580-599 99 99 99 157 157 157 F F F Max gen: 99, Max flush: 157 Begin loop 50 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:59.930810656Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1580,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:59.974486073Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1580,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:59.975051276Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1580} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"3818f0fc76fe787c6cc412af30fd59b930203885\",\n git_commit_timestamp: \"2024-11-01T16:08:30.000000000Z\",\n git_branch: \"renovate/hyper-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:59.975092351Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1580} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:59.975107933Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1580} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:59.975123265Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1580} {"msg":"Crucible f3b92130-0639-43bf-a256-99eb77129460 has session id: caaef4e2-174b-4b2f-b36a-f14b7dfabd13","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:59.975606848Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1580,"session_id":"caaef4e2-174b-4b2f-b36a-f14b7dfabd13"} {"msg":"Upstairs opts: Upstairs UUID: f3b92130-0639-43bf-a256-99eb77129460, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:59.975651053Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1580,"session_id":"caaef4e2-174b-4b2f-b36a-f14b7dfabd13"} {"msg":"Crucible stats registered with UUID: f3b92130-0639-43bf-a256-99eb77129460","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:59.975672576Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1580,"session_id":"caaef4e2-174b-4b2f-b36a-f14b7dfabd13"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:00.006347649Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1580,"task":"crutest"} {"msg":"The guest has requested activation with gen:100","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:00.006379583Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1580} {"msg":"f3b92130-0639-43bf-a256-99eb77129460 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:00.006416067Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1580,"session_id":"caaef4e2-174b-4b2f-b36a-f14b7dfabd13"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:00.0064329Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1580,"client":"0","":"downstairs","session_id":"caaef4e2-174b-4b2f-b36a-f14b7dfabd13"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:00.006461783Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1580,"client":"0","":"downstairs","session_id":"caaef4e2-174b-4b2f-b36a-f14b7dfabd13"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:00.006479455Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1580,"client":"1","":"downstairs","session_id":"caaef4e2-174b-4b2f-b36a-f14b7dfabd13"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:00.006496448Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1580,"client":"1","":"downstairs","session_id":"caaef4e2-174b-4b2f-b36a-f14b7dfabd13"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:00.00651286Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1580,"client":"2","":"downstairs","session_id":"caaef4e2-174b-4b2f-b36a-f14b7dfabd13"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:00.006531442Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1580,"client":"2","":"downstairs","session_id":"caaef4e2-174b-4b2f-b36a-f14b7dfabd13"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:00.006548324Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1580,"":"io task","client":"2","":"downstairs","session_id":"caaef4e2-174b-4b2f-b36a-f14b7dfabd13"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:00.006565566Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1580,"":"io task","client":"0","":"downstairs","session_id":"caaef4e2-174b-4b2f-b36a-f14b7dfabd13"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:00.0065922Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1580,"":"io task","client":"1","":"downstairs","session_id":"caaef4e2-174b-4b2f-b36a-f14b7dfabd13"} {"msg":"ds_connection connected from Ok(127.0.0.1:64399)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:00.006628375Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1580,"":"io task","client":"2","":"downstairs","session_id":"caaef4e2-174b-4b2f-b36a-f14b7dfabd13"} {"msg":"ds_connection connected from Ok(127.0.0.1:62379)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:00.006688622Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1580,"":"io task","client":"1","":"downstairs","session_id":"caaef4e2-174b-4b2f-b36a-f14b7dfabd13"} {"msg":"ds_connection connected from Ok(127.0.0.1:46837)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:00.006732528Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1580,"":"io task","client":"0","":"downstairs","session_id":"caaef4e2-174b-4b2f-b36a-f14b7dfabd13"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:00.007495797Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1580,"client":"2","":"downstairs","session_id":"caaef4e2-174b-4b2f-b36a-f14b7dfabd13"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:00.007534902Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1580,"client":"2","":"downstairs","session_id":"caaef4e2-174b-4b2f-b36a-f14b7dfabd13"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:00.007565365Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1580,"client":"1","":"downstairs","session_id":"caaef4e2-174b-4b2f-b36a-f14b7dfabd13"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:00.007594629Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1580,"client":"1","":"downstairs","session_id":"caaef4e2-174b-4b2f-b36a-f14b7dfabd13"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:00.007617562Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1580,"client":"0","":"downstairs","session_id":"caaef4e2-174b-4b2f-b36a-f14b7dfabd13"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:00.007637735Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1580,"client":"0","":"downstairs","session_id":"caaef4e2-174b-4b2f-b36a-f14b7dfabd13"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:00.008161272Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1580,"client":"1","":"downstairs","session_id":"caaef4e2-174b-4b2f-b36a-f14b7dfabd13"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:00.008190546Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1580,"client":"2","":"downstairs","session_id":"caaef4e2-174b-4b2f-b36a-f14b7dfabd13"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:00.008207578Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1580,"client":"0","":"downstairs","session_id":"caaef4e2-174b-4b2f-b36a-f14b7dfabd13"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:00.008726455Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1580,"client":"2","":"downstairs","session_id":"caaef4e2-174b-4b2f-b36a-f14b7dfabd13"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:00.008754729Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1580,"client":"2","":"downstairs","session_id":"caaef4e2-174b-4b2f-b36a-f14b7dfabd13"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:00.008770281Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1580,"session_id":"caaef4e2-174b-4b2f-b36a-f14b7dfabd13"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:00.008832889Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1580,"client":"1","":"downstairs","session_id":"caaef4e2-174b-4b2f-b36a-f14b7dfabd13"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:00.008853011Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1580,"client":"1","":"downstairs","session_id":"caaef4e2-174b-4b2f-b36a-f14b7dfabd13"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:00.008870634Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1580,"session_id":"caaef4e2-174b-4b2f-b36a-f14b7dfabd13"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:00.008883515Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1580,"client":"0","":"downstairs","session_id":"caaef4e2-174b-4b2f-b36a-f14b7dfabd13"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:00.008897087Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1580,"client":"0","":"downstairs","session_id":"caaef4e2-174b-4b2f-b36a-f14b7dfabd13"} {"msg":"[0]R flush_numbers[0..12]: [156, 157, 157, 155, 138, 157, 157, 157, 156, 150, 150, 147]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:00.008911089Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1580,"":"downstairs","":"downstairs","session_id":"caaef4e2-174b-4b2f-b36a-f14b7dfabd13"} {"msg":"[0]R generation[0..12]: [98, 99, 99, 97, 88, 99, 99, 99, 98, 94, 94, 93]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:00.008925061Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1580,"":"downstairs","":"downstairs","session_id":"caaef4e2-174b-4b2f-b36a-f14b7dfabd13"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:00.008939203Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1580,"":"downstairs","":"downstairs","session_id":"caaef4e2-174b-4b2f-b36a-f14b7dfabd13"} {"msg":"[1]R flush_numbers[0..12]: [156, 157, 157, 155, 138, 157, 157, 157, 156, 150, 150, 147]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:00.008953134Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1580,"":"downstairs","":"downstairs","session_id":"caaef4e2-174b-4b2f-b36a-f14b7dfabd13"} {"msg":"[1]R generation[0..12]: [98, 99, 99, 97, 88, 99, 99, 99, 98, 94, 94, 93]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:00.008966846Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1580,"":"downstairs","":"downstairs","session_id":"caaef4e2-174b-4b2f-b36a-f14b7dfabd13"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:00.008981178Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1580,"":"downstairs","":"downstairs","session_id":"caaef4e2-174b-4b2f-b36a-f14b7dfabd13"} {"msg":"[2]R flush_numbers[0..12]: [156, 157, 157, 155, 138, 157, 157, 157, 156, 150, 150, 147]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:00.00899535Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1580,"":"downstairs","":"downstairs","session_id":"caaef4e2-174b-4b2f-b36a-f14b7dfabd13"} {"msg":"[2]R generation[0..12]: [98, 99, 99, 97, 88, 99, 99, 99, 98, 94, 94, 93]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:00.009009281Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1580,"":"downstairs","":"downstairs","session_id":"caaef4e2-174b-4b2f-b36a-f14b7dfabd13"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:00.009023253Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1580,"":"downstairs","":"downstairs","session_id":"caaef4e2-174b-4b2f-b36a-f14b7dfabd13"} {"msg":"Max found gen is 100","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:00.009036285Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1580,"":"downstairs","":"downstairs","session_id":"caaef4e2-174b-4b2f-b36a-f14b7dfabd13"} {"msg":"Generation requested: 100 >= found:100","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:00.009049747Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1580,"":"downstairs","":"downstairs","session_id":"caaef4e2-174b-4b2f-b36a-f14b7dfabd13"} {"msg":"Next flush: 158","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:00.009065839Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1580,"":"downstairs","":"downstairs","session_id":"caaef4e2-174b-4b2f-b36a-f14b7dfabd13"} {"msg":"All extents match","v":0,"name":"Wait for a query_work_queue command to finish before sending IO crucible","level":30,"time":"2024-11-01T16:54:00.009087102Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1580,"":"downstairs","":"downstairs","session_id":"caaef4e2-174b-4b2f-b36a-f14b7dfabd13"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:00.009101573Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1580,"session_id":"caaef4e2-174b-4b2f-b36a-f14b7dfabd13"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:00.009113915Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1580,"client":"0","":"downstairs","session_id":"caaef4e2-174b-4b2f-b36a-f14b7dfabd13"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:00.009127497Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1580,"client":"1","":"downstairs","session_id":"caaef4e2-174b-4b2f-b36a-f14b7dfabd13"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:00.009140958Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1580,"client":"2","":"downstairs","session_id":"caaef4e2-174b-4b2f-b36a-f14b7dfabd13"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:00.00915451Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1580,"session_id":"caaef4e2-174b-4b2f-b36a-f14b7dfabd13"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:00.009167102Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1580,"session_id":"caaef4e2-174b-4b2f-b36a-f14b7dfabd13"} {"msg":"f3b92130-0639-43bf-a256-99eb77129460 is now active with session: caaef4e2-174b-4b2f-b36a-f14b7dfabd13","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:00.009179923Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1580,"session_id":"caaef4e2-174b-4b2f-b36a-f14b7dfabd13"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:00.009192005Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1580,"session_id":"caaef4e2-174b-4b2f-b36a-f14b7dfabd13"} {"msg":"The guest has finished waiting for activation with:100","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:00.009204326Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1580} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 551 len 16384 data: 12 12 12 12 02/30 Read block 465 len 28672 03/30 Write block 250 len 12288 data: 15 14 12 04/30 Read block 552 len 24576 05/30 Read block 584 len 28672 06/30 Read block 479 len 16384 07/30 Read block 280 len 28672 08/30 Read block 478 len 36864 09/30 Read block 292 len 36864 10/30 Write block 533 len 28672 data: 11 13 13 12 10 10 9 11/30 Write block 174 len 40960 data: 6 8 9 10 9 8 7 5 5 6 12/30 Read block 543 len 36864 13/30 Read block 453 len 12288 14/30 Read block 273 len 24576 15/30 Write block 517 len 24576 data: 4 4 5 5 4 5 16/30 Write block 25 len 20480 data: 12 10 11 12 11 17/30 Read block 19 len 28672 18/30 Read block 556 len 24576 19/30 Flush 20/30 Write block 427 len 12288 data: 12 13 11 21/30 Flush 22/30 Flush 23/30 Write block 197 len 8192 data: 10 9 24/30 Read block 557 len 24576 25/30 Read block 215 len 28672 26/30 Read block 321 len 20480 27/30 Write block 98 len 16384 data: 13 13 12 10 28/30 Write block 315 len 28672 data: 13 13 10 11 11 10 8 29/30 Write block 308 len 20480 data: 11 10 8 9 9 30/30 Read block 393 len 40960 ---------------------------------------------------------------- Crucible gen:100 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 2 Done Done Done false 1023 Acked Read 6 Done Done Done false 1024 Acked Read 7 Done Done Done false 1025 Acked Read 5 Done Done Done false 1026 Acked Write 4 Done Done Done false 1027 Acked Write 7 Done Done Done false 1028 Acked Write 5 Done Sent Done false 1029 Acked Read 10 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 2 1 3 Done 8 6 7 21 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1021 1021 1021 Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:54:00.033117058Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1580} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 98 98 98 156 156 156 F F F 1 020-039 100 100 100 158 158 158 F F F 2 040-059 99 99 99 157 157 157 F F F 3 060-079 97 97 97 155 155 155 F F F 4 080-099 88 88 88 138 138 138 T T T 5 100-119 99 99 99 157 157 157 T T T 6 120-139 99 99 99 157 157 157 F F F 7 140-159 99 99 99 157 157 157 F F F 8 160-179 100 100 100 158 158 158 F F F 9 180-199 100 100 100 158 158 158 T T T 10 200-219 94 94 94 150 150 150 F F F 11 220-239 93 93 93 147 147 147 F F F 12 240-259 100 100 100 158 158 158 F F F 13 260-279 98 98 98 156 156 156 F F F 14 280-299 99 99 99 157 157 157 F F F 15 300-319 96 96 96 152 152 152 T T T 16 320-339 96 96 96 152 152 152 T T T 17 340-359 98 98 98 156 156 156 F F F 18 360-379 98 98 98 156 156 156 F F F 19 380-399 98 98 98 156 156 156 F F F 20 400-419 95 95 95 151 151 151 F F F 21 420-439 100 100 100 159 159 159 F F F 22 440-459 96 96 96 152 152 152 F F F 23 460-479 93 93 93 147 147 147 F F F 24 480-499 99 99 99 157 157 157 F F F 25 500-519 100 100 100 158 158 158 F F F 26 520-539 100 100 100 158 158 158 F F F 27 540-559 100 100 100 158 158 158 F F F 28 560-579 95 95 95 151 151 151 F F F 29 580-599 99 99 99 157 157 157 F F F Max gen: 100, Max flush: 159 A Difference in extent metadata is expected here On loop 50 Verifying data now No change after verify Loop: 50 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 98 98 98 156 156 156 F F F 1 020-039 100 100 100 158 158 158 F F F 2 040-059 99 99 99 157 157 157 F F F 3 060-079 97 97 97 155 155 155 F F F 4 080-099 101 101 101 160 160 160 F F F 5 100-119 101 101 101 160 160 160 F F F 6 120-139 99 99 99 157 157 157 F F F 7 140-159 99 99 99 157 157 157 F F F 8 160-179 100 100 100 158 158 158 F F F 9 180-199 101 101 101 160 160 160 F F F 10 200-219 94 94 94 150 150 150 F F F 11 220-239 93 93 93 147 147 147 F F F 12 240-259 100 100 100 158 158 158 F F F 13 260-279 98 98 98 156 156 156 F F F 14 280-299 99 99 99 157 157 157 F F F 15 300-319 101 101 101 160 160 160 F F F 16 320-339 101 101 101 160 160 160 F F F 17 340-359 98 98 98 156 156 156 F F F 18 360-379 98 98 98 156 156 156 F F F 19 380-399 98 98 98 156 156 156 F F F 20 400-419 95 95 95 151 151 151 F F F 21 420-439 100 100 100 159 159 159 F F F 22 440-459 96 96 96 152 152 152 F F F 23 460-479 93 93 93 147 147 147 F F F 24 480-499 99 99 99 157 157 157 F F F 25 500-519 100 100 100 158 158 158 F F F 26 520-539 100 100 100 158 158 158 F F F 27 540-559 100 100 100 158 158 158 F F F 28 560-579 95 95 95 151 151 151 F F F 29 580-599 99 99 99 157 157 157 F F F Max gen: 101, Max flush: 160 Begin loop 51 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:04.343757343Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1588,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:04.387865306Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1588,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:04.388413387Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1588} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"3818f0fc76fe787c6cc412af30fd59b930203885\",\n git_commit_timestamp: \"2024-11-01T16:08:30.000000000Z\",\n git_branch: \"renovate/hyper-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:04.388452912Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1588} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:04.388468874Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1588} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:04.388483576Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1588} {"msg":"Crucible 8d030d0d-964d-4334-9aa6-0f554eec060a has session id: 19986c1d-7fa3-4ba6-bee7-122975a4f481","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:04.388915732Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1588,"session_id":"19986c1d-7fa3-4ba6-bee7-122975a4f481"} {"msg":"Upstairs opts: Upstairs UUID: 8d030d0d-964d-4334-9aa6-0f554eec060a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:04.388956377Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1588,"session_id":"19986c1d-7fa3-4ba6-bee7-122975a4f481"} {"msg":"Crucible stats registered with UUID: 8d030d0d-964d-4334-9aa6-0f554eec060a","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:04.3889804Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1588,"session_id":"19986c1d-7fa3-4ba6-bee7-122975a4f481"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:04.419599055Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1588,"task":"crutest"} {"msg":"The guest has requested activation with gen:102","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:04.419636Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1588} {"msg":"8d030d0d-964d-4334-9aa6-0f554eec060a active request set","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:04.419670264Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1588,"session_id":"19986c1d-7fa3-4ba6-bee7-122975a4f481"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:04.419686606Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1588,"client":"0","":"downstairs","session_id":"19986c1d-7fa3-4ba6-bee7-122975a4f481"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:04.41971516Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1588,"client":"0","":"downstairs","session_id":"19986c1d-7fa3-4ba6-bee7-122975a4f481"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:04.419733102Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1588,"client":"1","":"downstairs","session_id":"19986c1d-7fa3-4ba6-bee7-122975a4f481"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:04.419750374Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1588,"client":"1","":"downstairs","session_id":"19986c1d-7fa3-4ba6-bee7-122975a4f481"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:04.419767347Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1588,"client":"2","":"downstairs","session_id":"19986c1d-7fa3-4ba6-bee7-122975a4f481"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:04.419785919Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1588,"client":"2","":"downstairs","session_id":"19986c1d-7fa3-4ba6-bee7-122975a4f481"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:04.41986973Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1588,"":"io task","client":"2","":"downstairs","session_id":"19986c1d-7fa3-4ba6-bee7-122975a4f481"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:04.419904614Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1588,"":"io task","client":"0","":"downstairs","session_id":"19986c1d-7fa3-4ba6-bee7-122975a4f481"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:04.419934168Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1588,"":"io task","client":"1","":"downstairs","session_id":"19986c1d-7fa3-4ba6-bee7-122975a4f481"} {"msg":"ds_connection connected from Ok(127.0.0.1:36398)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:04.419967713Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1588,"":"io task","client":"2","":"downstairs","session_id":"19986c1d-7fa3-4ba6-bee7-122975a4f481"} {"msg":"ds_connection connected from Ok(127.0.0.1:61862)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:04.420225786Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1588,"":"io task","client":"1","":"downstairs","session_id":"19986c1d-7fa3-4ba6-bee7-122975a4f481"} {"msg":"ds_connection connected from Ok(127.0.0.1:58969)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:04.420263631Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1588,"":"io task","client":"0","":"downstairs","session_id":"19986c1d-7fa3-4ba6-bee7-122975a4f481"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:04.420720619Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1588,"client":"2","":"downstairs","session_id":"19986c1d-7fa3-4ba6-bee7-122975a4f481"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:04.420747423Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1588,"client":"2","":"downstairs","session_id":"19986c1d-7fa3-4ba6-bee7-122975a4f481"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:04.42087744Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1588,"client":"1","":"downstairs","session_id":"19986c1d-7fa3-4ba6-bee7-122975a4f481"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:04.420898342Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1588,"client":"1","":"downstairs","session_id":"19986c1d-7fa3-4ba6-bee7-122975a4f481"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:04.420912354Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1588,"client":"0","":"downstairs","session_id":"19986c1d-7fa3-4ba6-bee7-122975a4f481"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:04.420925756Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1588,"client":"0","":"downstairs","session_id":"19986c1d-7fa3-4ba6-bee7-122975a4f481"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:04.421385795Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1588,"client":"2","":"downstairs","session_id":"19986c1d-7fa3-4ba6-bee7-122975a4f481"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:04.42142091Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1588,"client":"0","":"downstairs","session_id":"19986c1d-7fa3-4ba6-bee7-122975a4f481"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:04.42157388Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1588,"client":"1","":"downstairs","session_id":"19986c1d-7fa3-4ba6-bee7-122975a4f481"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:04.42204114Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1588,"client":"0","":"downstairs","session_id":"19986c1d-7fa3-4ba6-bee7-122975a4f481"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:04.422066363Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1588,"client":"0","":"downstairs","session_id":"19986c1d-7fa3-4ba6-bee7-122975a4f481"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:04.422080745Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1588,"session_id":"19986c1d-7fa3-4ba6-bee7-122975a4f481"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:04.422093776Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1588,"client":"2","":"downstairs","session_id":"19986c1d-7fa3-4ba6-bee7-122975a4f481"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:04.422107068Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1588,"client":"2","":"downstairs","session_id":"19986c1d-7fa3-4ba6-bee7-122975a4f481"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:04.42212167Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1588,"session_id":"19986c1d-7fa3-4ba6-bee7-122975a4f481"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:04.422134062Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1588,"client":"1","":"downstairs","session_id":"19986c1d-7fa3-4ba6-bee7-122975a4f481"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:04.422147554Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1588,"client":"1","":"downstairs","session_id":"19986c1d-7fa3-4ba6-bee7-122975a4f481"} {"msg":"[0]R flush_numbers[0..12]: [156, 158, 157, 155, 160, 160, 157, 157, 158, 160, 150, 147]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:04.422164886Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1588,"":"downstairs","":"downstairs","session_id":"19986c1d-7fa3-4ba6-bee7-122975a4f481"} {"msg":"[0]R generation[0..12]: [98, 100, 99, 97, 101, 101, 99, 99, 100, 101, 94, 93]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:04.422179347Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1588,"":"downstairs","":"downstairs","session_id":"19986c1d-7fa3-4ba6-bee7-122975a4f481"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:04.422193469Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1588,"":"downstairs","":"downstairs","session_id":"19986c1d-7fa3-4ba6-bee7-122975a4f481"} {"msg":"[1]R flush_numbers[0..12]: [156, 158, 157, 155, 160, 160, 157, 157, 158, 160, 150, 147]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:04.422207191Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1588,"":"downstairs","":"downstairs","session_id":"19986c1d-7fa3-4ba6-bee7-122975a4f481"} {"msg":"[1]R generation[0..12]: [98, 100, 99, 97, 101, 101, 99, 99, 100, 101, 94, 93]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:04.422220803Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1588,"":"downstairs","":"downstairs","session_id":"19986c1d-7fa3-4ba6-bee7-122975a4f481"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:04.422234665Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1588,"":"downstairs","":"downstairs","session_id":"19986c1d-7fa3-4ba6-bee7-122975a4f481"} {"msg":"[2]R flush_numbers[0..12]: [156, 158, 157, 155, 160, 160, 157, 157, 158, 160, 150, 147]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:04.422248576Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1588,"":"downstairs","":"downstairs","session_id":"19986c1d-7fa3-4ba6-bee7-122975a4f481"} {"msg":"[2]R generation[0..12]: [98, 100, 99, 97, 101, 101, 99, 99, 100, 101, 94, 93]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:04.422262258Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1588,"":"downstairs","":"downstairs","session_id":"19986c1d-7fa3-4ba6-bee7-122975a4f481"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:04.42227602Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1588,"":"downstairs","":"downstairs","session_id":"19986c1d-7fa3-4ba6-bee7-122975a4f481"} {"msg":"Max found gen is 102","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:04.422289062Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1588,"":"downstairs","":"downstairs","session_id":"19986c1d-7fa3-4ba6-bee7-122975a4f481"} {"msg":"Generation requested: 102 >= found:102","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-01T16:54:04.422302403Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1588,"":"downstairs","":"downstairs","session_id":"19986c1d-7fa3-4ba6-bee7-122975a4f481"} {"msg":"Next flush: 161","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:04.422322346Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1588,"":"downstairs","":"downstairs","session_id":"19986c1d-7fa3-4ba6-bee7-122975a4f481"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:04.422335568Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1588,"":"downstairs","":"downstairs","session_id":"19986c1d-7fa3-4ba6-bee7-122975a4f481"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:04.422348769Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1588,"session_id":"19986c1d-7fa3-4ba6-bee7-122975a4f481"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:04.422362581Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1588,"client":"0","":"downstairs","session_id":"19986c1d-7fa3-4ba6-bee7-122975a4f481"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:04.422375883Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1588,"client":"1","":"downstairs","session_id":"19986c1d-7fa3-4ba6-bee7-122975a4f481"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:04.422389205Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1588,"client":"2","":"downstairs","session_id":"19986c1d-7fa3-4ba6-bee7-122975a4f481"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:04.422402406Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1588,"session_id":"19986c1d-7fa3-4ba6-bee7-122975a4f481"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:04.422414768Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1588,"session_id":"19986c1d-7fa3-4ba6-bee7-122975a4f481"} {"msg":"8d030d0d-964d-4334-9aa6-0f554eec060a is now active with session: 19986c1d-7fa3-4ba6-bee7-122975a4f481","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:04.42242771Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1588,"session_id":"19986c1d-7fa3-4ba6-bee7-122975a4f481"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:04.422439851Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1588,"session_id":"19986c1d-7fa3-4ba6-bee7-122975a4f481"} {"msg":"Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 The guest has finished waiting for activation with:102","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:04.422455203Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1588} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 159 len 12288 data: 7 9 9 02/30 Read block 30 len 28672 03/30 Read block 125 len 8192 04/30 Write block 518 len 36864 data: 5 6 6 5 6 5 4 5 5 05/30 Write block 85 len 24576 data: 11 9 10 11 12 15 06/30 Write block 49 len 16384 data: 10 10 8 8 07/30 Flush 08/30 Write block 450 len 32768 data: 10 10 7 8 7 4 5 4 09/30 Read block 460 len 36864 10/30 Read block 263 len 24576 11/30 Read block 134 len 40960 12/30 Write block 308 len 24576 data: 12 11 9 10 10 12 13/30 Read block 397 len 28672 14/30 Read block 570 len 12288 15/30 Read block 453 len 8192 16/30 Read block 276 len 4096 17/30 Write block 595 len 12288 data: 7 6 5 18/30 Read block 547 len 12288 19/30 Read block 108 len 4096 20/30 Read block 527 len 16384 21/30 Read block 172 len 16384 22/30 Read block 1 len 8192 23/30 Write block 508 len 12288 data: 9 8 8 24/30 Write block 100 len 28672 data: 13 11 10 8 9 9 11 25/30 Write block 321 len 32768 data: 9 8 7 7 6 6 4 3 26/30 Read block 114 len 40960 27/30 Write block 264 len 32768 data: 10 10 11 10 10 12 11 13 28/30 Write block 138 len 12288 data: 10 8 7 29/30 Write block 490 len 28672 data: 12 11 8 5 4 6 7 30/30 Read block 433 len 20480 ---------------------------------------------------------------- Crucible gen:102 GIO:true work queues: Upstairs:0 downstairs:23 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1007 Acked Write 8 Done Done Done false 1008 Acked Read 9 Done Done Done false 1009 Acked Read 6 Done Done Done false 1010 Acked Read 10 Done Done Done false 1011 Acked Write 6 Done Done Done false 1012 Acked Read 7 Done Done Done false 1013 Acked Read 3 Done Done Done false 1014 Acked Read 2 Done Done Done false 1015 Acked Read 1 Done Done Done false 1016 Acked Write 3 Done Done Done false 1017 Acked Read 3 Done Done Done false 1018 Acked Read 1 Done Done Done false 1019 Acked Read 4 Done Done Done false 1020 Acked Read 4 Done Done Done false 1021 Acked Read 2 Done Done Done false 1022 Acked Write 3 Done Done Done false 1023 Acked Write 7 Done Done Done false 1024 Acked Write 8 Done Done Done false 1025 Acked Read 10 Done Done Done false 1026 Acked Write 8 Done Done Done false 1027 Acked Write 3 Done Done Done false 1028 Acked Write 7 Done Done Sent false 1029 Acked Read 5 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 2 3 Done 23 22 21 66 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1006 1006 1006 Downstairs last five completed: 1006 1005 1004 1003 1002 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:54:04.444042444Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1588} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 98 98 98 156 156 156 F F F 1 020-039 100 100 100 158 158 158 F F F 2 040-059 102 102 102 161 161 161 F F F 3 060-079 97 97 97 155 155 155 F F F 4 080-099 102 102 102 161 161 161 F F F 5 100-119 101 101 101 160 160 160 T T T 6 120-139 99 99 99 157 157 157 T T T 7 140-159 102 102 102 161 161 161 T T T 8 160-179 102 102 102 161 161 161 F F F 9 180-199 101 101 101 160 160 160 F F F 10 200-219 94 94 94 150 150 150 F F F 11 220-239 93 93 93 147 147 147 F F F 12 240-259 100 100 100 158 158 158 F F F 13 260-279 98 98 98 156 156 156 T T T 14 280-299 99 99 99 157 157 157 F F F 15 300-319 101 101 101 160 160 160 T T T 16 320-339 101 101 101 160 160 160 T T T 17 340-359 98 98 98 156 156 156 F F F 18 360-379 98 98 98 156 156 156 F F F 19 380-399 98 98 98 156 156 156 F F F 20 400-419 95 95 95 151 151 151 F F F 21 420-439 100 100 100 159 159 159 F F F 22 440-459 96 96 96 152 152 152 T T T 23 460-479 93 93 93 147 147 147 F F F 24 480-499 99 99 99 157 157 157 T T T 25 500-519 102 102 102 161 161 161 T T T 26 520-539 102 102 102 161 161 161 F F F 27 540-559 100 100 100 158 158 158 F F F 28 560-579 95 95 95 151 151 151 F F F 29 580-599 99 99 99 157 157 157 T T T Max gen: 102, Max flush: 161 A Difference in extent metadata is expected here On loop 51 Verifying data now No change after verify Loop: 51 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 98 98 98 156 156 156 F F F 1 020-039 100 100 100 158 158 158 F F F 2 040-059 102 102 102 161 161 161 F F F 3 060-079 97 97 97 155 155 155 F F F 4 080-099 102 102 102 161 161 161 F F F 5 100-119 103 103 103 162 162 162 F F F 6 120-139 103 103 103 162 162 162 F F F 7 140-159 103 103 103 162 162 162 F F F 8 160-179 102 102 102 161 161 161 F F F 9 180-199 101 101 101 160 160 160 F F F 10 200-219 94 94 94 150 150 150 F F F 11 220-239 93 93 93 147 147 147 F F F 12 240-259 100 100 100 158 158 158 F F F 13 260-279 103 103 103 162 162 162 F F F 14 280-299 99 99 99 157 157 157 F F F 15 300-319 103 103 103 162 162 162 F F F 16 320-339 103 103 103 162 162 162 F F F 17 340-359 98 98 98 156 156 156 F F F 18 360-379 98 98 98 156 156 156 F F F 19 380-399 98 98 98 156 156 156 F F F 20 400-419 95 95 95 151 151 151 F F F 21 420-439 100 100 100 159 159 159 F F F 22 440-459 103 103 103 162 162 162 F F F 23 460-479 93 93 93 147 147 147 F F F 24 480-499 103 103 103 162 162 162 F F F 25 500-519 103 103 103 162 162 162 F F F 26 520-539 102 102 102 161 161 161 F F F 27 540-559 100 100 100 158 158 158 F F F 28 560-579 95 95 95 151 151 151 F F F 29 580-599 103 103 103 162 162 162 F F F Max gen: 103, Max flush: 162 Begin loop 52 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:08.814835688Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1596,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:08.858706411Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1596,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:08.859260543Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1596} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"3818f0fc76fe787c6cc412af30fd59b930203885\",\n git_commit_timestamp: \"2024-11-01T16:08:30.000000000Z\",\n git_branch: \"renovate/hyper-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:08.859300828Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1596} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:08.859315539Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1596} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:08.859330112Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1596} {"msg":"Crucible 4fff009a-ee54-4c7c-a46c-fb6d4af94042 has session id: c250a31b-c50c-4c80-8b6c-8288b64f7201","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:08.859699089Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1596,"session_id":"c250a31b-c50c-4c80-8b6c-8288b64f7201"} {"msg":"Upstairs opts: Upstairs UUID: 4fff009a-ee54-4c7c-a46c-fb6d4af94042, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:08.859807813Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1596,"session_id":"c250a31b-c50c-4c80-8b6c-8288b64f7201"} {"msg":"Crucible stats registered with UUID: 4fff009a-ee54-4c7c-a46c-fb6d4af94042","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:08.859843118Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1596,"session_id":"c250a31b-c50c-4c80-8b6c-8288b64f7201"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:08.89067313Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1596,"task":"crutest"} {"msg":"The guest has requested activation with gen:104","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:08.890717356Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1596} {"msg":"4fff009a-ee54-4c7c-a46c-fb6d4af94042 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:08.890737998Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1596,"session_id":"c250a31b-c50c-4c80-8b6c-8288b64f7201"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:08.89075105Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1596,"client":"0","":"downstairs","session_id":"c250a31b-c50c-4c80-8b6c-8288b64f7201"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:08.890775183Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1596,"client":"0","":"downstairs","session_id":"c250a31b-c50c-4c80-8b6c-8288b64f7201"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:08.890789495Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1596,"client":"1","":"downstairs","session_id":"c250a31b-c50c-4c80-8b6c-8288b64f7201"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:08.890803337Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1596,"client":"1","":"downstairs","session_id":"c250a31b-c50c-4c80-8b6c-8288b64f7201"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:08.890817069Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1596,"client":"2","":"downstairs","session_id":"c250a31b-c50c-4c80-8b6c-8288b64f7201"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:08.89083072Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1596,"client":"2","":"downstairs","session_id":"c250a31b-c50c-4c80-8b6c-8288b64f7201"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:08.890844112Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1596,"":"io task","client":"2","":"downstairs","session_id":"c250a31b-c50c-4c80-8b6c-8288b64f7201"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:08.890858164Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1596,"":"io task","client":"0","":"downstairs","session_id":"c250a31b-c50c-4c80-8b6c-8288b64f7201"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:08.890874236Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1596,"":"io task","client":"1","":"downstairs","session_id":"c250a31b-c50c-4c80-8b6c-8288b64f7201"} {"msg":"ds_connection connected from Ok(127.0.0.1:63881)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:08.890967518Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1596,"":"io task","client":"2","":"downstairs","session_id":"c250a31b-c50c-4c80-8b6c-8288b64f7201"} {"msg":"ds_connection connected from Ok(127.0.0.1:32994)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:08.891057029Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1596,"":"io task","client":"0","":"downstairs","session_id":"c250a31b-c50c-4c80-8b6c-8288b64f7201"} {"msg":"ds_connection connected from Ok(127.0.0.1:54954)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:08.891148361Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1596,"":"io task","client":"1","":"downstairs","session_id":"c250a31b-c50c-4c80-8b6c-8288b64f7201"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:08.891830119Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1596,"client":"0","":"downstairs","session_id":"c250a31b-c50c-4c80-8b6c-8288b64f7201"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:08.891870314Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1596,"client":"0","":"downstairs","session_id":"c250a31b-c50c-4c80-8b6c-8288b64f7201"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:08.891891037Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1596,"client":"2","":"downstairs","session_id":"c250a31b-c50c-4c80-8b6c-8288b64f7201"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:08.891905919Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1596,"client":"2","":"downstairs","session_id":"c250a31b-c50c-4c80-8b6c-8288b64f7201"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:08.891929242Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1596,"client":"1","":"downstairs","session_id":"c250a31b-c50c-4c80-8b6c-8288b64f7201"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:08.891957035Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1596,"client":"1","":"downstairs","session_id":"c250a31b-c50c-4c80-8b6c-8288b64f7201"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:08.89245937Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1596,"client":"0","":"downstairs","session_id":"c250a31b-c50c-4c80-8b6c-8288b64f7201"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:08.892501395Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1596,"client":"2","":"downstairs","session_id":"c250a31b-c50c-4c80-8b6c-8288b64f7201"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:08.892649805Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1596,"client":"1","":"downstairs","session_id":"c250a31b-c50c-4c80-8b6c-8288b64f7201"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:08.893059107Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1596,"client":"0","":"downstairs","session_id":"c250a31b-c50c-4c80-8b6c-8288b64f7201"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:08.893099993Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1596,"client":"0","":"downstairs","session_id":"c250a31b-c50c-4c80-8b6c-8288b64f7201"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:08.893130837Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1596,"session_id":"c250a31b-c50c-4c80-8b6c-8288b64f7201"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:08.893149259Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1596,"client":"2","":"downstairs","session_id":"c250a31b-c50c-4c80-8b6c-8288b64f7201"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:08.893163931Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1596,"client":"2","":"downstairs","session_id":"c250a31b-c50c-4c80-8b6c-8288b64f7201"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:08.893177893Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1596,"session_id":"c250a31b-c50c-4c80-8b6c-8288b64f7201"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:08.893190224Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1596,"client":"1","":"downstairs","session_id":"c250a31b-c50c-4c80-8b6c-8288b64f7201"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:08.893203726Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1596,"client":"1","":"downstairs","session_id":"c250a31b-c50c-4c80-8b6c-8288b64f7201"} {"msg":"[0]R flush_numbers[0..12]: [156, 158, 161, 155, 161, 162, 162, 162, 161, 160, 150, 147]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:08.893220718Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1596,"":"downstairs","":"downstairs","session_id":"c250a31b-c50c-4c80-8b6c-8288b64f7201"} {"msg":"[0]R generation[0..12]: [98, 100, 102, 97, 102, 103, 103, 103, 102, 101, 94, 93]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:08.89323522Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1596,"":"downstairs","":"downstairs","session_id":"c250a31b-c50c-4c80-8b6c-8288b64f7201"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:08.893249762Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1596,"":"downstairs","":"downstairs","session_id":"c250a31b-c50c-4c80-8b6c-8288b64f7201"} {"msg":"[1]R flush_numbers[0..12]: [156, 158, 161, 155, 161, 162, 162, 162, 161, 160, 150, 147]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:08.893263784Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1596,"":"downstairs","":"downstairs","session_id":"c250a31b-c50c-4c80-8b6c-8288b64f7201"} {"msg":"[1]R generation[0..12]: [98, 100, 102, 97, 102, 103, 103, 103, 102, 101, 94, 93]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:08.893277575Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1596,"":"downstairs","":"downstairs","session_id":"c250a31b-c50c-4c80-8b6c-8288b64f7201"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:08.893292077Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1596,"":"downstairs","":"downstairs","session_id":"c250a31b-c50c-4c80-8b6c-8288b64f7201"} {"msg":"[2]R flush_numbers[0..12]: [156, 158, 161, 155, 161, 162, 162, 162, 161, 160, 150, 147]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:08.893306079Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1596,"":"downstairs","":"downstairs","session_id":"c250a31b-c50c-4c80-8b6c-8288b64f7201"} {"msg":"[2]R generation[0..12]: [98, 100, 102, 97, 102, 103, 103, 103, 102, 101, 94, 93]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-01T16:54:08.893319911Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1596,"":"downstairs","":"downstairs","session_id":"c250a31b-c50c-4c80-8b6c-8288b64f7201"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:08.893340194Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1596,"":"downstairs","":"downstairs","session_id":"c250a31b-c50c-4c80-8b6c-8288b64f7201"} {"msg":"Max found gen is 104","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:08.893353425Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1596,"":"downstairs","":"downstairs","session_id":"c250a31b-c50c-4c80-8b6c-8288b64f7201"} {"msg":"Generation requested: 104 >= found:104","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:08.893366627Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1596,"":"downstairs","":"downstairs","session_id":"c250a31b-c50c-4c80-8b6c-8288b64f7201"} {"msg":"Next flush: 163","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:08.893380239Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1596,"":"downstairs","":"downstairs","session_id":"c250a31b-c50c-4c80-8b6c-8288b64f7201"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:08.893404682Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1596,"":"downstairs","":"downstairs","session_id":"c250a31b-c50c-4c80-8b6c-8288b64f7201"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2024-11-01T16:54:08.893429645Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1596,"session_id":"c250a31b-c50c-4c80-8b6c-8288b64f7201"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:08.893458559Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1596,"client":"0","":"downstairs","session_id":"c250a31b-c50c-4c80-8b6c-8288b64f7201"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:08.893475361Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1596,"client":"1","":"downstairs","session_id":"c250a31b-c50c-4c80-8b6c-8288b64f7201"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:08.893488993Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1596,"client":"2","":"downstairs","session_id":"c250a31b-c50c-4c80-8b6c-8288b64f7201"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:08.893502695Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1596,"session_id":"c250a31b-c50c-4c80-8b6c-8288b64f7201"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:08.893515216Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1596,"session_id":"c250a31b-c50c-4c80-8b6c-8288b64f7201"} {"msg":"4fff009a-ee54-4c7c-a46c-fb6d4af94042 is now active with session: c250a31b-c50c-4c80-8b6c-8288b64f7201","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:08.893528338Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1596,"session_id":"c250a31b-c50c-4c80-8b6c-8288b64f7201"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:08.8935407Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1596,"session_id":"c250a31b-c50c-4c80-8b6c-8288b64f7201"} {"msg":"The guest has finished waiting for activation with:104","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:08.893553371Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1596} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 1 len 8192 data: 4 7 02/30 Read block 315 len 28672 03/30 Write block 440 len 28672 data: 7 6 6 8 9 9 9 04/30 Write block 47 len 32768 data: 9 7 11 11 9 9 7 7 05/30 Read block 75 len 16384 06/30 Write block 128 len 28672 data: 10 11 12 13 14 14 10 07/30 Write block 563 len 8192 data: 10 10 08/30 Read block 571 len 32768 09/30 Write block 319 len 8192 data: 12 11 10/30 Flush 11/30 Write block 149 len 32768 data: 9 12 10 11 8 10 11 11 12/30 Write block 554 len 8192 data: 13 11 13/30 Write block 56 len 24576 data: 7 7 6 6 5 6 14/30 Read block 283 len 32768 15/30 Read block 73 len 16384 16/30 Write block 101 len 36864 data: 12 11 9 10 10 12 8 8 10 17/30 Write block 215 len 24576 data: 7 11 12 13 9 9 18/30 Read block 554 len 16384 19/30 Flush 20/30 Write block 255 len 16384 data: 8 7 6 8 21/30 Read block 499 len 12288 22/30 Write block 500 len 40960 data: 10 9 7 5 6 6 8 8 10 9 23/30 Read block 392 len 36864 24/30 Write block 483 len 16384 data: 9 12 13 12 25/30 Read block 147 len 36864 26/30 Read block 49 len 20480 27/30 Read block 138 len 4096 28/30 Write block 240 len 12288 data: 6 7 7 29/30 Read block 124 len 28672 30/30 Write block 37 len 32768 data: 10 11 11 9 10 11 14 13 ---------------------------------------------------------------- Crucible gen:104 GIO:true work queues: Upstairs:0 downstairs:11 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1019 Acked Write 4 Done Done Done false 1020 Acked Read 3 Done Done Done false 1021 Acked Write 10 Done Done Done false 1022 Acked Read 9 Done Done Done false 1023 Acked Write 4 Done Done Done false 1024 Acked Read 9 Done Done Done false 1025 Acked Read 5 Done Done Done false 1026 Acked Read 1 Done Done Done false 1027 Acked Write 3 Done Done Done false 1028 Acked Read 7 Done Done Done false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 10 10 10 30 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1018 1018 1018 Downstairs last five completed: 1018 1017 1016 1015 1014 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:54:08.917706783Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1596} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 104 104 104 163 163 163 F F F 1 020-039 100 100 100 158 158 158 T T T 2 040-059 104 104 104 164 164 164 T T T 3 060-079 104 104 104 164 164 164 F F F 4 080-099 102 102 102 161 161 161 F F F 5 100-119 104 104 104 164 164 164 F F F 6 120-139 104 104 104 163 163 163 F F F 7 140-159 104 104 104 164 164 164 F F F 8 160-179 102 102 102 161 161 161 F F F 9 180-199 101 101 101 160 160 160 F F F 10 200-219 104 104 104 164 164 164 F F F 11 220-239 104 104 104 164 164 164 F F F 12 240-259 100 100 100 158 158 158 T T T 13 260-279 103 103 103 162 162 162 F F F 14 280-299 99 99 99 157 157 157 F F F 15 300-319 104 104 104 163 163 163 F F F 16 320-339 104 104 104 163 163 163 F F F 17 340-359 98 98 98 156 156 156 F F F 18 360-379 98 98 98 156 156 156 F F F 19 380-399 98 98 98 156 156 156 F F F 20 400-419 95 95 95 151 151 151 F F F 21 420-439 100 100 100 159 159 159 F F F 22 440-459 104 104 104 163 163 163 F F F 23 460-479 93 93 93 147 147 147 F F F 24 480-499 103 103 103 162 162 162 T T T 25 500-519 103 103 103 162 162 162 T T T 26 520-539 102 102 102 161 161 161 F F F 27 540-559 104 104 104 164 164 164 F F F 28 560-579 104 104 104 163 163 163 F F F 29 580-599 103 103 103 162 162 162 F F F Max gen: 104, Max flush: 164 A Difference in extent metadata is expected here On loop 52 Verifying data now No change after verify Loop: 52 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 104 104 104 163 163 163 F F F 1 020-039 105 105 105 165 165 165 F F F 2 040-059 105 105 105 165 165 165 F F F 3 060-079 104 104 104 164 164 164 F F F 4 080-099 102 102 102 161 161 161 F F F 5 100-119 104 104 104 164 164 164 F F F 6 120-139 104 104 104 163 163 163 F F F 7 140-159 104 104 104 164 164 164 F F F 8 160-179 102 102 102 161 161 161 F F F 9 180-199 101 101 101 160 160 160 F F F 10 200-219 104 104 104 164 164 164 F F F 11 220-239 104 104 104 164 164 164 F F F 12 240-259 105 105 105 165 165 165 F F F 13 260-279 103 103 103 162 162 162 F F F 14 280-299 99 99 99 157 157 157 F F F 15 300-319 104 104 104 163 163 163 F F F 16 320-339 104 104 104 163 163 163 F F F 17 340-359 98 98 98 156 156 156 F F F 18 360-379 98 98 98 156 156 156 F F F 19 380-399 98 98 98 156 156 156 F F F 20 400-419 95 95 95 151 151 151 F F F 21 420-439 100 100 100 159 159 159 F F F 22 440-459 104 104 104 163 163 163 F F F 23 460-479 93 93 93 147 147 147 F F F 24 480-499 105 105 105 165 165 165 F F F 25 500-519 105 105 105 165 165 165 F F F 26 520-539 102 102 102 161 161 161 F F F 27 540-559 104 104 104 164 164 164 F F F 28 560-579 104 104 104 163 163 163 F F F 29 580-599 103 103 103 162 162 162 F F F Max gen: 105, Max flush: 165 Begin loop 53 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:13.219842312Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1604,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:13.264117706Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1604,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:13.264655956Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1604} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"3818f0fc76fe787c6cc412af30fd59b930203885\",\n git_commit_timestamp: \"2024-11-01T16:08:30.000000000Z\",\n git_branch: \"renovate/hyper-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:13.264696801Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1604} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:13.264712133Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1604} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:13.264727445Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1604} {"msg":"Crucible 06a0fa0a-6b59-40f1-afa4-04aa1800c689 has session id: a139f916-0a81-4be4-9914-c9bd161bc81a","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:13.265106214Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1604,"session_id":"a139f916-0a81-4be4-9914-c9bd161bc81a"} {"msg":"Upstairs opts: Upstairs UUID: 06a0fa0a-6b59-40f1-afa4-04aa1800c689, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:13.265145929Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1604,"session_id":"a139f916-0a81-4be4-9914-c9bd161bc81a"} {"msg":"Crucible stats registered with UUID: 06a0fa0a-6b59-40f1-afa4-04aa1800c689","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:13.265169182Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1604,"session_id":"a139f916-0a81-4be4-9914-c9bd161bc81a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:13.296445492Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1604,"task":"crutest"} {"msg":"The guest has requested activation with gen:106","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:13.296475286Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1604} {"msg":"06a0fa0a-6b59-40f1-afa4-04aa1800c689 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:13.296536883Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1604,"session_id":"a139f916-0a81-4be4-9914-c9bd161bc81a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:13.296570038Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1604,"client":"0","":"downstairs","session_id":"a139f916-0a81-4be4-9914-c9bd161bc81a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:13.296611313Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1604,"client":"0","":"downstairs","session_id":"a139f916-0a81-4be4-9914-c9bd161bc81a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:13.296646788Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1604,"client":"1","":"downstairs","session_id":"a139f916-0a81-4be4-9914-c9bd161bc81a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:13.296679822Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1604,"client":"1","":"downstairs","session_id":"a139f916-0a81-4be4-9914-c9bd161bc81a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:13.296714056Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1604,"client":"2","":"downstairs","session_id":"a139f916-0a81-4be4-9914-c9bd161bc81a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:13.296746321Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1604,"client":"2","":"downstairs","session_id":"a139f916-0a81-4be4-9914-c9bd161bc81a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:13.296776374Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1604,"":"io task","client":"2","":"downstairs","session_id":"a139f916-0a81-4be4-9914-c9bd161bc81a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:13.296808238Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1604,"":"io task","client":"0","":"downstairs","session_id":"a139f916-0a81-4be4-9914-c9bd161bc81a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:13.296840063Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1604,"":"io task","client":"1","":"downstairs","session_id":"a139f916-0a81-4be4-9914-c9bd161bc81a"} {"msg":"ds_connection connected from Ok(127.0.0.1:37693)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:13.296920183Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1604,"":"io task","client":"2","":"downstairs","session_id":"a139f916-0a81-4be4-9914-c9bd161bc81a"} {"msg":"ds_connection connected from Ok(127.0.0.1:63868)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:13.296956918Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1604,"":"io task","client":"1","":"downstairs","session_id":"a139f916-0a81-4be4-9914-c9bd161bc81a"} {"msg":"ds_connection connected from Ok(127.0.0.1:45614)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:13.296986881Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1604,"":"io task","client":"0","":"downstairs","session_id":"a139f916-0a81-4be4-9914-c9bd161bc81a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:13.297428618Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1604,"client":"2","":"downstairs","session_id":"a139f916-0a81-4be4-9914-c9bd161bc81a"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:13.297465843Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1604,"client":"2","":"downstairs","session_id":"a139f916-0a81-4be4-9914-c9bd161bc81a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:13.297647517Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1604,"client":"1","":"downstairs","session_id":"a139f916-0a81-4be4-9914-c9bd161bc81a"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:13.297677401Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1604,"client":"1","":"downstairs","session_id":"a139f916-0a81-4be4-9914-c9bd161bc81a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:13.297707394Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1604,"client":"0","":"downstairs","session_id":"a139f916-0a81-4be4-9914-c9bd161bc81a"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:13.297735898Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1604,"client":"0","":"downstairs","session_id":"a139f916-0a81-4be4-9914-c9bd161bc81a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:13.298129559Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1604,"client":"2","":"downstairs","session_id":"a139f916-0a81-4be4-9914-c9bd161bc81a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:13.298263776Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1604,"client":"0","":"downstairs","session_id":"a139f916-0a81-4be4-9914-c9bd161bc81a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:13.298469023Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1604,"client":"1","":"downstairs","session_id":"a139f916-0a81-4be4-9914-c9bd161bc81a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:13.298811216Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1604,"client":"2","":"downstairs","session_id":"a139f916-0a81-4be4-9914-c9bd161bc81a"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:13.298851662Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1604,"client":"2","":"downstairs","session_id":"a139f916-0a81-4be4-9914-c9bd161bc81a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:13.298878705Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1604,"session_id":"a139f916-0a81-4be4-9914-c9bd161bc81a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:13.298903259Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1604,"client":"0","":"downstairs","session_id":"a139f916-0a81-4be4-9914-c9bd161bc81a"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:13.298926081Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1604,"client":"0","":"downstairs","session_id":"a139f916-0a81-4be4-9914-c9bd161bc81a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:13.298940223Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1604,"session_id":"a139f916-0a81-4be4-9914-c9bd161bc81a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:13.298952505Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1604,"client":"1","":"downstairs","session_id":"a139f916-0a81-4be4-9914-c9bd161bc81a"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:13.298966147Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1604,"client":"1","":"downstairs","session_id":"a139f916-0a81-4be4-9914-c9bd161bc81a"} {"msg":"[0]R flush_numbers[0..12]: [163, 165, 165, 164, 161, 164, 163, 164, 161, 160, 164, 164]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:13.298980378Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1604,"":"downstairs","":"downstairs","session_id":"a139f916-0a81-4be4-9914-c9bd161bc81a"} {"msg":"[0]R generation[0..12]: [104, 105, 105, 104, 102, 104, 104, 104, 102, 101, 104, 104]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:13.29899448Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1604,"":"downstairs","":"downstairs","session_id":"a139f916-0a81-4be4-9914-c9bd161bc81a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:13.299011982Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1604,"":"downstairs","":"downstairs","session_id":"a139f916-0a81-4be4-9914-c9bd161bc81a"} {"msg":"[1]R flush_numbers[0..12]: [163, 165, 165, 164, 161, 164, 163, 164, 161, 160, 164, 164]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:13.299026074Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1604,"":"downstairs","":"downstairs","session_id":"a139f916-0a81-4be4-9914-c9bd161bc81a"} {"msg":"[1]R generation[0..12]: [104, 105, 105, 104, 102, 104, 104, 104, 102, 101, 104, 104]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:13.299039946Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1604,"":"downstairs","":"downstairs","session_id":"a139f916-0a81-4be4-9914-c9bd161bc81a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:13.299054058Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1604,"":"downstairs","":"downstairs","session_id":"a139f916-0a81-4be4-9914-c9bd161bc81a"} {"msg":"[2]R flush_numbers[0..12]: [163, 165, 165, 164, 161, 164, 163, 164, 161, 160, 164, 164]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:13.29906806Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1604,"":"downstairs","":"downstairs","session_id":"a139f916-0a81-4be4-9914-c9bd161bc81a"} {"msg":"[2]R generation[0..12]: [104, 105, 105, 104, 102, 104, 104, 104, 102, 101, 104, 104]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:13.299082011Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1604,"":"downstairs","":"downstairs","session_id":"a139f916-0a81-4be4-9914-c9bd161bc81a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:13.299095813Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1604,"":"downstairs","":"downstairs","session_id":"a139f916-0a81-4be4-9914-c9bd161bc81a"} {"msg":"Max found gen is 106","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:13.299108815Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1604,"":"downstairs","":"downstairs","session_id":"a139f916-0a81-4be4-9914-c9bd161bc81a"} {"msg":"Generation requested: 106 >= found:106","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:13.299121997Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1604,"":"downstairs","":"downstairs","session_id":"a139f916-0a81-4be4-9914-c9bd161bc81a"} {"msg":"Next flush: 166","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:13.299134968Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1604,"":"downstairs","":"downstairs","session_id":"a139f916-0a81-4be4-9914-c9bd161bc81a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:13.29914808Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1604,"":"downstairs","":"downstairs","session_id":"a139f916-0a81-4be4-9914-c9bd161bc81a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:13.299161352Z","hostname":"Wait for a query_work_queue command to finish before sending IO w-01JBMA2T0XD4W63AXFF321JBBP","pid":1604,"session_id":"a139f916-0a81-4be4-9914-c9bd161bc81a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:13.299181644Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1604,"client":"0","":"downstairs","session_id":"a139f916-0a81-4be4-9914-c9bd161bc81a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:13.299195246Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1604,"client":"1","":"downstairs","session_id":"a139f916-0a81-4be4-9914-c9bd161bc81a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:13.299208618Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1604,"client":"2","":"downstairs","session_id":"a139f916-0a81-4be4-9914-c9bd161bc81a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:13.29922343Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1604,"session_id":"a139f916-0a81-4be4-9914-c9bd161bc81a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:13.299236081Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1604,"session_id":"a139f916-0a81-4be4-9914-c9bd161bc81a"} {"msg":"06a0fa0a-6b59-40f1-afa4-04aa1800c689 is now active with session: a139f916-0a81-4be4-9914-c9bd161bc81a","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:13.299248813Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1604,"session_id":"a139f916-0a81-4be4-9914-c9bd161bc81a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:13.299260905Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1604,"session_id":"a139f916-0a81-4be4-9914-c9bd161bc81a"} {"msg":"The guest has finished waiting for activation with:106","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:13.299273206Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1604} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 245 len 28672 data: 10 12 14 14 14 16 15 02/30 Flush 03/30 Flush 04/30 Write block 225 len 24576 data: 11 9 7 5 8 8 05/30 Write block 376 len 16384 data: 4 6 8 8 06/30 Read block 391 len 16384 07/30 Write block 360 len 32768 data: 5 4 5 4 5 7 6 4 08/30 Read block 232 len 24576 09/30 Write block 539 len 8192 data: 10 7 10/30 Flush 11/30 Write block 9 len 4096 data: 11 12/30 Write block 488 len 12288 data: 15 13 13 13/30 Write block 309 len 40960 data: 12 10 11 11 13 11 14 14 11 12 14/30 Flush 15/30 Write block 190 len 24576 data: 11 9 9 10 11 11 16/30 Read block 425 len 36864 17/30 Read block 281 len 8192 18/30 Read block 58 len 4096 19/30 Write block 412 len 28672 data: 4 4 4 5 4 6 8 20/30 Write block 274 len 16384 data: 15 14 14 10 21/30 Write block 495 len 32768 data: 7 8 8 10 10 11 10 8 22/30 Read block 57 len 32768 23/30 Write block 430 len 32768 data: 10 10 5 4 3 4 5 4 24/30 Read block 557 len 12288 25/30 Flush 26/30 Write block 219 len 16384 data: 10 10 10 10 27/30 Read block 385 len 24576 28/30 Write block 213 len 20480 data: 9 9 8 12 13 29/30 Read block 291 len 16384 30/30 Read block 463 len 20480 ---------------------------------------------------------------- Crucible gen:106 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 4 Done Done Done false 1026 Acked Read 6 Done Done Done false 1027 Acked Write 5 Done Done Done false 1028 Acked Read 4 Done Done Done false 1029 Acked Read 5 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 5 4 4 13 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:54:13.32607616Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1604} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 106 106 106 169 169 169 F F F 1 020-039 105 105 105 165 165 165 F F F 2 040-059 105 105 105 165 165 165 F F F 3 060-079 104 104 104 164 164 164 F F F 4 080-099 102 102 102 161 161 161 F F F 5 100-119 104 104 104 164 164 164 F F F 6 120-139 104 104 104 163 163 163 F F F 7 140-159 104 104 104 164 164 164 F F F 8 160-179 102 102 102 161 161 161 F F F 9 180-199 106 106 106 170 170 170 F F F 10 200-219 104 104 104 164 164 164 T T T 11 220-239 106 106 106 168 168 168 T T T 12 240-259 106 106 106 166 166 166 F F F 13 260-279 106 106 106 170 170 170 F F F 14 280-299 99 99 99 157 157 157 F F F 15 300-319 106 106 106 169 169 169 F F F 16 320-339 104 104 104 163 163 163 F F F 17 340-359 98 98 98 156 156 156 F F F 18 360-379 106 106 106 168 168 168 F F F 19 380-399 98 98 98 156 156 156 F F F 20 400-419 106 106 106 170 170 170 F F F 21 420-439 106 106 106 170 170 170 F F F 22 440-459 104 104 104 163 163 163 F F F 23 460-479 93 93 93 147 147 147 F F F 24 480-499 106 106 106 170 170 170 F F F 25 500-519 106 106 106 170 170 170 F F F 26 520-539 106 106 106 168 168 168 F F F 27 540-559 106 106 106 168 168 168 F F F 28 560-579 104 104 104 163 163 163 F F F 29 580-599 103 103 103 162 162 162 F F F Max gen: 106, Max flush: 170 A Difference in extent metadata is expected here On loop 53 Verifying data now No change after verify Loop: 53 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 106 106 106 169 169 169 F F F 1 020-039 105 105 105 165 165 165 F F F 2 040-059 105 105 105 165 165 165 F F F 3 060-079 104 104 104 164 164 164 F F F 4 080-099 102 102 102 161 161 161 F F F 5 100-119 104 104 104 164 164 164 F F F 6 120-139 104 104 104 163 163 163 F F F 7 140-159 104 104 104 164 164 164 F F F 8 160-179 102 102 102 161 161 161 F F F 9 180-199 106 106 106 170 170 170 F F F 10 200-219 107 107 107 171 171 171 F F F 11 220-239 107 107 107 171 171 171 F F F 12 240-259 106 106 106 166 166 166 F F F 13 260-279 106 106 106 170 170 170 F F F 14 280-299 99 99 99 157 157 157 F F F 15 300-319 106 106 106 169 169 169 F F F 16 320-339 104 104 104 163 163 163 F F F 17 340-359 98 98 98 156 156 156 F F F 18 360-379 106 106 106 168 168 168 F F F 19 380-399 98 98 98 156 156 156 F F F 20 400-419 106 106 106 170 170 170 F F F 21 420-439 106 106 106 170 170 170 F F F 22 440-459 104 104 104 163 163 163 F F F 23 460-479 93 93 93 147 147 147 F F F 24 480-499 106 106 106 170 170 170 F F F 25 500-519 106 106 106 170 170 170 F F F 26 520-539 106 106 106 168 168 168 F F F 27 540-559 106 106 106 168 168 168 F F F 28 560-579 104 104 104 163 163 163 F F F 29 580-599 103 103 103 162 162 162 F F F Max gen: 107, Max flush: 171 Begin loop 54 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:17.590667501Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1612,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:17.634108768Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1612,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:17.634699844Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1612} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"3818f0fc76fe787c6cc412af30fd59b930203885\",\n git_commit_timestamp: \"2024-11-01T16:08:30.000000000Z\",\n git_branch: \"renovate/hyper-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:17.634738729Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1612} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:17.634754911Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1612} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:17.634770063Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1612} {"msg":"Crucible 120c6628-70bd-458a-8672-bdbf0383dddb has session id: 9e9a348b-d14f-4cef-bb2a-80200f9bbd58","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:17.635307142Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1612,"session_id":"9e9a348b-d14f-4cef-bb2a-80200f9bbd58"} {"msg":"Upstairs opts: Upstairs UUID: 120c6628-70bd-458a-8672-bdbf0383dddb, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:17.635348368Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1612,"session_id":"9e9a348b-d14f-4cef-bb2a-80200f9bbd58"} {"msg":"Crucible stats registered with UUID: 120c6628-70bd-458a-8672-bdbf0383dddb","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:17.635371661Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1612,"session_id":"9e9a348b-d14f-4cef-bb2a-80200f9bbd58"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:17.666242879Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1612,"task":"crutest"} {"msg":"The guest has requested activation with gen:108","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:17.666274303Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1612} {"msg":"120c6628-70bd-458a-8672-bdbf0383dddb active request set","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:17.66633658Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1612,"session_id":"9e9a348b-d14f-4cef-bb2a-80200f9bbd58"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:17.666370155Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1612,"client":"0","":"downstairs","session_id":"9e9a348b-d14f-4cef-bb2a-80200f9bbd58"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:17.666401209Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1612,"client":"0","":"downstairs","session_id":"9e9a348b-d14f-4cef-bb2a-80200f9bbd58"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:17.666419001Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1612,"client":"1","":"downstairs","session_id":"9e9a348b-d14f-4cef-bb2a-80200f9bbd58"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:17.666436363Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1612,"client":"1","":"downstairs","session_id":"9e9a348b-d14f-4cef-bb2a-80200f9bbd58"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:17.666457356Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1612,"client":"2","":"downstairs","session_id":"9e9a348b-d14f-4cef-bb2a-80200f9bbd58"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:17.666475208Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1612,"client":"2","":"downstairs","session_id":"9e9a348b-d14f-4cef-bb2a-80200f9bbd58"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:17.66649214Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1612,"":"io task","client":"2","":"downstairs","session_id":"9e9a348b-d14f-4cef-bb2a-80200f9bbd58"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:17.666509533Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1612,"":"io task","client":"0","":"downstairs","session_id":"9e9a348b-d14f-4cef-bb2a-80200f9bbd58"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:17.666526675Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1612,"":"io task","client":"1","":"downstairs","session_id":"9e9a348b-d14f-4cef-bb2a-80200f9bbd58"} {"msg":"ds_connection connected from Ok(127.0.0.1:34288)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:17.666573111Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1612,"":"io task","client":"2","":"downstairs","session_id":"9e9a348b-d14f-4cef-bb2a-80200f9bbd58"} {"msg":"ds_connection connected from Ok(127.0.0.1:43101)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:17.666609946Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1612,"":"io task","client":"0","":"downstairs","session_id":"9e9a348b-d14f-4cef-bb2a-80200f9bbd58"} {"msg":"ds_connection connected from Ok(127.0.0.1:48476)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:17.66664476Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1612,"":"io task","client":"1","":"downstairs","session_id":"9e9a348b-d14f-4cef-bb2a-80200f9bbd58"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:17.667352342Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1612,"client":"2","":"downstairs","session_id":"9e9a348b-d14f-4cef-bb2a-80200f9bbd58"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:17.667381855Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1612,"client":"2","":"downstairs","session_id":"9e9a348b-d14f-4cef-bb2a-80200f9bbd58"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:17.667408059Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1612,"client":"0","":"downstairs","session_id":"9e9a348b-d14f-4cef-bb2a-80200f9bbd58"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:17.667436192Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1612,"client":"0","":"downstairs","session_id":"9e9a348b-d14f-4cef-bb2a-80200f9bbd58"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:17.667504791Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1612,"client":"1","":"downstairs","session_id":"9e9a348b-d14f-4cef-bb2a-80200f9bbd58"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:17.667531034Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1612,"client":"1","":"downstairs","session_id":"9e9a348b-d14f-4cef-bb2a-80200f9bbd58"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:17.667959009Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1612,"client":"2","":"downstairs","session_id":"9e9a348b-d14f-4cef-bb2a-80200f9bbd58"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:17.6681201Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1612,"client":"1","":"downstairs","session_id":"9e9a348b-d14f-4cef-bb2a-80200f9bbd58"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:17.668230684Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1612,"client":"0","":"downstairs","session_id":"9e9a348b-d14f-4cef-bb2a-80200f9bbd58"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:17.668492898Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1612,"client":"2","":"downstairs","session_id":"9e9a348b-d14f-4cef-bb2a-80200f9bbd58"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:17.668517341Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1612,"client":"2","":"downstairs","session_id":"9e9a348b-d14f-4cef-bb2a-80200f9bbd58"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:17.668531943Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1612,"session_id":"9e9a348b-d14f-4cef-bb2a-80200f9bbd58"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:17.668555206Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1612,"client":"1","":"downstairs","session_id":"9e9a348b-d14f-4cef-bb2a-80200f9bbd58"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:17.668569368Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1612,"client":"1","":"downstairs","session_id":"9e9a348b-d14f-4cef-bb2a-80200f9bbd58"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:17.66858269Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1612,"session_id":"9e9a348b-d14f-4cef-bb2a-80200f9bbd58"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:17.668705446Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1612,"client":"0","":"downstairs","session_id":"9e9a348b-d14f-4cef-bb2a-80200f9bbd58"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:17.668725078Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1612,"client":"0","":"downstairs","session_id":"9e9a348b-d14f-4cef-bb2a-80200f9bbd58"} {"msg":"[0]R flush_numbers[0..12]: [169, 165, 165, 164, 161, 164, 163, 164, 161, 170, 171, 171]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:17.66873923Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1612,"":"downstairs","":"downstairs","session_id":"9e9a348b-d14f-4cef-bb2a-80200f9bbd58"} {"msg":"[0]R generation[0..12]: [106, 105, 105, 104, 102, 104, 104, 104, 102, 106, 107, 107]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:17.668753382Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1612,"":"downstairs","":"downstairs","session_id":"9e9a348b-d14f-4cef-bb2a-80200f9bbd58"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:17.668767414Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1612,"":"downstairs","":"downstairs","session_id":"9e9a348b-d14f-4cef-bb2a-80200f9bbd58"} {"msg":"[1]R flush_numbers[0..12]: [169, 165, 165, 164, 161, 164, 163, 164, 161, 170, 171, 171]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:17.668781146Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1612,"":"downstairs","":"downstairs","session_id":"9e9a348b-d14f-4cef-bb2a-80200f9bbd58"} {"msg":"[1]R generation[0..12]: [106, 105, 105, 104, 102, 104, 104, 104, 102, 106, 107, 107]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:17.668794907Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1612,"":"downstairs","":"downstairs","session_id":"9e9a348b-d14f-4cef-bb2a-80200f9bbd58"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:17.66881181Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1612,"":"downstairs","":"downstairs","session_id":"9e9a348b-d14f-4cef-bb2a-80200f9bbd58"} {"msg":"[2]R flush_numbers[0..12]: [169, 165, 165, 164, 161, 164, 163, 164, 161, 170, 171, 171]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:17.668825871Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1612,"":"downstairs","":"downstairs","session_id":"9e9a348b-d14f-4cef-bb2a-80200f9bbd58"} {"msg":"[2]R generation[0..12]: [106, 105, 105, 104, 102, 104, 104, 104, 102, 106, 107, 107]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:17.668839533Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1612,"":"downstairs","":"downstairs","session_id":"9e9a348b-d14f-4cef-bb2a-80200f9bbd58"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:17.668854295Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1612,"":"downstairs","":"downstairs","session_id":"9e9a348b-d14f-4cef-bb2a-80200f9bbd58"} {"msg":"Max found gen is 108","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:17.668869677Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1612,"":"downstairs","":"downstairs","session_id":"9e9a348b-d14f-4cef-bb2a-80200f9bbd58"} {"msg":"Generation requested: 108 >= found:108","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:17.668885459Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1612,"":"downstairs","":"downstairs","session_id":"9e9a348b-d14f-4cef-bb2a-80200f9bbd58"} {"msg":"Next flush: 172","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:17.668900641Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1612,"":"downstairs","":"downstairs","session_id":"9e9a348b-d14f-4cef-bb2a-80200f9bbd58"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:17.668915923Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1612,"":"downstairs","":"downstairs","session_id":"9e9a348b-d14f-4cef-bb2a-80200f9bbd58"} Wait for a query_work_queue command to finish before sending IO {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:17.669009995Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1612,"session_id":"9e9a348b-d14f-4cef-bb2a-80200f9bbd58"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:17.669031088Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1612,"client":"0","":"downstairs","session_id":"9e9a348b-d14f-4cef-bb2a-80200f9bbd58"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:17.66904792Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1612,"client":"1","":"downstairs","session_id":"9e9a348b-d14f-4cef-bb2a-80200f9bbd58"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:17.669063702Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1612,"client":"2","":"downstairs","session_id":"9e9a348b-d14f-4cef-bb2a-80200f9bbd58"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:17.669079374Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1612,"session_id":"9e9a348b-d14f-4cef-bb2a-80200f9bbd58"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:17.669092586Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1612,"session_id":"9e9a348b-d14f-4cef-bb2a-80200f9bbd58"} {"msg":"120c6628-70bd-458a-8672-bdbf0383dddb is now active with session: 9e9a348b-d14f-4cef-bb2a-80200f9bbd58","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:17.669105427Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1612,"session_id":"9e9a348b-d14f-4cef-bb2a-80200f9bbd58"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:17.669120639Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1612,"session_id":"9e9a348b-d14f-4cef-bb2a-80200f9bbd58"} {"msg":"The guest has finished waiting for activation with:108","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:17.669133141Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1612} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 584 len 16384 data: 10 12 11 15 02/30 Write block 581 len 20480 data: 9 8 9 11 13 03/30 Write block 557 len 36864 data: 10 13 15 10 9 9 11 11 6 04/30 Write block 556 len 36864 data: 10 11 14 16 11 10 10 12 12 05/30 Read block 250 len 20480 06/30 Flush 07/30 Write block 219 len 28672 data: 11 11 11 11 14 13 12 08/30 Read block 331 len 20480 09/30 Write block 27 len 28672 data: 12 13 12 9 11 12 10 10/30 Read block 40 len 8192 11/30 Read block 56 len 16384 12/30 Write block 353 len 20480 data: 12 9 8 7 9 13/30 Write block 130 len 36864 data: 13 14 15 15 11 8 6 7 11 14/30 Read block 124 len 32768 15/30 Write block 575 len 16384 data: 9 7 8 10 16/30 Write block 95 len 20480 data: 10 12 14 14 14 17/30 Write block 175 len 4096 data: 9 18/30 Read block 77 len 36864 19/30 Write block 27 len 40960 data: 13 14 13 10 12 13 11 9 10 11 20/30 Write block 368 len 20480 data: 4 6 5 4 4 21/30 Read block 508 len 24576 22/30 Read block 496 len 20480 23/30 Write block 249 len 8192 data: 15 17 24/30 Read block 87 len 8192 25/30 Read block 384 len 16384 26/30 Read block 324 len 36864 27/30 Read block 561 len 24576 28/30 Write block 571 len 4096 data: 7 29/30 Read block 545 len 36864 30/30 Write block 338 len 12288 data: 7 5 6 ---------------------------------------------------------------- Crucible gen:108 GIO:true work queues: Upstairs:0 downstairs:24 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1006 Acked Write 7 Done Done Done false 1007 Acked Read 5 Done Done Done false 1008 Acked Write 7 Done Done Done false 1009 Acked Read 2 Done Done Done false 1010 Acked Read 4 Done Done Done false 1011 Acked Write 5 Done Done Done false 1012 Acked Write 9 Done Done Done false 1013 Acked Read 8 Done Done Done false 1014 Acked Write 4 Done Done Done false 1015 Acked Write 5 Done Done Done false 1016 Acked Write 1 Done Done Done false 1017 Acked Read 9 Done Done Done false 1018 Acked Write 10 Done Done Done false 1019 Acked Write 5 Done Done Done false 1020 Acked Read 6 Done Done Done false 1021 Acked Read 5 Done Done Done false 1022 Acked Write 2 Done Done Done false 1023 Acked Read 2 Done Done Done false 1024 Acked Read 4 Done Done Done false 1025 Acked Read 9 Done Done Done false 1026 Acked Read 6 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Read 9 Done Done Done false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 23 23 23 69 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1005 1005 1005 Downstairs last five completed: 1005 1004 1003 1002 1001 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:54:17.689904857Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1612} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 106 106 106 169 169 169 F F F 1 020-039 105 105 105 165 165 165 T T T 2 040-059 105 105 105 165 165 165 F F F 3 060-079 104 104 104 164 164 164 F F F 4 080-099 102 102 102 161 161 161 T T T 5 100-119 104 104 104 164 164 164 F F F 6 120-139 104 104 104 163 163 163 T T T 7 140-159 104 104 104 164 164 164 F F F 8 160-179 102 102 102 161 161 161 T T T 9 180-199 106 106 106 170 170 170 F F F 10 200-219 107 107 107 171 171 171 T T T 11 220-239 107 107 107 171 171 171 T T T 12 240-259 106 106 106 166 166 166 T T T 13 260-279 106 106 106 170 170 170 F F F 14 280-299 99 99 99 157 157 157 F F F 15 300-319 106 106 106 169 169 169 F F F 16 320-339 104 104 104 163 163 163 T T T 17 340-359 98 98 98 156 156 156 T T T 18 360-379 106 106 106 168 168 168 T T T 19 380-399 98 98 98 156 156 156 F F F 20 400-419 106 106 106 170 170 170 F F F 21 420-439 106 106 106 170 170 170 F F F 22 440-459 104 104 104 163 163 163 F F F 23 460-479 93 93 93 147 147 147 F F F 24 480-499 106 106 106 170 170 170 F F F 25 500-519 106 106 106 170 170 170 F F F 26 520-539 106 106 106 168 168 168 F F F 27 540-559 108 108 108 172 172 172 F F F 28 560-579 108 108 108 172 172 172 T T T 29 580-599 108 108 108 172 172 172 F F F Max gen: 108, Max flush: 172 A Difference in extent metadata is expected here On loop 54 Verifying data now No change after verify Loop: 54 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 106 106 106 169 169 169 F F F 1 020-039 109 109 109 173 173 173 F F F 2 040-059 105 105 105 165 165 165 F F F 3 060-079 104 104 104 164 164 164 F F F 4 080-099 109 109 109 173 173 173 F F F 5 100-119 104 104 104 164 164 164 F F F 6 120-139 109 109 109 173 173 173 F F F 7 140-159 104 104 104 164 164 164 F F F 8 160-179 109 109 109 173 173 173 F F F 9 180-199 106 106 106 170 170 170 F F F 10 200-219 109 109 109 173 173 173 F F F 11 220-239 109 109 109 173 173 173 F F F 12 240-259 109 109 109 173 173 173 F F F 13 260-279 106 106 106 170 170 170 F F F 14 280-299 99 99 99 157 157 157 F F F 15 300-319 106 106 106 169 169 169 F F F 16 320-339 109 109 109 173 173 173 F F F 17 340-359 109 109 109 173 173 173 F F F 18 360-379 109 109 109 173 173 173 F F F 19 380-399 98 98 98 156 156 156 F F F 20 400-419 106 106 106 170 170 170 F F F 21 420-439 106 106 106 170 170 170 F F F 22 440-459 104 104 104 163 163 163 F F F 23 460-479 93 93 93 147 147 147 F F F 24 480-499 106 106 106 170 170 170 F F F 25 500-519 106 106 106 170 170 170 F F F 26 520-539 106 106 106 168 168 168 F F F 27 540-559 108 108 108 172 172 172 F F F 28 560-579 109 109 109 173 173 173 F F F 29 580-599 108 108 108 172 172 172 F F F Max gen: 109, Max flush: 173 Begin loop 55 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:22.063328931Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1620,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:22.11314806Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1620,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:22.113772771Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1620} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"3818f0fc76fe787c6cc412af30fd59b930203885\",\n git_commit_timestamp: \"2024-11-01T16:08:30.000000000Z\",\n git_branch: \"renovate/hyper-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:22.113809906Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1620} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:22.113824768Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1620} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:22.1138393Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1620} {"msg":"Crucible 7261c125-e48b-4bbb-9e61-6a3a161a8234 has session id: 725b7002-9253-46dc-8db9-e7ec3e6bf207","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:22.114273445Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1620,"session_id":"725b7002-9253-46dc-8db9-e7ec3e6bf207"} {"msg":"Upstairs opts: Upstairs UUID: 7261c125-e48b-4bbb-9e61-6a3a161a8234, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:22.114316381Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1620,"session_id":"725b7002-9253-46dc-8db9-e7ec3e6bf207"} {"msg":"Crucible stats registered with UUID: 7261c125-e48b-4bbb-9e61-6a3a161a8234","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:22.114341484Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1620,"session_id":"725b7002-9253-46dc-8db9-e7ec3e6bf207"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:22.145260108Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1620,"task":"crutest"} {"msg":"The guest has requested activation with gen:110","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:22.145293352Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1620} {"msg":"7261c125-e48b-4bbb-9e61-6a3a161a8234 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:22.145343689Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1620,"session_id":"725b7002-9253-46dc-8db9-e7ec3e6bf207"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:22.145363341Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1620,"client":"0","":"downstairs","session_id":"725b7002-9253-46dc-8db9-e7ec3e6bf207"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:22.145392795Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1620,"client":"0","":"downstairs","session_id":"725b7002-9253-46dc-8db9-e7ec3e6bf207"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:22.145410677Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1620,"client":"1","":"downstairs","session_id":"725b7002-9253-46dc-8db9-e7ec3e6bf207"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:22.14542773Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1620,"client":"1","":"downstairs","session_id":"725b7002-9253-46dc-8db9-e7ec3e6bf207"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:22.145448302Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1620,"client":"2","":"downstairs","session_id":"725b7002-9253-46dc-8db9-e7ec3e6bf207"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:22.145465695Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1620,"client":"2","":"downstairs","session_id":"725b7002-9253-46dc-8db9-e7ec3e6bf207"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:22.145482327Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1620,"":"io task","client":"0","":"downstairs","session_id":"725b7002-9253-46dc-8db9-e7ec3e6bf207"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:22.145503179Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1620,"":"io task","client":"2","":"downstairs","session_id":"725b7002-9253-46dc-8db9-e7ec3e6bf207"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:22.145535284Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1620,"":"io task","client":"1","":"downstairs","session_id":"725b7002-9253-46dc-8db9-e7ec3e6bf207"} {"msg":"ds_connection connected from Ok(127.0.0.1:44648)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:22.145571468Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1620,"":"io task","client":"0","":"downstairs","session_id":"725b7002-9253-46dc-8db9-e7ec3e6bf207"} {"msg":"ds_connection connected from Ok(127.0.0.1:62839)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:22.145715297Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1620,"":"io task","client":"1","":"downstairs","session_id":"725b7002-9253-46dc-8db9-e7ec3e6bf207"} {"msg":"ds_connection connected from Ok(127.0.0.1:41579)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:22.145764013Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1620,"":"io task","client":"2","":"downstairs","session_id":"725b7002-9253-46dc-8db9-e7ec3e6bf207"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:22.146376842Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1620,"client":"0","":"downstairs","session_id":"725b7002-9253-46dc-8db9-e7ec3e6bf207"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:22.146412237Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1620,"client":"0","":"downstairs","session_id":"725b7002-9253-46dc-8db9-e7ec3e6bf207"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:22.146578678Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1620,"client":"1","":"downstairs","session_id":"725b7002-9253-46dc-8db9-e7ec3e6bf207"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:22.146600301Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1620,"client":"1","":"downstairs","session_id":"725b7002-9253-46dc-8db9-e7ec3e6bf207"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:22.146633435Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1620,"client":"2","":"downstairs","session_id":"725b7002-9253-46dc-8db9-e7ec3e6bf207"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:22.146663049Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1620,"client":"2","":"downstairs","session_id":"725b7002-9253-46dc-8db9-e7ec3e6bf207"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:22.147069671Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1620,"client":"0","":"downstairs","session_id":"725b7002-9253-46dc-8db9-e7ec3e6bf207"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:22.147102155Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1620,"client":"1","":"downstairs","session_id":"725b7002-9253-46dc-8db9-e7ec3e6bf207"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:22.147250305Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1620,"client":"2","":"downstairs","session_id":"725b7002-9253-46dc-8db9-e7ec3e6bf207"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:22.147638255Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1620,"client":"0","":"downstairs","session_id":"725b7002-9253-46dc-8db9-e7ec3e6bf207"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:22.147663278Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1620,"client":"0","":"downstairs","session_id":"725b7002-9253-46dc-8db9-e7ec3e6bf207"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:22.14767853Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1620,"session_id":"725b7002-9253-46dc-8db9-e7ec3e6bf207"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:22.147753449Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1620,"client":"1","":"downstairs","session_id":"725b7002-9253-46dc-8db9-e7ec3e6bf207"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:22.147786674Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1620,"client":"1","":"downstairs","session_id":"725b7002-9253-46dc-8db9-e7ec3e6bf207"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:22.147802636Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1620,"session_id":"725b7002-9253-46dc-8db9-e7ec3e6bf207"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:22.147815037Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1620,"client":"2","":"downstairs","session_id":"725b7002-9253-46dc-8db9-e7ec3e6bf207"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:22.147828419Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1620,"client":"2","":"downstairs","session_id":"725b7002-9253-46dc-8db9-e7ec3e6bf207"} {"msg":"[0]R flush_numbers[0..12]: [169, 173, 165, 164, 173, 164, 173, 164, 173, 170, 173, 173]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:22.147842121Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1620,"":"downstairs","":"downstairs","session_id":"725b7002-9253-46dc-8db9-e7ec3e6bf207"} {"msg":"[0]R generation[0..12]: [106, 109, 105, 104, 109, 104, 109, 104, 109, 106, 109, 109]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:22.147855903Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1620,"":"downstairs","":"downstairs","session_id":"725b7002-9253-46dc-8db9-e7ec3e6bf207"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:22.147869834Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1620,"":"downstairs","":"downstairs","session_id":"725b7002-9253-46dc-8db9-e7ec3e6bf207"} {"msg":"[1]R flush_numbers[0..12]: [169, 173, 165, 164, 173, 164, 173, 164, 173, 170, 173, 173]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:22.147883676Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1620,"":"downstairs","":"downstairs","session_id":"725b7002-9253-46dc-8db9-e7ec3e6bf207"} {"msg":"[1]R generation[0..12]: [106, 109, 105, 104, 109, 104, 109, 104, 109, 106, 109, 109]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:22.147897328Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1620,"":"downstairs","":"downstairs","session_id":"725b7002-9253-46dc-8db9-e7ec3e6bf207"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:22.14791421Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1620,"":"downstairs","":"downstairs","session_id":"725b7002-9253-46dc-8db9-e7ec3e6bf207"} {"msg":"[2]R flush_numbers[0..12]: [169, 173, 165, 164, 173, 164, 173, 164, 173, 170, 173, 173]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:22.147928682Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1620,"":"downstairs","":"downstairs","session_id":"725b7002-9253-46dc-8db9-e7ec3e6bf207"} {"msg":"[2]R generation[0..12]: [106, 109, 105, 104, 109, 104, 109, 104, 109, 106, 109, 109]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:22.147942764Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1620,"":"downstairs","":"downstairs","session_id":"725b7002-9253-46dc-8db9-e7ec3e6bf207"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:22.147960056Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1620,"":"downstairs","":"downstairs","session_id":"725b7002-9253-46dc-8db9-e7ec3e6bf207"} {"msg":"Max found gen is 110","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:22.147984389Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1620,"":"downstairs","":"downstairs","session_id":"725b7002-9253-46dc-8db9-e7ec3e6bf207"} {"msg":"Generation requested: 110 >= found:110","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:22.148008072Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1620,"":"downstairs","":"downstairs","session_id":"725b7002-9253-46dc-8db9-e7ec3e6bf207"} Wait for a query_work_queue command to finish before sending IO {"msg":"Next flush: 174","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:22.148029975Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1620,"":"downstairs","":"downstairs","session_id":"725b7002-9253-46dc-8db9-e7ec3e6bf207"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:22.148043197Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1620,"":"downstairs","":"downstairs","session_id":"725b7002-9253-46dc-8db9-e7ec3e6bf207"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:22.148056338Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1620,"session_id":"725b7002-9253-46dc-8db9-e7ec3e6bf207"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:22.14806841Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1620,"client":"0","":"downstairs","session_id":"725b7002-9253-46dc-8db9-e7ec3e6bf207"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:22.148081652Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1620,"client":"1","":"downstairs","session_id":"725b7002-9253-46dc-8db9-e7ec3e6bf207"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:22.148094793Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1620,"client":"2","":"downstairs","session_id":"725b7002-9253-46dc-8db9-e7ec3e6bf207"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:22.148108085Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1620,"session_id":"725b7002-9253-46dc-8db9-e7ec3e6bf207"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:22.148120306Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1620,"session_id":"725b7002-9253-46dc-8db9-e7ec3e6bf207"} {"msg":"7261c125-e48b-4bbb-9e61-6a3a161a8234 is now active with session: 725b7002-9253-46dc-8db9-e7ec3e6bf207","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:22.148133048Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1620,"session_id":"725b7002-9253-46dc-8db9-e7ec3e6bf207"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:22.14814793Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1620,"session_id":"725b7002-9253-46dc-8db9-e7ec3e6bf207"} {"msg":"The guest has finished waiting for activation with:110","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:22.148160712Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1620} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 16 len 16384 data: 10 10 9 9 02/30 Read block 359 len 32768 03/30 Write block 121 len 12288 data: 10 11 8 04/30 Write block 200 len 20480 data: 7 8 8 10 10 05/30 Write block 438 len 12288 data: 6 7 8 06/30 Flush 07/30 Write block 86 len 16384 data: 10 11 12 13 08/30 Write block 193 len 12288 data: 11 12 12 09/30 Read block 302 len 20480 10/30 Write block 373 len 36864 data: 6 6 4 5 7 9 9 7 8 11/30 Read block 383 len 12288 12/30 Read block 343 len 16384 13/30 Read block 24 len 24576 14/30 Write block 207 len 36864 data: 13 11 9 11 13 11 10 10 9 15/30 Write block 383 len 12288 data: 9 8 9 16/30 Write block 232 len 40960 data: 6 5 6 6 7 7 7 7 7 8 17/30 Flush 18/30 Write block 257 len 24576 data: 7 9 8 8 12 12 19/30 Write block 487 len 12288 data: 15 16 14 20/30 Read block 170 len 32768 21/30 Write block 70 len 32768 data: 17 15 15 10 9 8 4 4 22/30 Read block 408 len 20480 23/30 Write block 404 len 16384 data: 5 3 6 6 24/30 Write block 287 len 12288 data: 4 4 5 25/30 Read block 124 len 20480 26/30 Read block 158 len 20480 27/30 Write block 271 len 12288 data: 14 11 12 28/30 Read block 507 len 36864 29/30 Write block 297 len 32768 data: 9 8 8 5 4 5 5 7 30/30 Read block 326 len 40960 ---------------------------------------------------------------- Crucible gen:110 GIO:true work queues: Upstairs:0 downstairs:13 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1017 Acked Write 6 Done Done Done false 1018 Acked Write 3 Done Done Done false 1019 Acked Read 8 Done Done Done false 1020 Acked Write 8 Done Done Done false 1021 Acked Read 5 Done Done Done false 1022 Acked Write 4 Done Done Done false 1023 Acked Write 3 Done Done Done false 1024 Acked Read 5 Done Done Done false 1025 Acked Read 5 Done Done Done false 1026 Acked Write 3 Done Done Done false 1027 Acked Read 9 Done Done Done false 1028 Acked Write 8 Done Done Sent false 1029 Acked Read 10 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 2 2 Done 13 13 11 37 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1016 1016 1016 Downstairs last five completed: 1016 1015 1014 1013 1012 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:54:22.170799769Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1620} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 110 110 110 174 174 174 F F F 1 020-039 109 109 109 173 173 173 F F F 2 040-059 105 105 105 165 165 165 F F F 3 060-079 104 104 104 164 164 164 T T T 4 080-099 110 110 110 175 175 175 F F F 5 100-119 104 104 104 164 164 164 F F F 6 120-139 110 110 110 174 174 174 F F F 7 140-159 104 104 104 164 164 164 F F F 8 160-179 109 109 109 173 173 173 F F F 9 180-199 110 110 110 175 175 175 F F F 10 200-219 110 110 110 175 175 175 F F F 11 220-239 110 110 110 175 175 175 F F F 12 240-259 110 110 110 175 175 175 T T T 13 260-279 106 106 106 170 170 170 T T T 14 280-299 99 99 99 157 157 157 T T T 15 300-319 106 106 106 169 169 169 T T T 16 320-339 109 109 109 173 173 173 F F F 17 340-359 109 109 109 173 173 173 F F F 18 360-379 110 110 110 175 175 175 F F F 19 380-399 110 110 110 175 175 175 F F F 20 400-419 106 106 106 170 170 170 T T T 21 420-439 110 110 110 174 174 174 F F F 22 440-459 110 110 110 174 174 174 F F F 23 460-479 93 93 93 147 147 147 F F F 24 480-499 106 106 106 170 170 170 T T T 25 500-519 106 106 106 170 170 170 F F F 26 520-539 106 106 106 168 168 168 F F F 27 540-559 108 108 108 172 172 172 F F F 28 560-579 109 109 109 173 173 173 F F F 29 580-599 108 108 108 172 172 172 F F F Max gen: 110, Max flush: 175 A Difference in extent metadata is expected here On loop 55 Verifying data now No change after verify Loop: 55 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 110 110 110 174 174 174 F F F 1 020-039 109 109 109 173 173 173 F F F 2 040-059 105 105 105 165 165 165 F F F 3 060-079 111 111 111 176 176 176 F F F 4 080-099 110 110 110 175 175 175 F F F 5 100-119 104 104 104 164 164 164 F F F 6 120-139 110 110 110 174 174 174 F F F 7 140-159 104 104 104 164 164 164 F F F 8 160-179 109 109 109 173 173 173 F F F 9 180-199 110 110 110 175 175 175 F F F 10 200-219 110 110 110 175 175 175 F F F 11 220-239 110 110 110 175 175 175 F F F 12 240-259 111 111 111 176 176 176 F F F 13 260-279 111 111 111 176 176 176 F F F 14 280-299 111 111 111 176 176 176 F F F 15 300-319 111 111 111 176 176 176 F F F 16 320-339 109 109 109 173 173 173 F F F 17 340-359 109 109 109 173 173 173 F F F 18 360-379 110 110 110 175 175 175 F F F 19 380-399 110 110 110 175 175 175 F F F 20 400-419 111 111 111 176 176 176 F F F 21 420-439 110 110 110 174 174 174 F F F 22 440-459 110 110 110 174 174 174 F F F 23 460-479 93 93 93 147 147 147 F F F 24 480-499 111 111 111 176 176 176 F F F 25 500-519 106 106 106 170 170 170 F F F 26 520-539 106 106 106 168 168 168 F F F 27 540-559 108 108 108 172 172 172 F F F 28 560-579 109 109 109 173 173 173 F F F 29 580-599 108 108 108 172 172 172 F F F Max gen: 111, Max flush: 176 Begin loop 56 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:26.501797557Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1628,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:26.545692513Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1628,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:26.546251875Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1628} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"3818f0fc76fe787c6cc412af30fd59b930203885\",\n git_commit_timestamp: \"2024-11-01T16:08:30.000000000Z\",\n git_branch: \"renovate/hyper-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:26.54629377Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1628} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:26.546312603Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1628} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:26.546327804Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1628} {"msg":"Crucible 8624399a-3115-413b-ae78-0c51c0cc853e has session id: 2d02c1ca-d8ff-4bd1-8b25-750e0ede1e9a","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:26.546792395Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1628,"session_id":"2d02c1ca-d8ff-4bd1-8b25-750e0ede1e9a"} {"msg":"Upstairs opts: Upstairs UUID: 8624399a-3115-413b-ae78-0c51c0cc853e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:26.54683327Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1628,"session_id":"2d02c1ca-d8ff-4bd1-8b25-750e0ede1e9a"} {"msg":"Crucible stats registered with UUID: 8624399a-3115-413b-ae78-0c51c0cc853e","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:26.546856523Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1628,"session_id":"2d02c1ca-d8ff-4bd1-8b25-750e0ede1e9a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:26.577698717Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1628,"task":"crutest"} {"msg":"The guest has requested activation with gen:112","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:26.577729311Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1628} {"msg":"8624399a-3115-413b-ae78-0c51c0cc853e active request set","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:26.577806771Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1628,"session_id":"2d02c1ca-d8ff-4bd1-8b25-750e0ede1e9a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:26.577839875Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1628,"client":"0","":"downstairs","session_id":"2d02c1ca-d8ff-4bd1-8b25-750e0ede1e9a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:26.577870289Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1628,"client":"0","":"downstairs","session_id":"2d02c1ca-d8ff-4bd1-8b25-750e0ede1e9a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:26.577888451Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1628,"client":"1","":"downstairs","session_id":"2d02c1ca-d8ff-4bd1-8b25-750e0ede1e9a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:26.577905583Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1628,"client":"1","":"downstairs","session_id":"2d02c1ca-d8ff-4bd1-8b25-750e0ede1e9a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:26.577922246Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1628,"client":"2","":"downstairs","session_id":"2d02c1ca-d8ff-4bd1-8b25-750e0ede1e9a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:26.577945678Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1628,"client":"2","":"downstairs","session_id":"2d02c1ca-d8ff-4bd1-8b25-750e0ede1e9a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:26.577978873Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1628,"":"io task","client":"2","":"downstairs","session_id":"2d02c1ca-d8ff-4bd1-8b25-750e0ede1e9a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:26.578001056Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1628,"":"io task","client":"0","":"downstairs","session_id":"2d02c1ca-d8ff-4bd1-8b25-750e0ede1e9a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:26.57803453Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1628,"":"io task","client":"1","":"downstairs","session_id":"2d02c1ca-d8ff-4bd1-8b25-750e0ede1e9a"} {"msg":"ds_connection connected from Ok(127.0.0.1:41182)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:26.578078526Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1628,"":"io task","client":"2","":"downstairs","session_id":"2d02c1ca-d8ff-4bd1-8b25-750e0ede1e9a"} {"msg":"ds_connection connected from Ok(127.0.0.1:36692)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:26.57811602Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1628,"":"io task","client":"1","":"downstairs","session_id":"2d02c1ca-d8ff-4bd1-8b25-750e0ede1e9a"} {"msg":"ds_connection connected from Ok(127.0.0.1:35967)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:26.578151225Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1628,"":"io task","client":"0","":"downstairs","session_id":"2d02c1ca-d8ff-4bd1-8b25-750e0ede1e9a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:26.578686324Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1628,"client":"2","":"downstairs","session_id":"2d02c1ca-d8ff-4bd1-8b25-750e0ede1e9a"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:26.578716758Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1628,"client":"2","":"downstairs","session_id":"2d02c1ca-d8ff-4bd1-8b25-750e0ede1e9a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:26.57873446Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1628,"client":"0","":"downstairs","session_id":"2d02c1ca-d8ff-4bd1-8b25-750e0ede1e9a"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:26.578844174Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1628,"client":"0","":"downstairs","session_id":"2d02c1ca-d8ff-4bd1-8b25-750e0ede1e9a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:26.578863927Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1628,"client":"1","":"downstairs","session_id":"2d02c1ca-d8ff-4bd1-8b25-750e0ede1e9a"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:26.578877749Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1628,"client":"1","":"downstairs","session_id":"2d02c1ca-d8ff-4bd1-8b25-750e0ede1e9a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:26.579339898Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1628,"client":"0","":"downstairs","session_id":"2d02c1ca-d8ff-4bd1-8b25-750e0ede1e9a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:26.579455953Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1628,"client":"2","":"downstairs","session_id":"2d02c1ca-d8ff-4bd1-8b25-750e0ede1e9a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:26.579479026Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1628,"client":"1","":"downstairs","session_id":"2d02c1ca-d8ff-4bd1-8b25-750e0ede1e9a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:26.579871827Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1628,"client":"0","":"downstairs","session_id":"2d02c1ca-d8ff-4bd1-8b25-750e0ede1e9a"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:26.57989758Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1628,"client":"0","":"downstairs","session_id":"2d02c1ca-d8ff-4bd1-8b25-750e0ede1e9a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:26.579912152Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1628,"session_id":"2d02c1ca-d8ff-4bd1-8b25-750e0ede1e9a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:26.580004224Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1628,"client":"2","":"downstairs","session_id":"2d02c1ca-d8ff-4bd1-8b25-750e0ede1e9a"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:26.580022916Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1628,"client":"2","":"downstairs","session_id":"2d02c1ca-d8ff-4bd1-8b25-750e0ede1e9a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:26.580036488Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1628,"session_id":"2d02c1ca-d8ff-4bd1-8b25-750e0ede1e9a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:26.580094465Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1628,"client":"1","":"downstairs","session_id":"2d02c1ca-d8ff-4bd1-8b25-750e0ede1e9a"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:26.580112448Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1628,"client":"1","":"downstairs","session_id":"2d02c1ca-d8ff-4bd1-8b25-750e0ede1e9a"} {"msg":"[0]R flush_numbers[0..12]: [174, 173, 165, 176, 175, 164, 174, 164, 173, 175, 175, 175]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:26.58012657Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1628,"":"downstairs","":"downstairs","session_id":"2d02c1ca-d8ff-4bd1-8b25-750e0ede1e9a"} {"msg":"[0]R generation[0..12]: [110, 109, 105, 111, 110, 104, 110, 104, 109, 110, 110, 110]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:26.580141371Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1628,"":"downstairs","":"downstairs","session_id":"2d02c1ca-d8ff-4bd1-8b25-750e0ede1e9a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:26.580155423Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1628,"":"downstairs","":"downstairs","session_id":"2d02c1ca-d8ff-4bd1-8b25-750e0ede1e9a"} {"msg":"[1]R flush_numbers[0..12]: [174, 173, 165, 176, 175, 164, 174, 164, 173, 175, 175, 175]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:26.580169265Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1628,"":"downstairs","":"downstairs","session_id":"2d02c1ca-d8ff-4bd1-8b25-750e0ede1e9a"} {"msg":"[1]R generation[0..12]: [110, 109, 105, 111, 110, 104, 110, 104, 109, 110, 110, 110]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:26.580182937Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1628,"":"downstairs","":"downstairs","session_id":"2d02c1ca-d8ff-4bd1-8b25-750e0ede1e9a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:26.580197048Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1628,"":"downstairs","":"downstairs","session_id":"2d02c1ca-d8ff-4bd1-8b25-750e0ede1e9a"} {"msg":"[2]R flush_numbers[0..12]: [174, 173, 165, 176, 175, 164, 174, 164, 173, 175, 175, 175]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:26.580215581Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1628,"":"downstairs","":"downstairs","session_id":"2d02c1ca-d8ff-4bd1-8b25-750e0ede1e9a"} {"msg":"[2]R generation[0..12]: [110, 109, 105, 111, 110, 104, 110, 104, 109, 110, 110, 110]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:26.580229813Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1628,"":"downstairs","":"downstairs","session_id":"2d02c1ca-d8ff-4bd1-8b25-750e0ede1e9a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:26.580243955Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1628,"":"downstairs","":"downstairs","session_id":"2d02c1ca-d8ff-4bd1-8b25-750e0ede1e9a"} {"msg":"Max found gen is 112","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:26.580256946Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1628,"":"downstairs","":"downstairs","session_id":"2d02c1ca-d8ff-4bd1-8b25-750e0ede1e9a"} {"msg":"Generation requested: 112 >= found:112","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:26.580270138Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1628,"":"downstairs","":"downstairs","session_id":"2d02c1ca-d8ff-4bd1-8b25-750e0ede1e9a"} {"msg":"Next flush: 177","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:26.5802832Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1628,"":"downstairs","":"downstairs","session_id":"2d02c1ca-d8ff-4bd1-8b25-750e0ede1e9a"} {"msg":"All extents match",Wait for a query_work_queue command to finish before sending IO "v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:26.580303532Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1628,"":"downstairs","":"downstairs","session_id":"2d02c1ca-d8ff-4bd1-8b25-750e0ede1e9a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:26.580324755Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1628,"session_id":"2d02c1ca-d8ff-4bd1-8b25-750e0ede1e9a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:26.580348398Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1628,"client":"0","":"downstairs","session_id":"2d02c1ca-d8ff-4bd1-8b25-750e0ede1e9a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:26.580374411Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1628,"client":"1","":"downstairs","session_id":"2d02c1ca-d8ff-4bd1-8b25-750e0ede1e9a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:26.580396184Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1628,"client":"2","":"downstairs","session_id":"2d02c1ca-d8ff-4bd1-8b25-750e0ede1e9a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:26.580419727Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1628,"session_id":"2d02c1ca-d8ff-4bd1-8b25-750e0ede1e9a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:26.58044316Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1628,"session_id":"2d02c1ca-d8ff-4bd1-8b25-750e0ede1e9a"} {"msg":"8624399a-3115-413b-ae78-0c51c0cc853e is now active with session: 2d02c1ca-d8ff-4bd1-8b25-750e0ede1e9a","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:26.580467143Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1628,"session_id":"2d02c1ca-d8ff-4bd1-8b25-750e0ede1e9a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:26.580490406Z","hostname":"Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 w-01JBMA2T0XD4W63AXFF321JBBP","pid":1628,"session_id":"2d02c1ca-d8ff-4bd1-8b25-750e0ede1e9a"} {"msg":"The guest has finished waiting for activation with:112","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:26.58052097Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1628} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 100 len 8192 data: 14 13 02/30 Read block 561 len 28672 03/30 Write block 207 len 24576 data: 14 12 10 12 14 12 04/30 Flush 05/30 Write block 558 len 32768 data: 15 17 12 11 11 13 13 7 06/30 Read block 302 len 12288 07/30 Flush 08/30 Write block 36 len 8192 data: 12 11 09/30 Read block 305 len 12288 10/30 Write block 105 len 16384 data: 11 13 9 9 11/30 Read block 467 len 28672 12/30 Write block 469 len 20480 data: 9 10 12 12 12 13/30 Flush 14/30 Write block 312 len 16384 data: 12 14 12 15 15/30 Read block 473 len 28672 16/30 Read block 71 len 24576 17/30 Write block 440 len 12288 data: 9 7 7 18/30 Write block 87 len 24576 data: 12 13 14 16 12 10 19/30 Read block 442 len 4096 20/30 Write block 524 len 28672 data: 5 6 6 7 8 9 11 21/30 Write block 494 len 24576 data: 5 8 9 9 11 11 22/30 Write block 540 len 4096 data: 8 23/30 Write block 100 len 36864 data: 15 14 12 10 11 12 14 10 10 24/30 Read block 341 len 8192 25/30 Write block 191 len 16384 data: 10 10 12 13 26/30 Read block 305 len 36864 27/30 Write block 128 len 16384 data: 11 12 14 15 28/30 Write block 199 len 16384 data: 10 8 9 9 29/30 Read block 478 len 32768 30/30 Read block 0 len 32768 ---------------------------------------------------------------- Crucible gen:112 GIO:true work queues: Upstairs:0 downstairs:17 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1013 Acked Write 4 Done Done Done false 1014 Acked Read 7 Done Done Done false 1015 Acked Read 6 Done Done Done false 1016 Acked Write 3 Done Done Done false 1017 Acked Write 6 Done Done Done false 1018 Acked Read 1 Done Done Done false 1019 Acked Write 7 Done Done Done false 1020 Acked Write 6 Done Done Done false 1021 Acked Write 1 Done Done Done false 1022 Acked Write 9 Done Done Done false 1023 Acked Read 2 Done Done Done false 1024 Acked Write 4 Done Done Done false 1025 Acked Read 9 Done Done Done false 1026 Acked Write 4 Done Done Done false 1027 Acked Write 4 Done Done Done false 1028 Acked Read 8 Sent Done Done false 1029 Acked Read 8 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 0 0 2 Done 15 17 17 49 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1012 1012 1012 Downstairs last five completed: 1012 1011 1010 1009 1008 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:54:26.604561018Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1628} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 110 110 110 174 174 174 F F F 1 020-039 112 112 112 179 179 179 F F F 2 040-059 105 105 105 165 165 165 F F F 3 060-079 111 111 111 176 176 176 F F F 4 080-099 110 110 110 175 175 175 T T T 5 100-119 112 112 112 179 179 179 T T T 6 120-139 110 110 110 174 174 174 T T T 7 140-159 104 104 104 164 164 164 F F F 8 160-179 109 109 109 173 173 173 F F F 9 180-199 110 110 110 175 175 175 T T T 10 200-219 112 112 112 177 177 177 T T T 11 220-239 110 110 110 175 175 175 F F F 12 240-259 111 111 111 176 176 176 F F F 13 260-279 111 111 111 176 176 176 F F F 14 280-299 111 111 111 176 176 176 F F F 15 300-319 111 111 111 176 176 176 T T T 16 320-339 109 109 109 173 173 173 F F F 17 340-359 109 109 109 173 173 173 F F F 18 360-379 110 110 110 175 175 175 F F F 19 380-399 110 110 110 175 175 175 F F F 20 400-419 111 111 111 176 176 176 F F F 21 420-439 110 110 110 174 174 174 F F F 22 440-459 110 110 110 174 174 174 T T T 23 460-479 112 112 112 179 179 179 F F F 24 480-499 111 111 111 176 176 176 T T T 25 500-519 106 106 106 170 170 170 F F F 26 520-539 106 106 106 168 168 168 T T T 27 540-559 112 112 112 178 178 178 T T T 28 560-579 112 112 112 178 178 178 F F F 29 580-599 108 108 108 172 172 172 F F F Max gen: 112, Max flush: 179 A Difference in extent metadata is expected here On loop 56 Verifying data now No change after verify Loop: 56 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 110 110 110 174 174 174 F F F 1 020-039 112 112 112 179 179 179 F F F 2 040-059 105 105 105 165 165 165 F F F 3 060-079 111 111 111 176 176 176 F F F 4 080-099 113 113 113 180 180 180 F F F 5 100-119 113 113 113 180 180 180 F F F 6 120-139 113 113 113 180 180 180 F F F 7 140-159 104 104 104 164 164 164 F F F 8 160-179 109 109 109 173 173 173 F F F 9 180-199 113 113 113 180 180 180 F F F 10 200-219 113 113 113 180 180 180 F F F 11 220-239 110 110 110 175 175 175 F F F 12 240-259 111 111 111 176 176 176 F F F 13 260-279 111 111 111 176 176 176 F F F 14 280-299 111 111 111 176 176 176 F F F 15 300-319 113 113 113 180 180 180 F F F 16 320-339 109 109 109 173 173 173 F F F 17 340-359 109 109 109 173 173 173 F F F 18 360-379 110 110 110 175 175 175 F F F 19 380-399 110 110 110 175 175 175 F F F 20 400-419 111 111 111 176 176 176 F F F 21 420-439 110 110 110 174 174 174 F F F 22 440-459 113 113 113 180 180 180 F F F 23 460-479 112 112 112 179 179 179 F F F 24 480-499 113 113 113 180 180 180 F F F 25 500-519 106 106 106 170 170 170 F F F 26 520-539 113 113 113 180 180 180 F F F 27 540-559 113 113 113 180 180 180 F F F 28 560-579 112 112 112 178 178 178 F F F 29 580-599 108 108 108 172 172 172 F F F Max gen: 113, Max flush: 180 Begin loop 57 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:30.967097198Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1636,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:31.01065075Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1636,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:31.011225714Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1636} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"3818f0fc76fe787c6cc412af30fd59b930203885\",\n git_commit_timestamp: \"2024-11-01T16:08:30.000000000Z\",\n git_branch: \"renovate/hyper-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:31.011263399Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1636} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:31.011281531Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1636} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:31.011296223Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1636} {"msg":"Crucible 191aee1f-ab7a-4021-b95a-65f9058f399d has session id: 30585707-3599-4b54-b372-579c34d46304","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:31.011820531Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1636,"session_id":"30585707-3599-4b54-b372-579c34d46304"} {"msg":"Upstairs opts: Upstairs UUID: 191aee1f-ab7a-4021-b95a-65f9058f399d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:31.011856805Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1636,"session_id":"30585707-3599-4b54-b372-579c34d46304"} {"msg":"Crucible stats registered with UUID: 191aee1f-ab7a-4021-b95a-65f9058f399d","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:31.011872687Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1636,"session_id":"30585707-3599-4b54-b372-579c34d46304"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:31.043202764Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1636,"task":"crutest"} {"msg":"The guest has requested activation with gen:114","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:31.043232638Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1636} {"msg":"191aee1f-ab7a-4021-b95a-65f9058f399d active request set","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:31.043300897Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1636,"session_id":"30585707-3599-4b54-b372-579c34d46304"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:31.0433237Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1636,"client":"0","":"downstairs","session_id":"30585707-3599-4b54-b372-579c34d46304"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:31.043351723Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1636,"client":"0","":"downstairs","session_id":"30585707-3599-4b54-b372-579c34d46304"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:31.043369075Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1636,"client":"1","":"downstairs","session_id":"30585707-3599-4b54-b372-579c34d46304"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:31.043386058Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1636,"client":"1","":"downstairs","session_id":"30585707-3599-4b54-b372-579c34d46304"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:31.043468538Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1636,"client":"2","":"downstairs","session_id":"30585707-3599-4b54-b372-579c34d46304"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:31.043492142Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1636,"client":"2","":"downstairs","session_id":"30585707-3599-4b54-b372-579c34d46304"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:31.043515965Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1636,"":"io task","client":"2","":"downstairs","session_id":"30585707-3599-4b54-b372-579c34d46304"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:31.043552689Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1636,"":"io task","client":"0","":"downstairs","session_id":"30585707-3599-4b54-b372-579c34d46304"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:31.043585214Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1636,"":"io task","client":"1","":"downstairs","session_id":"30585707-3599-4b54-b372-579c34d46304"} {"msg":"ds_connection connected from Ok(127.0.0.1:51050)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:31.043616938Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1636,"":"io task","client":"2","":"downstairs","session_id":"30585707-3599-4b54-b372-579c34d46304"} {"msg":"ds_connection connected from Ok(127.0.0.1:47211)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:31.04371495Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1636,"":"io task","client":"1","":"downstairs","session_id":"30585707-3599-4b54-b372-579c34d46304"} {"msg":"ds_connection connected from Ok(127.0.0.1:35805)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:31.043808802Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1636,"":"io task","client":"0","":"downstairs","session_id":"30585707-3599-4b54-b372-579c34d46304"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:31.044302126Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1636,"client":"0","":"downstairs","session_id":"30585707-3599-4b54-b372-579c34d46304"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:31.04433225Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1636,"client":"0","":"downstairs","session_id":"30585707-3599-4b54-b372-579c34d46304"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:31.044349002Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1636,"client":"1","":"downstairs","session_id":"30585707-3599-4b54-b372-579c34d46304"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:31.044362744Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1636,"client":"1","":"downstairs","session_id":"30585707-3599-4b54-b372-579c34d46304"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:31.044376575Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1636,"client":"2","":"downstairs","session_id":"30585707-3599-4b54-b372-579c34d46304"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:31.044426052Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1636,"client":"2","":"downstairs","session_id":"30585707-3599-4b54-b372-579c34d46304"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:31.044907214Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1636,"client":"0","":"downstairs","session_id":"30585707-3599-4b54-b372-579c34d46304"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:31.044936448Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1636,"client":"1","":"downstairs","session_id":"30585707-3599-4b54-b372-579c34d46304"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:31.045183779Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1636,"client":"2","":"downstairs","session_id":"30585707-3599-4b54-b372-579c34d46304"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:31.045450154Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1636,"client":"1","":"downstairs","session_id":"30585707-3599-4b54-b372-579c34d46304"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:31.045480568Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1636,"client":"1","":"downstairs","session_id":"30585707-3599-4b54-b372-579c34d46304"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:31.04549632Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1636,"session_id":"30585707-3599-4b54-b372-579c34d46304"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:31.045517332Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1636,"client":"0","":"downstairs","session_id":"30585707-3599-4b54-b372-579c34d46304"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:31.045531854Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1636,"client":"0","":"downstairs","session_id":"30585707-3599-4b54-b372-579c34d46304"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:31.045545396Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1636,"session_id":"30585707-3599-4b54-b372-579c34d46304"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:31.045694055Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1636,"client":"2","":"downstairs","session_id":"30585707-3599-4b54-b372-579c34d46304"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:31.045714788Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1636,"client":"2","":"downstairs","session_id":"30585707-3599-4b54-b372-579c34d46304"} {"msg":"[0]R flush_numbers[0..12]: [174, 179, 165, 176, 180, 180, 180, 164, 173, 180, 180, 175]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:31.04572897Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1636,"":"downstairs","":"downstairs","session_id":"30585707-3599-4b54-b372-579c34d46304"} {"msg":"[0]R generation[0..12]: [110, 112, 105, 111, 113, 113, 113, 104, 109, 113, 113, 110]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:31.045743101Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1636,"":"downstairs","":"downstairs","session_id":"30585707-3599-4b54-b372-579c34d46304"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:31.045757273Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1636,"":"downstairs","":"downstairs","session_id":"30585707-3599-4b54-b372-579c34d46304"} {"msg":"[1]R flush_numbers[0..12]: [174, 179, 165, 176, 180, 180, 180, 164, 173, 180, 180, 175]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:31.045771155Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1636,"":"downstairs","":"downstairs","session_id":"30585707-3599-4b54-b372-579c34d46304"} {"msg":"[1]R generation[0..12]: [110, 112, 105, 111, 113, 113, 113, 104, 109, 113, 113, 110]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:31.045784927Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1636,"":"downstairs","":"downstairs","session_id":"30585707-3599-4b54-b372-579c34d46304"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:31.045798969Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1636,"":"downstairs","":"downstairs","session_id":"30585707-3599-4b54-b372-579c34d46304"} {"msg":"[2]R flush_numbers[0..12]: [174, 179, 165, 176, 180, 180, 180, 164, 173, 180, 180, 175]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:31.045813661Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1636,"":"downstairs","":"downstairs","session_id":"30585707-3599-4b54-b372-579c34d46304"} {"msg":"[2]R generation[0..12]: [110, 112, 105, 111, 113, 113, 113, 104, 109, 113, 113, 110]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:31.045830273Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1636,"":"downstairs","":"downstairs","session_id":"30585707-3599-4b54-b372-579c34d46304"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:31.045845505Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1636,"":"downstairs","":"downstairs","session_id":"30585707-3599-4b54-b372-579c34d46304"} {"msg":"Max found gen is 114","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:31.045859636Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1636,"":"downstairs","":"downstairs","session_id":"30585707-3599-4b54-b372-579c34d46304"} {"msg":"Generation requested: 114 >= found:114","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:31.045873488Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1636,"":"downstairs","":"downstairs","session_id":"30585707-3599-4b54-b372-579c34d46304"} {"msg":"Next flush: 181","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:31.04588659Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1636,"":"downstairs","":"downstairs","session_id":"30585707-3599-4b54-b372-579c34d46304"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:31.045899622Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP",Wait for a query_work_queue command to finish before sending IO "pid":1636,"":"downstairs","":"downstairs","session_id":"30585707-3599-4b54-b372-579c34d46304"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:31.045922284Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1636,"session_id":"30585707-3599-4b54-b372-579c34d46304"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:31.045934476Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1636,"client":"0","":"downstairs","session_id":"30585707-3599-4b54-b372-579c34d46304"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:31.045948388Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1636,"client":"1","":"downstairs","session_id":"30585707-3599-4b54-b372-579c34d46304"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:31.04596184Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1636,"client":"2","":"downstairs","session_id":"30585707-3599-4b54-b372-579c34d46304"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:31.045975531Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1636,"session_id":"30585707-3599-4b54-b372-579c34d46304"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:31.045987863Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1636,"session_id":"30585707-3599-4b54-b372-579c34d46304"} {"msg":"191aee1f-ab7a-4021-b95a-65f9058f399d is now active with session: 30585707-3599-4b54-b372-579c34d46304","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:31.046016427Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1636,"session_id":"30585707-3599-4b54-b372-579c34d46304"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:31.046029508Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1636,"session_id":"30585707-3599-4b54-b372-579c34d46304"} {"msg":"The guest has finished waiting for activation with:114","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:31.04604181Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1636} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 307 len 20480 data: 11 13 13 11 12 02/30 Read block 283 len 36864 03/30 Read block 388 len 40960 04/30 Read block 135 len 32768 05/30 Write block 479 len 28672 data: 11 11 10 9 10 13 14 06/30 Write block 473 len 12288 data: 13 11 9 07/30 Read block 6 len 28672 08/30 Write block 579 len 16384 data: 10 8 10 9 09/30 Write block 496 len 32768 data: 10 10 12 12 12 11 9 6 10/30 Write block 372 len 24576 data: 5 7 7 5 6 8 11/30 Read block 267 len 20480 12/30 Write block 488 len 24576 data: 17 15 14 12 9 6 13/30 Write block 339 len 40960 data: 6 7 4 5 6 8 8 9 9 10 14/30 Flush 15/30 Flush 16/30 Flush 17/30 Write block 86 len 28672 data: 11 13 14 15 17 13 11 18/30 Read block 41 len 32768 19/30 Flush 20/30 Write block 540 len 32768 data: 9 6 4 6 5 6 8 8 21/30 Flush 22/30 Write block 477 len 40960 data: 10 13 12 12 11 10 11 14 15 13 23/30 Read block 236 len 8192 24/30 Read block 359 len 20480 25/30 Write block 448 len 20480 data: 9 9 11 11 8 26/30 Write block 242 len 40960 data: 8 7 7 11 13 15 15 16 18 16 27/30 Write block 444 len 8192 data: 10 10 28/30 Read block 374 len 8192 29/30 Write block 123 len 32768 data: 9 9 10 11 11 12 13 15 30/30 Write block 270 len 28672 data: 12 15 12 13 16 15 15 ---------------------------------------------------------------- Crucible gen:114 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 10 Done Done Done false 1022 Acked Read 2 Done Done Done false 1023 Acked Read 5 Done Done Done false 1024 Acked Write 5 Done Done Done false 1025 Acked Write 10 Done Done Done false 1026 Acked Write 2 Done Done Done false 1027 Acked Read 2 Done Done Done false 1028 Acked Write 8 Sent Sent Sent false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 7 7 7 21 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1020 1020 1020 Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:54:31.072763153Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1636} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 110 110 110 174 174 174 F F F 1 020-039 112 112 112 179 179 179 F F F 2 040-059 105 105 105 165 165 165 F F F 3 060-079 111 111 111 176 176 176 F F F 4 080-099 114 114 114 184 184 184 F F F 5 100-119 113 113 113 180 180 180 F F F 6 120-139 113 113 113 180 180 180 T T T 7 140-159 104 104 104 164 164 164 F F F 8 160-179 109 109 109 173 173 173 F F F 9 180-199 113 113 113 180 180 180 F F F 10 200-219 113 113 113 180 180 180 F F F 11 220-239 110 110 110 175 175 175 F F F 12 240-259 111 111 111 176 176 176 T T T 13 260-279 111 111 111 176 176 176 T T T 14 280-299 111 111 111 176 176 176 F F F 15 300-319 114 114 114 181 181 181 F F F 16 320-339 114 114 114 181 181 181 F F F 17 340-359 114 114 114 181 181 181 F F F 18 360-379 114 114 114 181 181 181 F F F 19 380-399 110 110 110 175 175 175 F F F 20 400-419 111 111 111 176 176 176 F F F 21 420-439 110 110 110 174 174 174 F F F 22 440-459 113 113 113 180 180 180 T T T 23 460-479 114 114 114 181 181 181 T T T 24 480-499 114 114 114 181 181 181 T T T 25 500-519 114 114 114 181 181 181 F F F 26 520-539 113 113 113 180 180 180 F F F 27 540-559 114 114 114 185 185 185 F F F 28 560-579 114 114 114 181 181 181 F F F 29 580-599 114 114 114 181 181 181 F F F Max gen: 114, Max flush: 185 A Difference in extent metadata is expected here On loop 57 Verifying data now No change after verify Loop: 57 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 110 110 110 174 174 174 F F F 1 020-039 112 112 112 179 179 179 F F F 2 040-059 105 105 105 165 165 165 F F F 3 060-079 111 111 111 176 176 176 F F F 4 080-099 114 114 114 184 184 184 F F F 5 100-119 113 113 113 180 180 180 F F F 6 120-139 115 115 115 186 186 186 F F F 7 140-159 104 104 104 164 164 164 F F F 8 160-179 109 109 109 173 173 173 F F F 9 180-199 113 113 113 180 180 180 F F F 10 200-219 113 113 113 180 180 180 F F F 11 220-239 110 110 110 175 175 175 F F F 12 240-259 115 115 115 186 186 186 F F F 13 260-279 115 115 115 186 186 186 F F F 14 280-299 111 111 111 176 176 176 F F F 15 300-319 114 114 114 181 181 181 F F F 16 320-339 114 114 114 181 181 181 F F F 17 340-359 114 114 114 181 181 181 F F F 18 360-379 114 114 114 181 181 181 F F F 19 380-399 110 110 110 175 175 175 F F F 20 400-419 111 111 111 176 176 176 F F F 21 420-439 110 110 110 174 174 174 F F F 22 440-459 115 115 115 186 186 186 F F F 23 460-479 115 115 115 186 186 186 F F F 24 480-499 115 115 115 186 186 186 F F F 25 500-519 114 114 114 181 181 181 F F F 26 520-539 113 113 113 180 180 180 F F F 27 540-559 114 114 114 185 185 185 F F F 28 560-579 114 114 114 181 181 181 F F F 29 580-599 114 114 114 181 181 181 F F F Max gen: 115, Max flush: 186 Begin loop 58 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:35.394525191Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1644,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:35.438516139Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1644,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:35.439230351Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1644} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"3818f0fc76fe787c6cc412af30fd59b930203885\",\n git_commit_timestamp: \"2024-11-01T16:08:30.000000000Z\",\n git_branch: \"renovate/hyper-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:35.439271156Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1644} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:35.439285949Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1644} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:35.43930024Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1644} {"msg":"Crucible f1853d1c-3763-4c7b-969e-4e423f4b9ce1 has session id: 49cada38-f3f7-4b82-a4ea-6d62b0de2853","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:35.43968594Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1644,"session_id":"49cada38-f3f7-4b82-a4ea-6d62b0de2853"} {"msg":"Upstairs opts: Upstairs UUID: f1853d1c-3763-4c7b-969e-4e423f4b9ce1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:35.439725515Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1644,"session_id":"49cada38-f3f7-4b82-a4ea-6d62b0de2853"} {"msg":"Crucible stats registered with UUID: f1853d1c-3763-4c7b-969e-4e423f4b9ce1","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:35.439748728Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1644,"session_id":"49cada38-f3f7-4b82-a4ea-6d62b0de2853"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:35.470594432Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1644,"task":"crutest"} {"msg":"The guest has requested activation with gen:116","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:35.470625856Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1644} {"msg":"f1853d1c-3763-4c7b-969e-4e423f4b9ce1 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:35.470660821Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1644,"session_id":"49cada38-f3f7-4b82-a4ea-6d62b0de2853"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:35.470677163Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1644,"client":"0","":"downstairs","session_id":"49cada38-f3f7-4b82-a4ea-6d62b0de2853"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:35.470707257Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1644,"client":"0","":"downstairs","session_id":"49cada38-f3f7-4b82-a4ea-6d62b0de2853"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:35.470724729Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1644,"client":"1","":"downstairs","session_id":"49cada38-f3f7-4b82-a4ea-6d62b0de2853"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:35.470741851Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1644,"client":"1","":"downstairs","session_id":"49cada38-f3f7-4b82-a4ea-6d62b0de2853"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:35.470758764Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1644,"client":"2","":"downstairs","session_id":"49cada38-f3f7-4b82-a4ea-6d62b0de2853"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:35.470785337Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1644,"client":"2","":"downstairs","session_id":"49cada38-f3f7-4b82-a4ea-6d62b0de2853"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:35.470817181Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1644,"":"io task","client":"2","":"downstairs","session_id":"49cada38-f3f7-4b82-a4ea-6d62b0de2853"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:35.470851285Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1644,"":"io task","client":"0","":"downstairs","session_id":"49cada38-f3f7-4b82-a4ea-6d62b0de2853"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:35.47088539Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1644,"":"io task","client":"1","":"downstairs","session_id":"49cada38-f3f7-4b82-a4ea-6d62b0de2853"} {"msg":"ds_connection connected from Ok(127.0.0.1:38279)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:35.470948928Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1644,"":"io task","client":"1","":"downstairs","session_id":"49cada38-f3f7-4b82-a4ea-6d62b0de2853"} {"msg":"ds_connection connected from Ok(127.0.0.1:35114)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:35.470990154Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1644,"":"io task","client":"2","":"downstairs","session_id":"49cada38-f3f7-4b82-a4ea-6d62b0de2853"} {"msg":"ds_connection connected from Ok(127.0.0.1:49575)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:35.471115039Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1644,"":"io task","client":"0","":"downstairs","session_id":"49cada38-f3f7-4b82-a4ea-6d62b0de2853"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:35.471831652Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1644,"client":"0","":"downstairs","session_id":"49cada38-f3f7-4b82-a4ea-6d62b0de2853"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:35.471871127Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1644,"client":"0","":"downstairs","session_id":"49cada38-f3f7-4b82-a4ea-6d62b0de2853"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:35.471902391Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1644,"client":"1","":"downstairs","session_id":"49cada38-f3f7-4b82-a4ea-6d62b0de2853"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:35.471933455Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1644,"client":"1","":"downstairs","session_id":"49cada38-f3f7-4b82-a4ea-6d62b0de2853"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:35.471963379Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1644,"client":"2","":"downstairs","session_id":"49cada38-f3f7-4b82-a4ea-6d62b0de2853"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:35.471989332Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1644,"client":"2","":"downstairs","session_id":"49cada38-f3f7-4b82-a4ea-6d62b0de2853"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:35.472530742Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1644,"client":"2","":"downstairs","session_id":"49cada38-f3f7-4b82-a4ea-6d62b0de2853"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:35.472563846Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1644,"client":"0","":"downstairs","session_id":"49cada38-f3f7-4b82-a4ea-6d62b0de2853"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:35.4725931Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1644,"client":"1","":"downstairs","session_id":"49cada38-f3f7-4b82-a4ea-6d62b0de2853"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:35.473124238Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1644,"client":"2","":"downstairs","session_id":"49cada38-f3f7-4b82-a4ea-6d62b0de2853"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:35.473149602Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1644,"client":"2","":"downstairs","session_id":"49cada38-f3f7-4b82-a4ea-6d62b0de2853"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:35.473163734Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1644,"session_id":"49cada38-f3f7-4b82-a4ea-6d62b0de2853"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:35.473237063Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1644,"client":"1","":"downstairs","session_id":"49cada38-f3f7-4b82-a4ea-6d62b0de2853"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:35.473257476Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1644,"client":"1","":"downstairs","session_id":"49cada38-f3f7-4b82-a4ea-6d62b0de2853"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:35.473270987Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1644,"session_id":"49cada38-f3f7-4b82-a4ea-6d62b0de2853"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:35.473283159Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1644,"client":"0","":"downstairs","session_id":"49cada38-f3f7-4b82-a4ea-6d62b0de2853"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:35.473296341Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1644,"client":"0","":"downstairs","session_id":"49cada38-f3f7-4b82-a4ea-6d62b0de2853"} {"msg":"[0]R flush_numbers[0..12]: [174, 179, 165, 176, 184, 180, 186, 164, 173, 180, 180, 175]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:35.473309962Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1644,"":"downstairs","":"downstairs","session_id":"49cada38-f3f7-4b82-a4ea-6d62b0de2853"} {"msg":"[0]R generation[0..12]: [110, 112, 105, 111, 114, 113, 115, 104, 109, 113, 113, 110]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:35.473323624Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1644,"":"downstairs","":"downstairs","session_id":"49cada38-f3f7-4b82-a4ea-6d62b0de2853"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:35.473337566Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1644,"":"downstairs","":"downstairs","session_id":"49cada38-f3f7-4b82-a4ea-6d62b0de2853"} {"msg":"[1]R flush_numbers[0..12]: [174, 179, 165, 176, 184, 180, 186, 164, 173, 180, 180, 175]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:35.473351558Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1644,"":"downstairs","":"downstairs","session_id":"49cada38-f3f7-4b82-a4ea-6d62b0de2853"} {"msg":"[1]R generation[0..12]: [110, 112, 105, 111, 114, 113, 115, 104, 109, 113, 113, 110]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:35.473365249Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1644,"":"downstairs","":"downstairs","session_id":"49cada38-f3f7-4b82-a4ea-6d62b0de2853"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:35.473379121Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1644,"":"downstairs","":"downstairs","session_id":"49cada38-f3f7-4b82-a4ea-6d62b0de2853"} {"msg":"[2]R flush_numbers[0..12]: [174, 179, 165, 176, 184, 180, 186, 164, 173, 180, 180, 175]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:35.473392853Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1644,"":"downstairs","":"downstairs","session_id":"49cada38-f3f7-4b82-a4ea-6d62b0de2853"} {"msg":"[2]R generation[0..12]: [110, 112, 105, 111, 114, 113, 115, 104, 109, 113, 113, 110]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:35.473406575Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1644,"":"downstairs","":"downstairs","session_id":"49cada38-f3f7-4b82-a4ea-6d62b0de2853"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:35.473420467Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1644,"":"downstairs","":"downstairs","session_id":"49cada38-f3f7-4b82-a4ea-6d62b0de2853"} {"msg":"Max found gen is 116","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:35.473436429Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1644,"":"downstairs","":"downstairs","session_id":"49cada38-f3f7-4b82-a4ea-6d62b0de2853"} {"msg":"Generation requested: 116 >= found:116","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:35.47344985Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1644,"":"downstairs","":"downstairs","session_id":"49cada38-f3f7-4b82-a4ea-6d62b0de2853"} {"msg":"Next flush: 187","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:35.473463052Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1644,"":"downstairs","":"downstairs","session_id":"49cada38-f3f7-4b82-a4ea-6d62b0de2853"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:35.473476054Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1644,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO 49cada38-f3f7-4b82-a4ea-6d62b0de2853"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:35.473500037Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1644,"session_id":"49cada38-f3f7-4b82-a4ea-6d62b0de2853"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:35.473512079Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1644,"client":"0","":"downstairs","session_id":"49cada38-f3f7-4b82-a4ea-6d62b0de2853"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:35.4735252Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1644,"client":"1","":"downstairs","session_id":"49cada38-f3f7-4b82-a4ea-6d62b0de2853"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:35.473538452Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1644,"client":"2","":"downstairs","session_id":"49cada38-f3f7-4b82-a4ea-6d62b0de2853"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:35.473551564Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1644,"session_id":"49cada38-f3f7-4b82-a4ea-6d62b0de2853"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:35.473563785Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1644,"session_id":"49cada38-f3f7-4b82-a4ea-6d62b0de2853"} {"msg":"f1853d1c-3763-4c7b-969e-4e423f4b9ce1 is now active with session: 49cada38-f3f7-4b82-a4ea-6d62b0de2853","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:35.473576537Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1644,"session_id":"49cada38-f3f7-4b82-a4ea-6d62b0de2853"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:35.473588448Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1644,"session_id":"49cada38-f3f7-4b82-a4ea-6d62b0de2853"} {"msg":"The guest has finished waiting for activation with:116","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:35.47360063Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1644} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 400 len 24576 data: 10 8 8 7 6 4 02/30 Read block 154 len 16384 03/30 Write block 438 len 4096 data: 7 04/30 Write block 491 len 8192 data: 13 10 05/30 Read block 373 len 20480 06/30 Read block 116 len 8192 07/30 Write block 280 len 40960 data: 8 9 9 5 3 3 5 5 5 6 08/30 Read block 301 len 32768 09/30 Read block 436 len 32768 10/30 Read block 4 len 4096 11/30 Flush 12/30 Write block 497 len 16384 data: 11 13 13 13 13/30 Read block 378 len 16384 14/30 Read block 297 len 36864 15/30 Flush 16/30 Write block 594 len 24576 data: 7 8 7 6 5 3 17/30 Read block 416 len 32768 18/30 Read block 538 len 4096 19/30 Write block 272 len 40960 data: 13 14 17 16 16 11 9 9 9 10 20/30 Write block 538 len 8192 data: 11 11 21/30 Write block 388 len 24576 data: 7 8 7 7 7 7 22/30 Write block 361 len 4096 data: 5 23/30 Read block 106 len 40960 24/30 Write block 389 len 36864 data: 9 8 8 8 8 6 8 10 11 25/30 Flush 26/30 Flush 27/30 Write block 258 len 4096 data: 10 28/30 Read block 557 len 8192 29/30 Write block 584 len 12288 data: 12 14 12 30/30 Read block 79 len 16384 ---------------------------------------------------------------- Crucible gen:116 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 1 Done Done Done false 1027 Acked Read 2 Done Done Done false 1028 Acked Write 3 Sent Done Sent false 1029 Acked Read 4 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 0 2 4 Done 2 4 2 8 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:54:35.498729558Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1644} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 110 110 110 174 174 174 F F F 1 020-039 112 112 112 179 179 179 F F F 2 040-059 105 105 105 165 165 165 F F F 3 060-079 111 111 111 176 176 176 F F F 4 080-099 114 114 114 184 184 184 F F F 5 100-119 113 113 113 180 180 180 F F F 6 120-139 115 115 115 186 186 186 F F F 7 140-159 104 104 104 164 164 164 F F F 8 160-179 109 109 109 173 173 173 F F F 9 180-199 113 113 113 180 180 180 F F F 10 200-219 113 113 113 180 180 180 F F F 11 220-239 110 110 110 175 175 175 F F F 12 240-259 115 115 115 186 186 186 T T T 13 260-279 116 116 116 189 189 189 F F F 14 280-299 116 116 116 189 189 189 F F F 15 300-319 114 114 114 181 181 181 F F F 16 320-339 114 114 114 181 181 181 F F F 17 340-359 114 114 114 181 181 181 F F F 18 360-379 116 116 116 189 189 189 F F F 19 380-399 116 116 116 189 189 189 F F F 20 400-419 116 116 116 187 187 187 F F F 21 420-439 116 116 116 187 187 187 F F F 22 440-459 115 115 115 186 186 186 F F F 23 460-479 115 115 115 186 186 186 F F F 24 480-499 116 116 116 188 188 188 F F F 25 500-519 116 116 116 188 188 188 F F F 26 520-539 116 116 116 189 189 189 F F F 27 540-559 114 114 114 185 185 185 F F F 28 560-579 114 114 114 181 181 181 F F F 29 580-599 116 116 116 189 189 189 T T T Max gen: 116, Max flush: 189 A Difference in extent metadata is expected here On loop 58 Verifying data now No change after verify Loop: 58 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 110 110 110 174 174 174 F F F 1 020-039 112 112 112 179 179 179 F F F 2 040-059 105 105 105 165 165 165 F F F 3 060-079 111 111 111 176 176 176 F F F 4 080-099 114 114 114 184 184 184 F F F 5 100-119 113 113 113 180 180 180 F F F 6 120-139 115 115 115 186 186 186 F F F 7 140-159 104 104 104 164 164 164 F F F 8 160-179 109 109 109 173 173 173 F F F 9 180-199 113 113 113 180 180 180 F F F 10 200-219 113 113 113 180 180 180 F F F 11 220-239 110 110 110 175 175 175 F F F 12 240-259 117 117 117 190 190 190 F F F 13 260-279 116 116 116 189 189 189 F F F 14 280-299 116 116 116 189 189 189 F F F 15 300-319 114 114 114 181 181 181 F F F 16 320-339 114 114 114 181 181 181 F F F 17 340-359 114 114 114 181 181 181 F F F 18 360-379 116 116 116 189 189 189 F F F 19 380-399 116 116 116 189 189 189 F F F 20 400-419 116 116 116 187 187 187 F F F 21 420-439 116 116 116 187 187 187 F F F 22 440-459 115 115 115 186 186 186 F F F 23 460-479 115 115 115 186 186 186 F F F 24 480-499 116 116 116 188 188 188 F F F 25 500-519 116 116 116 188 188 188 F F F 26 520-539 116 116 116 189 189 189 F F F 27 540-559 114 114 114 185 185 185 F F F 28 560-579 114 114 114 181 181 181 F F F 29 580-599 117 117 117 190 190 190 F F F Max gen: 117, Max flush: 190 Begin loop 59 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:39.767811137Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1652,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:39.812044946Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1652,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:39.812683448Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1652} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"3818f0fc76fe787c6cc412af30fd59b930203885\",\n git_commit_timestamp: \"2024-11-01T16:08:30.000000000Z\",\n git_branch: \"renovate/hyper-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:39.812725663Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1652} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:39.812740685Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1652} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:39.812755317Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1652} {"msg":"Crucible d56265f0-88d6-4c63-953e-8a943040a5b1 has session id: e24740e3-1f28-41a3-b7c8-4811187345e6","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:39.813151148Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1652,"session_id":"e24740e3-1f28-41a3-b7c8-4811187345e6"} {"msg":"Upstairs opts: Upstairs UUID: d56265f0-88d6-4c63-953e-8a943040a5b1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:39.813194044Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1652,"session_id":"e24740e3-1f28-41a3-b7c8-4811187345e6"} {"msg":"Crucible stats registered with UUID: d56265f0-88d6-4c63-953e-8a943040a5b1","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:39.813218797Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1652,"session_id":"e24740e3-1f28-41a3-b7c8-4811187345e6"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:39.844003884Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1652,"task":"crutest"} {"msg":"The guest has requested activation with gen:118","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:39.844034328Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1652} {"msg":"d56265f0-88d6-4c63-953e-8a943040a5b1 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:39.844083294Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1652,"session_id":"e24740e3-1f28-41a3-b7c8-4811187345e6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:39.844102566Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1652,"client":"0","":"downstairs","session_id":"e24740e3-1f28-41a3-b7c8-4811187345e6"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:39.84413226Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1652,"client":"0","":"downstairs","session_id":"e24740e3-1f28-41a3-b7c8-4811187345e6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:39.844150363Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1652,"client":"1","":"downstairs","session_id":"e24740e3-1f28-41a3-b7c8-4811187345e6"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:39.844167955Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1652,"client":"1","":"downstairs","session_id":"e24740e3-1f28-41a3-b7c8-4811187345e6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:39.844198199Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1652,"client":"2","":"downstairs","session_id":"e24740e3-1f28-41a3-b7c8-4811187345e6"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:39.844230593Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1652,"client":"2","":"downstairs","session_id":"e24740e3-1f28-41a3-b7c8-4811187345e6"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:39.844262707Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1652,"":"io task","client":"2","":"downstairs","session_id":"e24740e3-1f28-41a3-b7c8-4811187345e6"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:39.844296501Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1652,"":"io task","client":"0","":"downstairs","session_id":"e24740e3-1f28-41a3-b7c8-4811187345e6"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:39.844329606Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1652,"":"io task","client":"1","":"downstairs","session_id":"e24740e3-1f28-41a3-b7c8-4811187345e6"} {"msg":"ds_connection connected from Ok(127.0.0.1:64935)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:39.844452831Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1652,"":"io task","client":"1","":"downstairs","session_id":"e24740e3-1f28-41a3-b7c8-4811187345e6"} {"msg":"ds_connection connected from Ok(127.0.0.1:34235)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:39.844614312Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1652,"":"io task","client":"2","":"downstairs","session_id":"e24740e3-1f28-41a3-b7c8-4811187345e6"} {"msg":"ds_connection connected from Ok(127.0.0.1:57642)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:39.844648507Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1652,"":"io task","client":"0","":"downstairs","session_id":"e24740e3-1f28-41a3-b7c8-4811187345e6"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:39.845260946Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1652,"client":"1","":"downstairs","session_id":"e24740e3-1f28-41a3-b7c8-4811187345e6"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:39.845288569Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1652,"client":"1","":"downstairs","session_id":"e24740e3-1f28-41a3-b7c8-4811187345e6"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:39.845330255Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1652,"client":"2","":"downstairs","session_id":"e24740e3-1f28-41a3-b7c8-4811187345e6"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:39.845359268Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1652,"client":"2","":"downstairs","session_id":"e24740e3-1f28-41a3-b7c8-4811187345e6"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:39.845388132Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1652,"client":"0","":"downstairs","session_id":"e24740e3-1f28-41a3-b7c8-4811187345e6"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:39.845418126Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1652,"client":"0","":"downstairs","session_id":"e24740e3-1f28-41a3-b7c8-4811187345e6"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:39.845854922Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1652,"client":"0","":"downstairs","session_id":"e24740e3-1f28-41a3-b7c8-4811187345e6"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:39.845939123Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1652,"client":"1","":"downstairs","session_id":"e24740e3-1f28-41a3-b7c8-4811187345e6"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:39.846048877Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1652,"client":"2","":"downstairs","session_id":"e24740e3-1f28-41a3-b7c8-4811187345e6"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:39.846420475Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1652,"client":"0","":"downstairs","session_id":"e24740e3-1f28-41a3-b7c8-4811187345e6"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:39.846447188Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1652,"client":"0","":"downstairs","session_id":"e24740e3-1f28-41a3-b7c8-4811187345e6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:39.846468981Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1652,"session_id":"e24740e3-1f28-41a3-b7c8-4811187345e6"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:39.846524329Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1652,"client":"1","":"downstairs","session_id":"e24740e3-1f28-41a3-b7c8-4811187345e6"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:39.846550712Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1652,"client":"1","":"downstairs","session_id":"e24740e3-1f28-41a3-b7c8-4811187345e6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:39.846565384Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1652,"session_id":"e24740e3-1f28-41a3-b7c8-4811187345e6"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:39.846578085Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1652,"client":"2","":"downstairs","session_id":"e24740e3-1f28-41a3-b7c8-4811187345e6"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:39.846591587Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1652,"client":"2","":"downstairs","session_id":"e24740e3-1f28-41a3-b7c8-4811187345e6"} {"msg":"[0]R flush_numbers[0..12]: [174, 179, 165, 176, 184, 180, 186, 164, 173, 180, 180, 175]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:39.846608939Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1652,"":"downstairs","":"downstairs","session_id":"e24740e3-1f28-41a3-b7c8-4811187345e6"} {"msg":"[0]R generation[0..12]: [110, 112, 105, 111, 114, 113, 115, 104, 109, 113, 113, 110]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:39.846623111Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1652,"":"downstairs","":"downstairs","session_id":"e24740e3-1f28-41a3-b7c8-4811187345e6"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:39.846637233Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1652,"":"downstairs","":"downstairs","session_id":"e24740e3-1f28-41a3-b7c8-4811187345e6"} {"msg":"[1]R flush_numbers[0..12]: [174, 179, 165, 176, 184, 180, 186, 164, 173, 180, 180, 175]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:39.846651135Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1652,"":"downstairs","":"downstairs","session_id":"e24740e3-1f28-41a3-b7c8-4811187345e6"} {"msg":"[1]R generation[0..12]: [110, 112, 105, 111, 114, 113, 115, 104, 109, 113, 113, 110]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:39.846664936Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1652,"":"downstairs","":"downstairs","session_id":"e24740e3-1f28-41a3-b7c8-4811187345e6"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:39.846679378Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1652,"":"downstairs","":"downstairs","session_id":"e24740e3-1f28-41a3-b7c8-4811187345e6"} {"msg":"[2]R flush_numbers[0..12]: [174, 179, 165, 176, 184, 180, 186, 164, 173, 180, 180, 175]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:39.84669342Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1652,"":"downstairs","":"downstairs","session_id":"e24740e3-1f28-41a3-b7c8-4811187345e6"} {"msg":"[2]R generation[0..12]: [110, 112, 105, 111, 114, 113, 115, 104, 109, 113, 113, 110]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:39.846707272Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1652,"":"downstairs","":"downstairs","session_id":"e24740e3-1f28-41a3-b7c8-4811187345e6"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:39.846721114Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1652,"":"downstairs","":"downstairs","session_id":"e24740e3-1f28-41a3-b7c8-4811187345e6"} {"msg":"Max found gen is 118","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:39.846734205Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1652,"":"downstairs","":"downstairs","session_id":"e24740e3-1f28-41a3-b7c8-4811187345e6"} {"msg":"Generation requested: 118 >= found:118","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:39.846747367Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1652,"":"downstairs","":"downstairs","session_id":"e24740e3-1f28-41a3-b7c8-4811187345e6"} {"msg":"Next flush: 191","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:39.846760429Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1652Wait for a query_work_queue command to finish before sending IO ,"":"downstairs","":"downstairs","session_id":"e24740e3-1f28-41a3-b7c8-4811187345e6"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:39.846781151Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1652,"":"downstairs","":"downstairs","session_id":"e24740e3-1f28-41a3-b7c8-4811187345e6"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:39.846794423Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1652,"session_id":"e24740e3-1f28-41a3-b7c8-4811187345e6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:39.846808315Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1652,"client":"0","":"downstairs","session_id":"e24740e3-1f28-41a3-b7c8-4811187345e6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:39.846821687Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1652,"client":"1","":"downstairs","session_id":"e24740e3-1f28-41a3-b7c8-4811187345e6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:39.846834669Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1652,"client":"2","":"downstairs","session_id":"e24740e3-1f28-41a3-b7c8-4811187345e6"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:39.84684778Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1652,"session_id":"e24740e3-1f28-41a3-b7c8-4811187345e6"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:39.846860192Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1652,"session_id":"e24740e3-1f28-41a3-b7c8-4811187345e6"} {"msg":"d56265f0-88d6-4c63-953e-8a943040a5b1 is now active with session: e24740e3-1f28-41a3-b7c8-4811187345e6","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:39.846873033Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1652,"session_id":"e24740e3-1f28-41a3-b7c8-4811187345e6"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:39.846885175Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1652,"session_id":"e24740e3-1f28-41a3-b7c8-4811187345e6"} {"msg":"The guest has finished waiting for activation with:118","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:39.846897386Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1652} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 540 len 24576 data: 10 7 5 7 6 7 02/30 Write block 173 len 4096 data: 6 03/30 Flush 04/30 Write block 504 len 4096 data: 7 05/30 Read block 397 len 20480 06/30 Write block 319 len 28672 data: 13 12 10 9 8 8 7 07/30 Write block 192 len 36864 data: 11 13 14 13 11 11 10 11 9 08/30 Write block 497 len 4096 data: 12 09/30 Read block 280 len 16384 10/30 Write block 246 len 12288 data: 14 16 16 11/30 Flush 12/30 Write block 484 len 4096 data: 15 13/30 Read block 524 len 32768 14/30 Write block 307 len 24576 data: 12 14 14 12 13 13 15/30 Write block 15 len 24576 data: 12 11 11 10 10 8 16/30 Write block 178 len 36864 data: 10 9 8 6 6 7 7 8 9 17/30 Read block 120 len 40960 18/30 Read block 378 len 24576 19/30 Read block 354 len 12288 20/30 Write block 44 len 40960 data: 14 11 13 10 8 12 12 10 10 8 21/30 Write block 295 len 16384 data: 9 10 10 9 22/30 Read block 520 len 36864 23/30 Write block 152 len 8192 data: 12 9 24/30 Read block 393 len 4096 25/30 Read block 287 len 24576 26/30 Write block 433 len 8192 data: 5 4 27/30 Write block 345 len 12288 data: 9 10 10 28/30 Write block 450 len 40960 data: 12 12 9 9 8 5 6 5 4 4 29/30 Read block 201 len 32768 30/30 Read block 241 len 12288 ---------------------------------------------------------------- Crucible gen:118 GIO:true work queues: Upstairs:0 downstairs:19 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1011 Acked Write 1 Done Done Done false 1012 Acked Read 8 Done Done Done false 1013 Acked Write 6 Done Done Done false 1014 Acked Write 6 Done Done Done false 1015 Acked Write 9 Done Done Done false 1016 Acked Read 10 Done Done Done false 1017 Acked Read 6 Done Done Done false 1018 Acked Read 3 Done Done Done false 1019 Acked Write 10 Done Done Done false 1020 Acked Write 4 Done Done Done false 1021 Acked Read 9 Done Done Done false 1022 Acked Write 2 Done Done Done false 1023 Acked Read 1 Done Done Done false 1024 Acked Read 6 Done Done Done false 1025 Acked Write 2 Done Done Done false 1026 Acked Write 3 Done Done Done false 1027 Acked Write 10 Sent Done Done false 1028 Acked Read 8 Sent Done Done false 1029 Acked Read 3 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 0 0 3 Done 16 19 19 54 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1010 1010 1010 Downstairs last five completed: 1010 1009 1008 1007 1006 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:54:39.868470356Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1652} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 110 110 110 174 174 174 T T T 1 020-039 112 112 112 179 179 179 T T T 2 040-059 105 105 105 165 165 165 T T T 3 060-079 111 111 111 176 176 176 F F F 4 080-099 114 114 114 184 184 184 F F F 5 100-119 113 113 113 180 180 180 F F F 6 120-139 115 115 115 186 186 186 F F F 7 140-159 104 104 104 164 164 164 T T T 8 160-179 118 118 118 191 191 191 T T T 9 180-199 118 118 118 192 192 192 T T T 10 200-219 118 118 118 192 192 192 F F F 11 220-239 110 110 110 175 175 175 F F F 12 240-259 118 118 118 192 192 192 F F F 13 260-279 116 116 116 189 189 189 F F F 14 280-299 116 116 116 189 189 189 T T T 15 300-319 118 118 118 192 192 192 T T T 16 320-339 118 118 118 192 192 192 F F F 17 340-359 114 114 114 181 181 181 T T T 18 360-379 116 116 116 189 189 189 F F F 19 380-399 116 116 116 189 189 189 F F F 20 400-419 116 116 116 187 187 187 F F F 21 420-439 116 116 116 187 187 187 T T T 22 440-459 115 115 115 186 186 186 T T T 23 460-479 115 115 115 186 186 186 F F F 24 480-499 118 118 118 192 192 192 T T T 25 500-519 118 118 118 192 192 192 F F F 26 520-539 116 116 116 189 189 189 F F F 27 540-559 118 118 118 191 191 191 F F F 28 560-579 114 114 114 181 181 181 F F F 29 580-599 117 117 117 190 190 190 F F F Max gen: 118, Max flush: 192 A Difference in extent metadata is expected here On loop 59 Verifying data now No change after verify Loop: 59 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 119 119 119 193 193 193 F F F 1 020-039 119 119 119 193 193 193 F F F 2 040-059 119 119 119 193 193 193 F F F 3 060-079 111 111 111 176 176 176 F F F 4 080-099 114 114 114 184 184 184 F F F 5 100-119 113 113 113 180 180 180 F F F 6 120-139 115 115 115 186 186 186 F F F 7 140-159 119 119 119 193 193 193 F F F 8 160-179 119 119 119 193 193 193 F F F 9 180-199 119 119 119 193 193 193 F F F 10 200-219 118 118 118 192 192 192 F F F 11 220-239 110 110 110 175 175 175 F F F 12 240-259 118 118 118 192 192 192 F F F 13 260-279 116 116 116 189 189 189 F F F 14 280-299 119 119 119 193 193 193 F F F 15 300-319 119 119 119 193 193 193 F F F 16 320-339 118 118 118 192 192 192 F F F 17 340-359 119 119 119 193 193 193 F F F 18 360-379 116 116 116 189 189 189 F F F 19 380-399 116 116 116 189 189 189 F F F 20 400-419 116 116 116 187 187 187 F F F 21 420-439 119 119 119 193 193 193 F F F 22 440-459 119 119 119 193 193 193 F F F 23 460-479 115 115 115 186 186 186 F F F 24 480-499 119 119 119 193 193 193 F F F 25 500-519 118 118 118 192 192 192 F F F 26 520-539 116 116 116 189 189 189 F F F 27 540-559 118 118 118 191 191 191 F F F 28 560-579 114 114 114 181 181 181 F F F 29 580-599 117 117 117 190 190 190 F F F Max gen: 119, Max flush: 193 Begin loop 60 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:44.275439811Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1660,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:54:44.317861037Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1660,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:44.318248517Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1660,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:44.350225967Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1660,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:44.350763727Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1660} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"3818f0fc76fe787c6cc412af30fd59b930203885\",\n git_commit_timestamp: \"2024-11-01T16:08:30.000000000Z\",\n git_branch: \"renovate/hyper-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:44.350800461Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1660} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:44.350814753Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1660} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:44.350828845Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1660} {"msg":"Crucible 440dcdc0-fbbd-4d6b-a497-e748b45fe70e has session id: 1f34e398-0412-411f-999e-ae9f94ec9c0d","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:44.351232467Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1660,"session_id":"1f34e398-0412-411f-999e-ae9f94ec9c0d"} {"msg":"Upstairs opts: Upstairs UUID: 440dcdc0-fbbd-4d6b-a497-e748b45fe70e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:44.351270292Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1660,"session_id":"1f34e398-0412-411f-999e-ae9f94ec9c0d"} {"msg":"Crucible stats registered with UUID: 440dcdc0-fbbd-4d6b-a497-e748b45fe70e","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:44.351297356Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1660,"session_id":"1f34e398-0412-411f-999e-ae9f94ec9c0d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:44.382639644Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1660,"task":"crutest"} {"msg":"The guest has requested activation with gen:120","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:44.382668718Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1660} {"msg":"440dcdc0-fbbd-4d6b-a497-e748b45fe70e active request set","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:44.382735626Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1660,"session_id":"1f34e398-0412-411f-999e-ae9f94ec9c0d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:44.382769181Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1660,"client":"0","":"downstairs","session_id":"1f34e398-0412-411f-999e-ae9f94ec9c0d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:44.382801705Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1660,"client":"0","":"downstairs","session_id":"1f34e398-0412-411f-999e-ae9f94ec9c0d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:44.382820117Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1660,"client":"1","":"downstairs","session_id":"1f34e398-0412-411f-999e-ae9f94ec9c0d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:44.38283746Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1660,"client":"1","":"downstairs","session_id":"1f34e398-0412-411f-999e-ae9f94ec9c0d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:44.382854602Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1660,"client":"2","":"downstairs","session_id":"1f34e398-0412-411f-999e-ae9f94ec9c0d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:44.382871564Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1660,"client":"2","":"downstairs","session_id":"1f34e398-0412-411f-999e-ae9f94ec9c0d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:44.382888146Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1660,"":"io task","client":"2","":"downstairs","session_id":"1f34e398-0412-411f-999e-ae9f94ec9c0d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:44.382905538Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1660,"":"io task","client":"0","":"downstairs","session_id":"1f34e398-0412-411f-999e-ae9f94ec9c0d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:44.382929471Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1660,"":"io task","client":"1","":"downstairs","session_id":"1f34e398-0412-411f-999e-ae9f94ec9c0d"} {"msg":"ds_connection connected from Ok(127.0.0.1:35783)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:44.382964196Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1660,"":"io task","client":"2","":"downstairs","session_id":"1f34e398-0412-411f-999e-ae9f94ec9c0d"} {"msg":"ds_connection connected from Ok(127.0.0.1:34460)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:44.38299749Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1660,"":"io task","client":"0","":"downstairs","session_id":"1f34e398-0412-411f-999e-ae9f94ec9c0d"} {"msg":"ds_connection connected from Ok(127.0.0.1:40284)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:44.383215228Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1660,"":"io task","client":"1","":"downstairs","session_id":"1f34e398-0412-411f-999e-ae9f94ec9c0d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:44.383756198Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1660,"client":"1","":"downstairs","session_id":"1f34e398-0412-411f-999e-ae9f94ec9c0d"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:44.383784081Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1660,"client":"1","":"downstairs","session_id":"1f34e398-0412-411f-999e-ae9f94ec9c0d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:44.383799373Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1660,"client":"2","":"downstairs","session_id":"1f34e398-0412-411f-999e-ae9f94ec9c0d"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:44.383813035Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1660,"client":"2","":"downstairs","session_id":"1f34e398-0412-411f-999e-ae9f94ec9c0d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:44.383826577Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1660,"client":"0","":"downstairs","session_id":"1f34e398-0412-411f-999e-ae9f94ec9c0d"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:44.383840149Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1660,"client":"0","":"downstairs","session_id":"1f34e398-0412-411f-999e-ae9f94ec9c0d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:44.384350244Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1660,"client":"1","":"downstairs","session_id":"1f34e398-0412-411f-999e-ae9f94ec9c0d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:44.384491193Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1660,"client":"0","":"downstairs","session_id":"1f34e398-0412-411f-999e-ae9f94ec9c0d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:44.384600767Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1660,"client":"2","":"downstairs","session_id":"1f34e398-0412-411f-999e-ae9f94ec9c0d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:44.384951932Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1660,"client":"1","":"downstairs","session_id":"1f34e398-0412-411f-999e-ae9f94ec9c0d"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:44.384978765Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1660,"client":"1","":"downstairs","session_id":"1f34e398-0412-411f-999e-ae9f94ec9c0d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:44.384993607Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1660,"session_id":"1f34e398-0412-411f-999e-ae9f94ec9c0d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:44.385043764Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1660,"client":"0","":"downstairs","session_id":"1f34e398-0412-411f-999e-ae9f94ec9c0d"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:44.385072697Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1660,"client":"0","":"downstairs","session_id":"1f34e398-0412-411f-999e-ae9f94ec9c0d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:44.385088959Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1660,"session_id":"1f34e398-0412-411f-999e-ae9f94ec9c0d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:44.385101721Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1660,"client":"2","":"downstairs","session_id":"1f34e398-0412-411f-999e-ae9f94ec9c0d"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:44.385119234Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1660,"client":"2","":"downstairs","session_id":"1f34e398-0412-411f-999e-ae9f94ec9c0d"} {"msg":"[0]R flush_numbers[0..12]: [193, 193, 193, 176, 184, 180, 186, 193, 193, 193, 192, 175]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:44.385133835Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1660,"":"downstairs","":"downstairs","session_id":"1f34e398-0412-411f-999e-ae9f94ec9c0d"} {"msg":"[0]R generation[0..12]: [119, 119, 119, 111, 114, 113, 115, 119, 119, 119, 118, 110]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:44.385147967Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1660,"":"downstairs","":"downstairs","session_id":"1f34e398-0412-411f-999e-ae9f94ec9c0d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:44.385161989Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1660,"":"downstairs","":"downstairs","session_id":"1f34e398-0412-411f-999e-ae9f94ec9c0d"} {"msg":"[1]R flush_numbers[0..12]: [193, 193, 193, 176, 184, 180, 186, 193, 193, 193, 192, 175]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:44.385175831Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1660,"":"downstairs","":"downstairs","session_id":"1f34e398-0412-411f-999e-ae9f94ec9c0d"} {"msg":"[1]R generation[0..12]: [119, 119, 119, 111, 114, 113, 115, 119, 119, 119, 118, 110]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:44.385189692Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1660,"":"downstairs","":"downstairs","session_id":"1f34e398-0412-411f-999e-ae9f94ec9c0d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:44.385203794Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1660,"":"downstairs","":"downstairs","session_id":"1f34e398-0412-411f-999e-ae9f94ec9c0d"} {"msg":"[2]R flush_numbers[0..12]: [193, 193, 193, 176, 184, 180, 186, 193, 193, 193, 192, 175]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:44.385223207Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1660,"":"downstairs","":"downstairs","session_id":"1f34e398-0412-411f-999e-ae9f94ec9c0d"} {"msg":"[2]R generation[0..12]: [119, 119, 119, 111, 114, 113, 115, 119, 119, 119, 118, 110]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:44.3852495Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1660,"":"downstairs","":"downstairs","session_id":"1f34e398-0412-411f-999e-ae9f94ec9c0d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:44.385278614Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1660,"":"downstairs","":"downstairs","session_id":"1f34e398-0412-411f-999e-ae9f94ec9c0d"} {"msg":"Max found gen is 120","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-01T16:54:44.385294986Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1660,"":"downstairs","":"downstairs","session_id":"1f34e398-0412-411f-999e-ae9f94ec9c0d"} {"msg":"Generation requested: 120 >= found:120","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:44.385374976Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1660,"":"downstairs","":"downstairs","session_id":"1f34e398-0412-411f-999e-ae9f94ec9c0d"} {"msg":"Next flush: 194","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:44.385396639Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1660,"":"downstairs","":"downstairs","session_id":"1f34e398-0412-411f-999e-ae9f94ec9c0d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:44.385415212Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1660,"":"downstairs","":"downstairs","session_id":"1f34e398-0412-411f-999e-ae9f94ec9c0d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:44.385440115Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1660,"session_id":"1f34e398-0412-411f-999e-ae9f94ec9c0d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:44.385463608Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1660,"client":"0","":"downstairs","session_id":"1f34e398-0412-411f-999e-ae9f94ec9c0d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:44.385490071Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1660,"client":"1","":"downstairs","session_id":"1f34e398-0412-411f-999e-ae9f94ec9c0d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:44.385505423Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1660,"client":"2","":"downstairs","session_id":"1f34e398-0412-411f-999e-ae9f94ec9c0d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:44.385526216Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1660,"session_id":"1f34e398-0412-411f-999e-ae9f94ec9c0d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:44.385549659Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1660,"session_id":"1f34e398-0412-411f-999e-ae9f94ec9c0d"} {"msg":"440dcdc0-fbbd-4d6b-a497-e748b45fe70e is now active with session: 1f34e398-0412-411f-999e-ae9f94ec9c0d","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:44.385575242Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1660,"session_id":"1f34e398-0412-411f-999e-ae9f94ec9c0d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:44.385592104Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1660,"session_id":"1f34e398-0412-411f-999e-ae9f94ec9c0d"} {"msg":"The guest has finished waiting for activation with:120","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:44.385605616Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1660} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 180 len 28672 data: 9 7 7 8 8 9 10 02/30 Write block 286 len 36864 data: 6 6 6 7 5 7 10 10 12 03/30 Read block 58 len 20480 04/30 Write block 110 len 28672 data: 11 11 10 10 10 9 7 05/30 Flush 06/30 Write block 122 len 8192 data: 12 10 07/30 Write block 294 len 8192 data: 13 10 08/30 Read block 442 len 4096 09/30 Read block 582 len 24576 10/30 Read block 13 len 40960 11/30 Read block 29 len 20480 12/30 Read block 438 len 24576 13/30 Read block 387 len 24576 14/30 Flush 15/30 Flush 16/30 Write block 426 len 4096 data: 12 17/30 Read block 383 len 36864 18/30 Flush 19/30 Flush 20/30 Write block 355 len 28672 data: 9 8 10 9 7 6 6 21/30 Read block 317 len 16384 22/30 Write block 392 len 28672 data: 9 9 7 9 11 12 12 23/30 Read block 337 len 8192 24/30 Read block 207 len 32768 25/30 Read block 120 len 28672 26/30 Write block 380 len 32768 data: 8 9 10 10 9 10 8 7 27/30 Write block 595 len 16384 data: 9 8 7 6 28/30 Write block 38 len 36864 data: 12 12 10 11 12 15 15 12 14 29/30 Read block 463 len 12288 30/30 Read block 152 len 32768 ---------------------------------------------------------------- Crucible gen:120 GIO:true work queues: Upstairs:0 downstairs:11 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1019 Acked Write 7 Done Done Done false 1020 Acked Read 4 Done Done Done false 1021 Acked Write 7 Done Done Done false 1022 Acked Read 2 Done Done Done false 1023 Acked Read 8 Done Done Done false 1024 Acked Read 7 Done Done Done false 1025 Acked Write 8 Done Done Done false 1026 Acked Write 4 Done Done Done false 1027 Acked Write 9 Done Done Done false 1028 Acked Read 3 Done Done Done false 1029 Acked Read 8 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 0 2 Done 10 10 11 31 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1018 1018 1018 Downstairs last five completed: 1018 1017 1016 1015 1014 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:54:44.410027623Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1660} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 119 119 119 193 193 193 F F F 1 020-039 119 119 119 193 193 193 T T T 2 040-059 119 119 119 193 193 193 T T T 3 060-079 111 111 111 176 176 176 F F F 4 080-099 114 114 114 184 184 184 F F F 5 100-119 120 120 120 194 194 194 F F F 6 120-139 120 120 120 195 195 195 F F F 7 140-159 119 119 119 193 193 193 F F F 8 160-179 119 119 119 193 193 193 F F F 9 180-199 120 120 120 194 194 194 F F F 10 200-219 118 118 118 192 192 192 F F F 11 220-239 110 110 110 175 175 175 F F F 12 240-259 118 118 118 192 192 192 F F F 13 260-279 116 116 116 189 189 189 F F F 14 280-299 120 120 120 195 195 195 F F F 15 300-319 119 119 119 193 193 193 F F F 16 320-339 118 118 118 192 192 192 F F F 17 340-359 119 119 119 193 193 193 T T T 18 360-379 116 116 116 189 189 189 T T T 19 380-399 116 116 116 189 189 189 T T T 20 400-419 116 116 116 187 187 187 F F F 21 420-439 120 120 120 197 197 197 F F F 22 440-459 119 119 119 193 193 193 F F F 23 460-479 115 115 115 186 186 186 F F F 24 480-499 119 119 119 193 193 193 F F F 25 500-519 118 118 118 192 192 192 F F F 26 520-539 116 116 116 189 189 189 F F F 27 540-559 118 118 118 191 191 191 F F F 28 560-579 114 114 114 181 181 181 F F F 29 580-599 117 117 117 190 190 190 T T T Max gen: 120, Max flush: 197 A Difference in extent metadata is expected here On loop 60 Verifying data now No change after verify Loop: 60 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 119 119 119 193 193 193 F F F 1 020-039 121 121 121 198 198 198 F F F 2 040-059 121 121 121 198 198 198 F F F 3 060-079 111 111 111 176 176 176 F F F 4 080-099 114 114 114 184 184 184 F F F 5 100-119 120 120 120 194 194 194 F F F 6 120-139 120 120 120 195 195 195 F F F 7 140-159 119 119 119 193 193 193 F F F 8 160-179 119 119 119 193 193 193 F F F 9 180-199 120 120 120 194 194 194 F F F 10 200-219 118 118 118 192 192 192 F F F 11 220-239 110 110 110 175 175 175 F F F 12 240-259 118 118 118 192 192 192 F F F 13 260-279 116 116 116 189 189 189 F F F 14 280-299 120 120 120 195 195 195 F F F 15 300-319 119 119 119 193 193 193 F F F 16 320-339 118 118 118 192 192 192 F F F 17 340-359 121 121 121 198 198 198 F F F 18 360-379 121 121 121 198 198 198 F F F 19 380-399 121 121 121 198 198 198 F F F 20 400-419 116 116 116 187 187 187 F F F 21 420-439 120 120 120 197 197 197 F F F 22 440-459 119 119 119 193 193 193 F F F 23 460-479 115 115 115 186 186 186 F F F 24 480-499 119 119 119 193 193 193 F F F 25 500-519 118 118 118 192 192 192 F F F 26 520-539 116 116 116 189 189 189 F F F 27 540-559 118 118 118 191 191 191 F F F 28 560-579 114 114 114 181 181 181 F F F 29 580-599 121 121 121 198 198 198 F F F Max gen: 121, Max flush: 198 Begin loop 61 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:48.766738814Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1668,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:54:48.809354485Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1668,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:48.809712191Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1668,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:48.841508198Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1668,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:48.842021394Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1668} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"3818f0fc76fe787c6cc412af30fd59b930203885\",\n git_commit_timestamp: \"2024-11-01T16:08:30.000000000Z\",\n git_branch: \"renovate/hyper-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:48.842053508Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1668} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:48.84206699Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1668} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:48.842081171Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1668} {"msg":"Crucible b072df9a-403d-4edc-a2cb-0a11f1636bfd has session id: be078436-665a-4fbf-9ed8-a96cd8c43f5d","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:48.842517388Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1668,"session_id":"be078436-665a-4fbf-9ed8-a96cd8c43f5d"} {"msg":"Upstairs opts: Upstairs UUID: b072df9a-403d-4edc-a2cb-0a11f1636bfd, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:48.842556923Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1668,"session_id":"be078436-665a-4fbf-9ed8-a96cd8c43f5d"} {"msg":"Crucible stats registered with UUID: b072df9a-403d-4edc-a2cb-0a11f1636bfd","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:48.842580536Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1668,"session_id":"be078436-665a-4fbf-9ed8-a96cd8c43f5d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:48.873352411Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1668,"task":"crutest"} {"msg":"The guest has requested activation with gen:122","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:48.873395236Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1668} {"msg":"b072df9a-403d-4edc-a2cb-0a11f1636bfd active request set","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:48.873466035Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1668,"session_id":"be078436-665a-4fbf-9ed8-a96cd8c43f5d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:48.87350123Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1668,"client":"0","":"downstairs","session_id":"be078436-665a-4fbf-9ed8-a96cd8c43f5d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:48.873532764Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1668,"client":"0","":"downstairs","session_id":"be078436-665a-4fbf-9ed8-a96cd8c43f5d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:48.873557977Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1668,"client":"1","":"downstairs","session_id":"be078436-665a-4fbf-9ed8-a96cd8c43f5d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:48.873591211Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1668,"client":"1","":"downstairs","session_id":"be078436-665a-4fbf-9ed8-a96cd8c43f5d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:48.873623036Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1668,"client":"2","":"downstairs","session_id":"be078436-665a-4fbf-9ed8-a96cd8c43f5d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:48.87365562Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1668,"client":"2","":"downstairs","session_id":"be078436-665a-4fbf-9ed8-a96cd8c43f5d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:48.873687684Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1668,"":"io task","client":"2","":"downstairs","session_id":"be078436-665a-4fbf-9ed8-a96cd8c43f5d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:48.873722828Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1668,"":"io task","client":"0","":"downstairs","session_id":"be078436-665a-4fbf-9ed8-a96cd8c43f5d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:48.873755453Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1668,"":"io task","client":"1","":"downstairs","session_id":"be078436-665a-4fbf-9ed8-a96cd8c43f5d"} {"msg":"ds_connection connected from Ok(127.0.0.1:61662)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:48.873787377Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1668,"":"io task","client":"1","":"downstairs","session_id":"be078436-665a-4fbf-9ed8-a96cd8c43f5d"} {"msg":"ds_connection connected from Ok(127.0.0.1:52026)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:48.873819191Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1668,"":"io task","client":"0","":"downstairs","session_id":"be078436-665a-4fbf-9ed8-a96cd8c43f5d"} {"msg":"ds_connection connected from Ok(127.0.0.1:46567)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:48.873849895Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1668,"":"io task","client":"2","":"downstairs","session_id":"be078436-665a-4fbf-9ed8-a96cd8c43f5d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:48.874527362Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1668,"client":"2","":"downstairs","session_id":"be078436-665a-4fbf-9ed8-a96cd8c43f5d"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:48.874557906Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1668,"client":"2","":"downstairs","session_id":"be078436-665a-4fbf-9ed8-a96cd8c43f5d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:48.874573648Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1668,"client":"0","":"downstairs","session_id":"be078436-665a-4fbf-9ed8-a96cd8c43f5d"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:48.874590971Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1668,"client":"0","":"downstairs","session_id":"be078436-665a-4fbf-9ed8-a96cd8c43f5d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:48.874616004Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1668,"client":"1","":"downstairs","session_id":"be078436-665a-4fbf-9ed8-a96cd8c43f5d"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:48.874642407Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1668,"client":"1","":"downstairs","session_id":"be078436-665a-4fbf-9ed8-a96cd8c43f5d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:48.875145812Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1668,"client":"2","":"downstairs","session_id":"be078436-665a-4fbf-9ed8-a96cd8c43f5d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:48.875224822Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1668,"client":"1","":"downstairs","session_id":"be078436-665a-4fbf-9ed8-a96cd8c43f5d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:48.875257106Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1668,"client":"0","":"downstairs","session_id":"be078436-665a-4fbf-9ed8-a96cd8c43f5d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:48.875708004Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1668,"client":"2","":"downstairs","session_id":"be078436-665a-4fbf-9ed8-a96cd8c43f5d"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:48.875733888Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1668,"client":"2","":"downstairs","session_id":"be078436-665a-4fbf-9ed8-a96cd8c43f5d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:48.875748049Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1668,"session_id":"be078436-665a-4fbf-9ed8-a96cd8c43f5d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:48.875840912Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1668,"client":"1","":"downstairs","session_id":"be078436-665a-4fbf-9ed8-a96cd8c43f5d"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:48.875860794Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1668,"client":"1","":"downstairs","session_id":"be078436-665a-4fbf-9ed8-a96cd8c43f5d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:48.875874326Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1668,"session_id":"be078436-665a-4fbf-9ed8-a96cd8c43f5d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:48.875886387Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1668,"client":"0","":"downstairs","session_id":"be078436-665a-4fbf-9ed8-a96cd8c43f5d"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:48.875899779Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1668,"client":"0","":"downstairs","session_id":"be078436-665a-4fbf-9ed8-a96cd8c43f5d"} {"msg":"[0]R flush_numbers[0..12]: [193, 198, 198, 176, 184, 194, 195, 193, 193, 194, 192, 175]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:48.875916691Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1668,"":"downstairs","":"downstairs","session_id":"be078436-665a-4fbf-9ed8-a96cd8c43f5d"} {"msg":"[0]R generation[0..12]: [119, 121, 121, 111, 114, 120, 120, 119, 119, 120, 118, 110]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:48.875930963Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1668,"":"downstairs","":"downstairs","session_id":"be078436-665a-4fbf-9ed8-a96cd8c43f5d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:48.875944815Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1668,"":"downstairs","":"downstairs","session_id":"be078436-665a-4fbf-9ed8-a96cd8c43f5d"} {"msg":"[1]R flush_numbers[0..12]: [193, 198, 198, 176, 184, 194, 195, 193, 193, 194, 192, 175]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:48.875958697Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1668,"":"downstairs","":"downstairs","session_id":"be078436-665a-4fbf-9ed8-a96cd8c43f5d"} {"msg":"[1]R generation[0..12]: [119, 121, 121, 111, 114, 120, 120, 119, 119, 120, 118, 110]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:48.875972388Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1668,"":"downstairs","":"downstairs","session_id":"be078436-665a-4fbf-9ed8-a96cd8c43f5d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:48.87598617Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1668,"":"downstairs","":"downstairs","session_id":"be078436-665a-4fbf-9ed8-a96cd8c43f5d"} {"msg":"[2]R flush_numbers[0..12]: [193, 198, 198, 176, 184, 194, 195, 193, 193, 194, 192, 175]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:48.875999992Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1668,"":"downstairs","":"downstairs","session_id":"be078436-665a-4fbf-9ed8-a96cd8c43f5d"} {"msg":"[2]R generation[0..12]: [119, 121, 121, 111, 114, 120, 120, 119, 119, 120, 118, 110]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:48.876014674Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1668,"":"downstairs","":"downstairs","session_id":"be078436-665a-4fbf-9ed8-a96cd8c43f5d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:48.876028546Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1668,"":"downstairs","":"downstairs","session_id":"be078436-665a-4fbf-9ed8-a96cd8c43f5d"} {"msg":"Max found gen is 122","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:48.876041707Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1668,"":"downstairs","":"downstairs","session_id":"be078436-665a-4fbf-9ed8-a96cd8c43f5d"} {"msg":"Generation requested: 122 >= found:122","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:48.876054879Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1668,"":"downstairs","":"downstairs","session_id":"be078436-665a-4fbf-9ed8-a96cd8c43f5d"} {"msg":"Next flush: 199","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:48.876068071Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1668,"":"downstairs","":"downstairs","session_id":"be078436-665a-4fbf-9ed8-a96cd8c43f5d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:48.876081092Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1668,"":"downstairs","":"downstairs","session_id":"be078436-665a-4fbf-9ed8-a96cd8c43f5d"} {"Wait for a query_work_queue command to finish before sending IO msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:48.876102295Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1668,"session_id":"be078436-665a-4fbf-9ed8-a96cd8c43f5d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:48.876116207Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1668,"client":"0","":"downstairs","session_id":"be078436-665a-4fbf-9ed8-a96cd8c43f5d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:48.876129489Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1668,"client":"1","":"downstairs","session_id":"be078436-665a-4fbf-9ed8-a96cd8c43f5d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:48.87614258Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1668,"client":"2","":"downstairs","session_id":"be078436-665a-4fbf-9ed8-a96cd8c43f5d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:48.876155822Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1668,"session_id":"be078436-665a-4fbf-9ed8-a96cd8c43f5d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:48.876167974Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1668,"session_id":"be078436-665a-4fbf-9ed8-a96cd8c43f5d"} {"msg":"b072df9a-403d-4edc-a2cb-0a11f1636bfd is now active with session: be078436-665a-4fbf-9ed8-a96cd8c43f5d","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:48.876180845Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1668,"session_id":"be078436-665a-4fbf-9ed8-a96cd8c43f5d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:48.876192817Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1668,"session_id":"be078436-665a-4fbf-9ed8-a96cd8c43f5d"} {"msg":"The guest has finished waiting for activation with:122","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:48.876204818Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1668} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 367 len 4096 data: 5 03/30 Flush 04/30 Write block 191 len 32768 data: 11 12 14 15 14 12 12 11 05/30 Read block 52 len 12288 06/30 Read block 369 len 40960 07/30 Write block 71 len 20480 data: 16 16 11 10 9 08/30 Read block 208 len 8192 09/30 Write block 286 len 32768 data: 7 7 7 8 6 8 11 11 10/30 Read block 266 len 12288 11/30 Read block 586 len 16384 12/30 Write block 401 len 32768 data: 9 9 8 7 5 7 7 7 13/30 Write block 253 len 24576 data: 12 11 9 8 8 11 14/30 Flush 15/30 Write block 531 len 40960 data: 11 11 12 14 14 13 11 12 12 11 16/30 Write block 377 len 12288 data: 9 10 10 17/30 Flush 18/30 Write block 522 len 4096 data: 7 19/30 Write block 559 len 24576 data: 18 13 12 12 14 14 20/30 Read block 535 len 36864 21/30 Read block 207 len 4096 22/30 Read block 1 len 20480 23/30 Flush 24/30 Write block 321 len 16384 data: 11 10 9 9 25/30 Write block 96 len 4096 data: 13 26/30 Write block 156 len 8192 data: 12 9 27/30 Write block 393 len 28672 data: 10 8 10 12 13 13 11 28/30 Read block 12 len 8192 29/30 Write block 236 len 4096 data: 8 30/30 Write block 24 len 20480 data: 12 13 11 14 15 ---------------------------------------------------------------- Crucible gen:122 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 4 Done Done Done false 1024 Acked Write 1 Done Done Done false 1025 Acked Write 2 Done Done Done false 1026 Acked Write 7 Done Done Done false 1027 Acked Read 2 Done Done Done false 1028 Acked Write 1 Done Done Done false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 6 6 6 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:54:48.901964338Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1668} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 119 119 119 193 193 193 F F F 1 020-039 121 121 121 198 198 198 T T T 2 040-059 121 121 121 198 198 198 F F F 3 060-079 122 122 122 201 201 201 F F F 4 080-099 114 114 114 184 184 184 T T T 5 100-119 120 120 120 194 194 194 F F F 6 120-139 120 120 120 195 195 195 F F F 7 140-159 119 119 119 193 193 193 T T T 8 160-179 119 119 119 193 193 193 F F F 9 180-199 122 122 122 201 201 201 F F F 10 200-219 118 118 118 192 192 192 F F F 11 220-239 110 110 110 175 175 175 T T T 12 240-259 122 122 122 201 201 201 F F F 13 260-279 116 116 116 189 189 189 F F F 14 280-299 122 122 122 201 201 201 F F F 15 300-319 119 119 119 193 193 193 F F F 16 320-339 118 118 118 192 192 192 T T T 17 340-359 121 121 121 198 198 198 F F F 18 360-379 122 122 122 202 202 202 F F F 19 380-399 121 121 121 198 198 198 T T T 20 400-419 122 122 122 201 201 201 F F F 21 420-439 120 120 120 197 197 197 F F F 22 440-459 119 119 119 193 193 193 F F F 23 460-479 115 115 115 186 186 186 F F F 24 480-499 119 119 119 193 193 193 F F F 25 500-519 118 118 118 192 192 192 F F F 26 520-539 122 122 122 203 203 203 F F F 27 540-559 122 122 122 203 203 203 F F F 28 560-579 122 122 122 203 203 203 F F F 29 580-599 121 121 121 198 198 198 F F F Max gen: 122, Max flush: 203 A Difference in extent metadata is expected here On loop 61 Verifying data now No change after verify Loop: 61 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 119 119 119 193 193 193 F F F 1 020-039 123 123 123 204 204 204 F F F 2 040-059 121 121 121 198 198 198 F F F 3 060-079 122 122 122 201 201 201 F F F 4 080-099 123 123 123 204 204 204 F F F 5 100-119 120 120 120 194 194 194 F F F 6 120-139 120 120 120 195 195 195 F F F 7 140-159 123 123 123 204 204 204 F F F 8 160-179 119 119 119 193 193 193 F F F 9 180-199 122 122 122 201 201 201 F F F 10 200-219 118 118 118 192 192 192 F F F 11 220-239 123 123 123 204 204 204 F F F 12 240-259 122 122 122 201 201 201 F F F 13 260-279 116 116 116 189 189 189 F F F 14 280-299 122 122 122 201 201 201 F F F 15 300-319 119 119 119 193 193 193 F F F 16 320-339 123 123 123 204 204 204 F F F 17 340-359 121 121 121 198 198 198 F F F 18 360-379 122 122 122 202 202 202 F F F 19 380-399 123 123 123 204 204 204 F F F 20 400-419 122 122 122 201 201 201 F F F 21 420-439 120 120 120 197 197 197 F F F 22 440-459 119 119 119 193 193 193 F F F 23 460-479 115 115 115 186 186 186 F F F 24 480-499 119 119 119 193 193 193 F F F 25 500-519 118 118 118 192 192 192 F F F 26 520-539 122 122 122 203 203 203 F F F 27 540-559 122 122 122 203 203 203 F F F 28 560-579 122 122 122 203 203 203 F F F 29 580-599 121 121 121 198 198 198 F F F Max gen: 123, Max flush: 204 Begin loop 62 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:53.253439374Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1676,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:53.298677162Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1676,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:53.299364341Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1676} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"3818f0fc76fe787c6cc412af30fd59b930203885\",\n git_commit_timestamp: \"2024-11-01T16:08:30.000000000Z\",\n git_branch: \"renovate/hyper-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:53.299417528Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1676} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:53.299439361Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1676} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:53.299479666Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1676} {"msg":"Crucible 516111c5-6321-4dde-9388-cd7cb80966fe has session id: 8059958e-ed6c-40cd-b565-55e73fa99903","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:53.299955097Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1676,"session_id":"8059958e-ed6c-40cd-b565-55e73fa99903"} {"msg":"Upstairs opts: Upstairs UUID: 516111c5-6321-4dde-9388-cd7cb80966fe, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:53.300015475Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1676,"session_id":"8059958e-ed6c-40cd-b565-55e73fa99903"} {"msg":"Crucible stats registered with UUID: 516111c5-6321-4dde-9388-cd7cb80966fe","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:53.300041708Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1676,"session_id":"8059958e-ed6c-40cd-b565-55e73fa99903"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:53.330674765Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1676,"task":"crutest"} {"msg":"The guest has requested activation with gen:124","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:53.330771687Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1676} {"msg":"516111c5-6321-4dde-9388-cd7cb80966fe active request set","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:53.330802771Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1676,"session_id":"8059958e-ed6c-40cd-b565-55e73fa99903"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:53.330820464Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1676,"client":"0","":"downstairs","session_id":"8059958e-ed6c-40cd-b565-55e73fa99903"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:53.330853268Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1676,"client":"0","":"downstairs","session_id":"8059958e-ed6c-40cd-b565-55e73fa99903"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:53.330872971Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1676,"client":"1","":"downstairs","session_id":"8059958e-ed6c-40cd-b565-55e73fa99903"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:53.330902704Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1676,"client":"1","":"downstairs","session_id":"8059958e-ed6c-40cd-b565-55e73fa99903"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:53.330934759Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1676,"client":"2","":"downstairs","session_id":"8059958e-ed6c-40cd-b565-55e73fa99903"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:53.330966073Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1676,"client":"2","":"downstairs","session_id":"8059958e-ed6c-40cd-b565-55e73fa99903"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:53.330997907Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1676,"":"io task","client":"2","":"downstairs","session_id":"8059958e-ed6c-40cd-b565-55e73fa99903"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:53.331034051Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1676,"":"io task","client":"0","":"downstairs","session_id":"8059958e-ed6c-40cd-b565-55e73fa99903"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:53.331067836Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1676,"":"io task","client":"1","":"downstairs","session_id":"8059958e-ed6c-40cd-b565-55e73fa99903"} {"msg":"ds_connection connected from Ok(127.0.0.1:49070)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:53.331188611Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1676,"":"io task","client":"2","":"downstairs","session_id":"8059958e-ed6c-40cd-b565-55e73fa99903"} {"msg":"ds_connection connected from Ok(127.0.0.1:45284)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:53.331235967Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1676,"":"io task","client":"0","":"downstairs","session_id":"8059958e-ed6c-40cd-b565-55e73fa99903"} {"msg":"ds_connection connected from Ok(127.0.0.1:64670)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:53.331274072Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1676,"":"io task","client":"1","":"downstairs","session_id":"8059958e-ed6c-40cd-b565-55e73fa99903"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:53.331819573Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1676,"client":"2","":"downstairs","session_id":"8059958e-ed6c-40cd-b565-55e73fa99903"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:53.331855827Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1676,"client":"2","":"downstairs","session_id":"8059958e-ed6c-40cd-b565-55e73fa99903"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:53.331871869Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1676,"client":"0","":"downstairs","session_id":"8059958e-ed6c-40cd-b565-55e73fa99903"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:53.331887541Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1676,"client":"0","":"downstairs","session_id":"8059958e-ed6c-40cd-b565-55e73fa99903"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:53.332096788Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1676,"client":"1","":"downstairs","session_id":"8059958e-ed6c-40cd-b565-55e73fa99903"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:53.332138754Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1676,"client":"1","":"downstairs","session_id":"8059958e-ed6c-40cd-b565-55e73fa99903"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:53.332512882Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1676,"client":"0","":"downstairs","session_id":"8059958e-ed6c-40cd-b565-55e73fa99903"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:53.332586141Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1676,"client":"2","":"downstairs","session_id":"8059958e-ed6c-40cd-b565-55e73fa99903"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:53.332646669Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1676,"client":"1","":"downstairs","session_id":"8059958e-ed6c-40cd-b565-55e73fa99903"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:53.333056202Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1676,"client":"2","":"downstairs","session_id":"8059958e-ed6c-40cd-b565-55e73fa99903"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:53.333080865Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1676,"client":"2","":"downstairs","session_id":"8059958e-ed6c-40cd-b565-55e73fa99903"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:53.333095207Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1676,"session_id":"8059958e-ed6c-40cd-b565-55e73fa99903"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:53.333151254Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1676,"client":"0","":"downstairs","session_id":"8059958e-ed6c-40cd-b565-55e73fa99903"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:53.333170147Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1676,"client":"0","":"downstairs","session_id":"8059958e-ed6c-40cd-b565-55e73fa99903"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:53.333183618Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1676,"session_id":"8059958e-ed6c-40cd-b565-55e73fa99903"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:53.333220703Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1676,"client":"1","":"downstairs","session_id":"8059958e-ed6c-40cd-b565-55e73fa99903"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:53.333237025Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1676,"client":"1","":"downstairs","session_id":"8059958e-ed6c-40cd-b565-55e73fa99903"} {"msg":"[0]R flush_numbers[0..12]: [193, 204, 198, 201, 204, 194, 195, 204, 193, 201, 192, 204]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:53.333251117Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1676,"":"downstairs","":"downstairs","session_id":"8059958e-ed6c-40cd-b565-55e73fa99903"} {"msg":"[0]R generation[0..12]: [119, 123, 121, 122, 123, 120, 120, 123, 119, 122, 118, 123]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:53.333265259Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1676,"":"downstairs","":"downstairs","session_id":"8059958e-ed6c-40cd-b565-55e73fa99903"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:53.333279621Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1676,"":"downstairs","":"downstairs","session_id":"8059958e-ed6c-40cd-b565-55e73fa99903"} {"msg":"[1]R flush_numbers[0..12]: [193, 204, 198, 201, 204, 194, 195, 204, 193, 201, 192, 204]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:53.333297003Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1676,"":"downstairs","":"downstairs","session_id":"8059958e-ed6c-40cd-b565-55e73fa99903"} {"msg":"[1]R generation[0..12]: [119, 123, 121, 122, 123, 120, 120, 123, 119, 122, 118, 123]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:53.333311035Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1676,"":"downstairs","":"downstairs","session_id":"8059958e-ed6c-40cd-b565-55e73fa99903"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:53.333324927Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1676,"":"downstairs","":"downstairs","session_id":"8059958e-ed6c-40cd-b565-55e73fa99903"} {"msg":"[2]R flush_numbers[0..12]: [193, 204, 198, 201, 204, 194, 195, 204, 193, 201, 192, 204]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:53.333338838Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1676,"":"downstairs","":"downstairs","session_id":"8059958e-ed6c-40cd-b565-55e73fa99903"} {"msg":"[2]R generation[0..12]: [119, 123, 121, 122, 123, 120, 120, 123, 119, 122, 118, 123]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:53.33335252Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1676,"":"downstairs","":"downstairs","session_id":"8059958e-ed6c-40cd-b565-55e73fa99903"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:53.333366642Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1676,"":"downstairs","":"downstairs","session_id":"8059958e-ed6c-40cd-b565-55e73fa99903"} {"msg":"Max found gen is 124","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:53.333379674Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1676,"":"downstairs","":"downstairs","session_id":"8059958e-ed6c-40cd-b565-55e73fa99903"} {"msg":"Generation requested: 124 >= found:124","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:53.333392855Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1676,"":"downstairs","":"downstairs","session_id":"8059958e-ed6c-40cd-b565-55e73fa99903"} {"msg":"Next flush: 205","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:53.333405887Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1676,"":"downstairs","":"downstairs","session_id":"8059958e-ed6c-40cd-b565-55e73fa99903"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:53.333418859Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1676,"":"downstairs","":"downstairs","session_id":"8059958e-ed6c-40cd-b565-55e73fa99903"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:53.33343219Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1676,"session_id":"8059958e-ed6c-40cd-b565-55e73fa99903"} {"Wait for a query_work_queue command to finish before sending IO msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:53.333452433Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1676,"client":"0","":"downstairs","session_id":"8059958e-ed6c-40cd-b565-55e73fa99903"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:53.333466145Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1676,"client":"1","":"downstairs","session_id":"8059958e-ed6c-40cd-b565-55e73fa99903"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:53.333479307Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1676,"client":"2","":"downstairs","session_id":"8059958e-ed6c-40cd-b565-55e73fa99903"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:53.333494218Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1676,"session_id":"8059958e-ed6c-40cd-b565-55e73fa99903"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:53.33350684Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1676,"session_id":"8059958e-ed6c-40cd-b565-55e73fa99903"} {"msg":"516111c5-6321-4dde-9388-cd7cb80966fe is now active with session: 8059958e-ed6c-40cd-b565-55e73fa99903","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:53.333519892Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1676,"session_id":"8059958e-ed6c-40cd-b565-55e73fa99903"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:53.333532013Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1676,"session_id":"8059958e-ed6c-40cd-b565-55e73fa99903"} {"msg":"The guest has finished waiting for activation with:124","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:53.333544285Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1676} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 215 len 40960 data: 10 13 14 14 12 12 12 12 15 14 02/30 Read block 289 len 32768 03/30 Write block 141 len 40960 data: 7 7 4 5 6 5 7 7 10 13 04/30 Write block 566 len 8192 data: 6 7 05/30 Read block 50 len 8192 06/30 Read block 95 len 36864 07/30 Read block 219 len 28672 08/30 Write block 95 len 8192 data: 11 14 09/30 Read block 551 len 20480 10/30 Write block 195 len 8192 data: 15 13 11/30 Write block 556 len 32768 data: 11 12 16 19 14 13 13 15 12/30 Flush 13/30 Write block 502 len 12288 data: 10 7 8 14/30 Flush 15/30 Write block 163 len 36864 data: 9 8 9 7 6 7 7 8 6 16/30 Write block 576 len 16384 data: 8 9 11 11 17/30 Read block 298 len 40960 18/30 Flush 19/30 Write block 157 len 32768 data: 10 9 8 10 10 8 10 9 20/30 Read block 363 len 8192 21/30 Read block 36 len 36864 22/30 Read block 481 len 28672 23/30 Write block 290 len 12288 data: 7 9 12 24/30 Read block 580 len 28672 25/30 Read block 155 len 12288 26/30 Write block 528 len 40960 data: 9 10 12 12 12 13 15 15 14 12 27/30 Write block 321 len 8192 data: 12 11 28/30 Read block 133 len 32768 29/30 Read block 130 len 40960 30/30 Read block 429 len 16384 ---------------------------------------------------------------- Crucible gen:124 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1018 Acked Write 8 Done Done Done false 1019 Acked Read 2 Done Done Done false 1020 Acked Read 9 Done Done Done false 1021 Acked Read 7 Done Done Done false 1022 Acked Write 3 Done Done Done false 1023 Acked Read 7 Done Done Done false 1024 Acked Read 3 Done Done Done false 1025 Acked Write 10 Done Done Done false 1026 Acked Write 2 Done Done Done false 1027 Acked Read 8 Done Done Done false 1028 Acked Read 10 Done Done Done false 1029 Acked Read 4 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 12 12 11 35 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1017 1017 1017 Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:54:53.359322606Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1676} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 119 119 119 193 193 193 F F F 1 020-039 123 123 123 204 204 204 F F F 2 040-059 121 121 121 198 198 198 F F F 3 060-079 122 122 122 201 201 201 F F F 4 080-099 124 124 124 205 205 205 F F F 5 100-119 120 120 120 194 194 194 F F F 6 120-139 120 120 120 195 195 195 F F F 7 140-159 124 124 124 205 205 205 T T T 8 160-179 124 124 124 207 207 207 T T T 9 180-199 124 124 124 205 205 205 F F F 10 200-219 124 124 124 205 205 205 F F F 11 220-239 124 124 124 205 205 205 F F F 12 240-259 122 122 122 201 201 201 F F F 13 260-279 116 116 116 189 189 189 F F F 14 280-299 122 122 122 201 201 201 T T T 15 300-319 119 119 119 193 193 193 F F F 16 320-339 123 123 123 204 204 204 T T T 17 340-359 121 121 121 198 198 198 F F F 18 360-379 122 122 122 202 202 202 F F F 19 380-399 123 123 123 204 204 204 F F F 20 400-419 122 122 122 201 201 201 F F F 21 420-439 120 120 120 197 197 197 F F F 22 440-459 119 119 119 193 193 193 F F F 23 460-479 115 115 115 186 186 186 F F F 24 480-499 119 119 119 193 193 193 F F F 25 500-519 124 124 124 206 206 206 F F F 26 520-539 122 122 122 203 203 203 T T T 27 540-559 124 124 124 205 205 205 F F F 28 560-579 124 124 124 207 207 207 F F F 29 580-599 121 121 121 198 198 198 F F F Max gen: 124, Max flush: 207 A Difference in extent metadata is expected here On loop 62 Verifying data now No change after verify Loop: 62 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 119 119 119 193 193 193 F F F 1 020-039 123 123 123 204 204 204 F F F 2 040-059 121 121 121 198 198 198 F F F 3 060-079 122 122 122 201 201 201 F F F 4 080-099 124 124 124 205 205 205 F F F 5 100-119 120 120 120 194 194 194 F F F 6 120-139 120 120 120 195 195 195 F F F 7 140-159 125 125 125 208 208 208 F F F 8 160-179 125 125 125 208 208 208 F F F 9 180-199 124 124 124 205 205 205 F F F 10 200-219 124 124 124 205 205 205 F F F 11 220-239 124 124 124 205 205 205 F F F 12 240-259 122 122 122 201 201 201 F F F 13 260-279 116 116 116 189 189 189 F F F 14 280-299 125 125 125 208 208 208 F F F 15 300-319 119 119 119 193 193 193 F F F 16 320-339 125 125 125 208 208 208 F F F 17 340-359 121 121 121 198 198 198 F F F 18 360-379 122 122 122 202 202 202 F F F 19 380-399 123 123 123 204 204 204 F F F 20 400-419 122 122 122 201 201 201 F F F 21 420-439 120 120 120 197 197 197 F F F 22 440-459 119 119 119 193 193 193 F F F 23 460-479 115 115 115 186 186 186 F F F 24 480-499 119 119 119 193 193 193 F F F 25 500-519 124 124 124 206 206 206 F F F 26 520-539 125 125 125 208 208 208 F F F 27 540-559 124 124 124 205 205 205 F F F 28 560-579 124 124 124 207 207 207 F F F 29 580-599 121 121 121 198 198 198 F F F Max gen: 125, Max flush: 208 Begin loop 63 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:57.665989478Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1684,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:57.709794362Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1684,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:57.710343083Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1684} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"3818f0fc76fe787c6cc412af30fd59b930203885\",\n git_commit_timestamp: \"2024-11-01T16:08:30.000000000Z\",\n git_branch: \"renovate/hyper-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:57.710384538Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1684} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:57.71040028Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1684} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:57.710415192Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1684} {"msg":"Crucible ac8df1c9-621f-49ba-b191-8bc2eb99b185 has session id: 437feb57-f193-47db-aca7-46c2e66bee57","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:57.710852118Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1684,"session_id":"437feb57-f193-47db-aca7-46c2e66bee57"} {"msg":"Upstairs opts: Upstairs UUID: ac8df1c9-621f-49ba-b191-8bc2eb99b185, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:57.710900985Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1684,"session_id":"437feb57-f193-47db-aca7-46c2e66bee57"} {"msg":"Crucible stats registered with UUID: ac8df1c9-621f-49ba-b191-8bc2eb99b185","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:57.710930658Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1684,"session_id":"437feb57-f193-47db-aca7-46c2e66bee57"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:57.74175682Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1684,"task":"crutest"} {"msg":"The guest has requested activation with gen:126","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:57.741801516Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1684} {"msg":"ac8df1c9-621f-49ba-b191-8bc2eb99b185 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:57.74183434Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1684,"session_id":"437feb57-f193-47db-aca7-46c2e66bee57"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:57.741865124Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1684,"client":"0","":"downstairs","session_id":"437feb57-f193-47db-aca7-46c2e66bee57"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:57.741913861Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1684,"client":"0","":"downstairs","session_id":"437feb57-f193-47db-aca7-46c2e66bee57"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:57.741953266Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1684,"client":"1","":"downstairs","session_id":"437feb57-f193-47db-aca7-46c2e66bee57"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:57.742031136Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1684,"client":"1","":"downstairs","session_id":"437feb57-f193-47db-aca7-46c2e66bee57"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:57.742071291Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1684,"client":"2","":"downstairs","session_id":"437feb57-f193-47db-aca7-46c2e66bee57"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:57.742100485Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1684,"client":"2","":"downstairs","session_id":"437feb57-f193-47db-aca7-46c2e66bee57"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:57.742127438Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1684,"":"io task","client":"2","":"downstairs","session_id":"437feb57-f193-47db-aca7-46c2e66bee57"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:57.742155842Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1684,"":"io task","client":"0","":"downstairs","session_id":"437feb57-f193-47db-aca7-46c2e66bee57"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:57.742183485Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1684,"":"io task","client":"1","":"downstairs","session_id":"437feb57-f193-47db-aca7-46c2e66bee57"} {"msg":"ds_connection connected from Ok(127.0.0.1:62904)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:57.742212279Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1684,"":"io task","client":"2","":"downstairs","session_id":"437feb57-f193-47db-aca7-46c2e66bee57"} {"msg":"ds_connection connected from Ok(127.0.0.1:46801)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:57.742281938Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1684,"":"io task","client":"1","":"downstairs","session_id":"437feb57-f193-47db-aca7-46c2e66bee57"} {"msg":"ds_connection connected from Ok(127.0.0.1:56457)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:57.742317202Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1684,"":"io task","client":"0","":"downstairs","session_id":"437feb57-f193-47db-aca7-46c2e66bee57"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:57.742890656Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1684,"client":"0","":"downstairs","session_id":"437feb57-f193-47db-aca7-46c2e66bee57"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:57.742930041Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1684,"client":"0","":"downstairs","session_id":"437feb57-f193-47db-aca7-46c2e66bee57"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:57.742960715Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1684,"client":"1","":"downstairs","session_id":"437feb57-f193-47db-aca7-46c2e66bee57"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:57.742990289Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1684,"client":"1","":"downstairs","session_id":"437feb57-f193-47db-aca7-46c2e66bee57"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:57.743020553Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1684,"client":"2","":"downstairs","session_id":"437feb57-f193-47db-aca7-46c2e66bee57"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:57.743046816Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1684,"client":"2","":"downstairs","session_id":"437feb57-f193-47db-aca7-46c2e66bee57"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:57.743573184Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1684,"client":"0","":"downstairs","session_id":"437feb57-f193-47db-aca7-46c2e66bee57"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:57.743651674Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1684,"client":"1","":"downstairs","session_id":"437feb57-f193-47db-aca7-46c2e66bee57"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:57.743795663Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1684,"client":"2","":"downstairs","session_id":"437feb57-f193-47db-aca7-46c2e66bee57"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:57.744231169Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1684,"client":"0","":"downstairs","session_id":"437feb57-f193-47db-aca7-46c2e66bee57"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:57.744260433Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1684,"client":"0","":"downstairs","session_id":"437feb57-f193-47db-aca7-46c2e66bee57"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:57.744276085Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1684,"session_id":"437feb57-f193-47db-aca7-46c2e66bee57"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:57.744289056Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1684,"client":"1","":"downstairs","session_id":"437feb57-f193-47db-aca7-46c2e66bee57"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:57.744303218Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1684,"client":"1","":"downstairs","session_id":"437feb57-f193-47db-aca7-46c2e66bee57"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:57.74431669Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1684,"session_id":"437feb57-f193-47db-aca7-46c2e66bee57"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:57.744329152Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1684,"client":"2","":"downstairs","session_id":"437feb57-f193-47db-aca7-46c2e66bee57"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:57.744342533Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1684,"client":"2","":"downstairs","session_id":"437feb57-f193-47db-aca7-46c2e66bee57"} {"msg":"[0]R flush_numbers[0..12]: [193, 204, 198, 201, 205, 194, 195, 208, 208, 205, 205, 205]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:57.744356645Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1684,"":"downstairs","":"downstairs","session_id":"437feb57-f193-47db-aca7-46c2e66bee57"} {"msg":"[0]R generation[0..12]: [119, 123, 121, 122, 124, 120, 120, 125, 125, 124, 124, 124]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:57.744370597Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1684,"":"downstairs","":"downstairs","session_id":"437feb57-f193-47db-aca7-46c2e66bee57"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:57.744384829Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1684,"":"downstairs","":"downstairs","session_id":"437feb57-f193-47db-aca7-46c2e66bee57"} {"msg":"[1]R flush_numbers[0..12]: [193, 204, 198, 201, 205, 194, 195, 208, 208, 205, 205, 205]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:57.744398801Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1684,"":"downstairs","":"downstairs","session_id":"437feb57-f193-47db-aca7-46c2e66bee57"} {"msg":"[1]R generation[0..12]: [119, 123, 121, 122, 124, 120, 120, 125, 125, 124, 124, 124]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:57.744412603Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1684,"":"downstairs","":"downstairs","session_id":"437feb57-f193-47db-aca7-46c2e66bee57"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:57.744429795Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1684,"":"downstairs","":"downstairs","session_id":"437feb57-f193-47db-aca7-46c2e66bee57"} {"msg":"[2]R flush_numbers[0..12]: [193, 204, 198, 201, 205, 194, 195, 208, 208, 205, 205, 205]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:57.744444087Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1684,"":"downstairs","":"downstairs","session_id":"437feb57-f193-47db-aca7-46c2e66bee57"} {"msg":"[2]R generation[0..12]: [119, 123, 121, 122, 124, 120, 120, 125, 125, 124, 124, 124]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:57.744457738Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1684,"":"downstairs","":"downstairs","session_id":"437feb57-f193-47db-aca7-46c2e66bee57"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:57.74447172Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1684,"":"downstairs","":"downstairs","session_id":"437feb57-f193-47db-aca7-46c2e66bee57"} {"msg":"Max found gen is 126","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:57.744485622Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1684,"":"downstairs","":"downstairs","session_id":"437feb57-f193-47db-aca7-46c2e66bee57"} {"msg":"Generation requested: 126 >= found:126","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:57.744499023Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1684,"":"downstairs","":"downstairs","session_id":"437feb57-f193-47db-aca7-46c2e66bee57"} {"msg":"Next flush: 209","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:57.744512155Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1684,"":"downstairs","":"downstairs","session_id":"437feb57-f193-47db-aca7-46c2e66bee57"} Wait for a query_work_queue command to finish before sending IO {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:57.744531638Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1684,"":"downstairs","":"downstairs","session_id":"437feb57-f193-47db-aca7-46c2e66bee57"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:57.74455281Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1684,"session_id":"437feb57-f193-47db-aca7-46c2e66bee57"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:57.744577314Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1684,"client":"0","":"downstairs","session_id":"437feb57-f193-47db-aca7-46c2e66bee57"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:57.744601657Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1684,"client":"1","":"downstairs","session_id":"437feb57-f193-47db-aca7-46c2e66bee57"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:57.74462651Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1684,"client":"2","":"downstairs","session_id":"437feb57-f193-47db-aca7-46c2e66bee57"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:57.744651263Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1684,"session_id":"437feb57-f193-47db-aca7-46c2e66bee57"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v"Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 :0,"name":"crucible","level":30,"time":"2024-11-01T16:54:57.744678027Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1684,"session_id":"437feb57-f193-47db-aca7-46c2e66bee57"} {"msg":"ac8df1c9-621f-49ba-b191-8bc2eb99b185 is now active with session: 437feb57-f193-47db-aca7-46c2e66bee57","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:57.744699869Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1684,"session_id":"437feb57-f193-47db-aca7-46c2e66bee57"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:57.744715691Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1684,"session_id":"437feb57-f193-47db-aca7-46c2e66bee57"} {"msg":"The guest has finished waiting for activation with:126","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:54:57.744728413Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1684} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 66 len 40960 data: 14 13 13 14 18 17 17 12 11 10 02/30 Write block 194 len 4096 data: 16 03/30 Read block 110 len 8192 04/30 Read block 508 len 16384 05/30 Read block 339 len 40960 06/30 Read block 29 len 40960 07/30 Write block 458 len 4096 data: 5 08/30 Write block 563 len 4096 data: 16 09/30 Read block 589 len 4096 10/30 Read block 480 len 4096 11/30 Read block 581 len 36864 12/30 Read block 477 len 36864 13/30 Read block 313 len 12288 14/30 Read block 52 len 32768 15/30 Read block 83 len 40960 16/30 Write block 402 len 16384 data: 10 9 8 6 17/30 Read block 523 len 8192 18/30 Write block 270 len 24576 data: 13 16 14 15 18 17 19/30 Write block 107 len 40960 data: 11 11 11 12 12 11 11 11 10 8 20/30 Write block 53 len 36864 data: 9 8 7 8 8 7 7 6 7 21/30 Write block 244 len 36864 data: 8 12 15 17 17 17 19 17 13 22/30 Read block 177 len 20480 23/30 Read block 317 len 12288 24/30 Read block 161 len 8192 25/30 Read block 486 len 12288 26/30 Read block 97 len 36864 27/30 Read block 531 len 12288 28/30 Write block 583 len 28672 data: 10 13 15 13 16 12 12 29/30 Write block 524 len 28672 data: 6 7 7 8 10 11 13 30/30 Read block 150 len 24576 ---------------------------------------------------------------- Crucible gen:126 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 10 Done Done Done false 1001 Acked Write 1 Done Done Done false 1002 Acked Read 2 Done Done Done false 1003 Acked Read 4 Done Done Done false 1004 Acked Read 10 Done Done Done false 1005 Acked Read 10 Done Done Done false 1006 Acked Write 1 Done Done Done false 1007 Acked Write 1 Done Done Done false 1008 Acked Read 1 Done Done Done false 1009 Acked Read 1 Done Done Done false 1010 Acked Read 9 Done Done Done false 1011 Acked Read 9 Done Done Done false 1012 Acked Read 3 Done Done Done false 1013 Acked Read 8 Done Done Done false 1014 Acked Read 10 Done Done Done false 1015 Acked Write 4 Done Done Done false 1016 Acked Read 2 Done Done Done false 1017 Acked Write 6 Done Done Done false 1018 Acked Write 10 Done Done Done false 1019 Acked Write 9 Done Done Done false 1020 Acked Write 9 Done Done Done false 1021 Acked Read 5 Done Done Done false 1022 Acked Read 3 Done Done Done false 1023 Acked Read 2 Done Done Done false 1024 Acked Read 3 Done Done Done false 1025 Acked Read 9 Done Done Done false 1026 Acked Read 3 Done Done Done false 1027 Acked Write 7 Done Done Done false 1028 Acked Write 7 Sent Done Done false 1029 Acked Read 6 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 0 1 3 Done 28 30 29 87 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 0 0 0 Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:54:57.764247208Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1684} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 119 119 119 193 193 193 F F F 1 020-039 123 123 123 204 204 204 F F F 2 040-059 121 121 121 198 198 198 T T T 3 060-079 122 122 122 201 201 201 T T T 4 080-099 124 124 124 205 205 205 F F F 5 100-119 120 120 120 194 194 194 T T T 6 120-139 120 120 120 195 195 195 F F F 7 140-159 125 125 125 208 208 208 F F F 8 160-179 125 125 125 208 208 208 F F F 9 180-199 124 124 124 205 205 205 T T T 10 200-219 124 124 124 205 205 205 F F F 11 220-239 124 124 124 205 205 205 F F F 12 240-259 122 122 122 201 201 201 T T T 13 260-279 116 116 116 189 189 189 T T T 14 280-299 125 125 125 208 208 208 F F F 15 300-319 119 119 119 193 193 193 F F F 16 320-339 125 125 125 208 208 208 F F F 17 340-359 121 121 121 198 198 198 F F F 18 360-379 122 122 122 202 202 202 F F F 19 380-399 123 123 123 204 204 204 F F F 20 400-419 122 122 122 201 201 201 T T T 21 420-439 120 120 120 197 197 197 F F F 22 440-459 119 119 119 193 193 193 T T T 23 460-479 115 115 115 186 186 186 F F F 24 480-499 119 119 119 193 193 193 F F F 25 500-519 124 124 124 206 206 206 F F F 26 520-539 125 125 125 208 208 208 T T T 27 540-559 124 124 124 205 205 205 F F F 28 560-579 124 124 124 207 207 207 T T T 29 580-599 121 121 121 198 198 198 T T T Max gen: 125, Max flush: 208 A Difference in extent metadata is expected here On loop 63 Verifying data now No change after verify Loop: 63 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 119 119 119 193 193 193 F F F 1 020-039 123 123 123 204 204 204 F F F 2 040-059 126 126 126 209 209 209 F F F 3 060-079 126 126 126 209 209 209 F F F 4 080-099 124 124 124 205 205 205 F F F 5 100-119 126 126 126 209 209 209 F F F 6 120-139 120 120 120 195 195 195 F F F 7 140-159 125 125 125 208 208 208 F F F 8 160-179 125 125 125 208 208 208 F F F 9 180-199 126 126 126 209 209 209 F F F 10 200-219 124 124 124 205 205 205 F F F 11 220-239 124 124 124 205 205 205 F F F 12 240-259 126 126 126 209 209 209 F F F 13 260-279 126 126 126 209 209 209 F F F 14 280-299 125 125 125 208 208 208 F F F 15 300-319 119 119 119 193 193 193 F F F 16 320-339 125 125 125 208 208 208 F F F 17 340-359 121 121 121 198 198 198 F F F 18 360-379 122 122 122 202 202 202 F F F 19 380-399 123 123 123 204 204 204 F F F 20 400-419 126 126 126 209 209 209 F F F 21 420-439 120 120 120 197 197 197 F F F 22 440-459 126 126 126 209 209 209 F F F 23 460-479 115 115 115 186 186 186 F F F 24 480-499 119 119 119 193 193 193 F F F 25 500-519 124 124 124 206 206 206 F F F 26 520-539 126 126 126 209 209 209 F F F 27 540-559 124 124 124 205 205 205 F F F 28 560-579 126 126 126 209 209 209 F F F 29 580-599 126 126 126 209 209 209 F F F Max gen: 126, Max flush: 209 Begin loop 64 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:02.158734587Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1692,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:55:02.200919153Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1692,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:02.201273168Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1692,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:02.232837395Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1692,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:02.233348571Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1692} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"3818f0fc76fe787c6cc412af30fd59b930203885\",\n git_commit_timestamp: \"2024-11-01T16:08:30.000000000Z\",\n git_branch: \"renovate/hyper-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:02.233380695Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1692} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:02.233395147Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1692} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:02.233409259Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1692} {"msg":"Crucible 5902ceb8-012b-49a0-bdfc-98feba5dbba1 has session id: 73be0d74-a0ba-4f61-bbfe-69fd821bf738","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:02.233840264Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1692,"session_id":"73be0d74-a0ba-4f61-bbfe-69fd821bf738"} {"msg":"Upstairs opts: Upstairs UUID: 5902ceb8-012b-49a0-bdfc-98feba5dbba1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:02.233877549Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1692,"session_id":"73be0d74-a0ba-4f61-bbfe-69fd821bf738"} {"msg":"Crucible stats registered with UUID: 5902ceb8-012b-49a0-bdfc-98feba5dbba1","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:02.233901432Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1692,"session_id":"73be0d74-a0ba-4f61-bbfe-69fd821bf738"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:02.264269055Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1692,"task":"crutest"} {"msg":"The guest has requested activation with gen:128","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:02.26430427Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1692} {"msg":"5902ceb8-012b-49a0-bdfc-98feba5dbba1 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:02.264338334Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1692,"session_id":"73be0d74-a0ba-4f61-bbfe-69fd821bf738"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:02.264360537Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1692,"client":"0","":"downstairs","session_id":"73be0d74-a0ba-4f61-bbfe-69fd821bf738"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:02.264389361Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1692,"client":"0","":"downstairs","session_id":"73be0d74-a0ba-4f61-bbfe-69fd821bf738"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:02.264408743Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1692,"client":"1","":"downstairs","session_id":"73be0d74-a0ba-4f61-bbfe-69fd821bf738"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:02.264426125Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1692,"client":"1","":"downstairs","session_id":"73be0d74-a0ba-4f61-bbfe-69fd821bf738"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:02.264442918Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1692,"client":"2","":"downstairs","session_id":"73be0d74-a0ba-4f61-bbfe-69fd821bf738"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:02.264468581Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1692,"client":"2","":"downstairs","session_id":"73be0d74-a0ba-4f61-bbfe-69fd821bf738"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:02.264511486Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1692,"":"io task","client":"2","":"downstairs","session_id":"73be0d74-a0ba-4f61-bbfe-69fd821bf738"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:02.264546131Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1692,"":"io task","client":"0","":"downstairs","session_id":"73be0d74-a0ba-4f61-bbfe-69fd821bf738"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:02.264578185Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1692,"":"io task","client":"1","":"downstairs","session_id":"73be0d74-a0ba-4f61-bbfe-69fd821bf738"} {"msg":"ds_connection connected from Ok(127.0.0.1:40232)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:02.264598107Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1692,"":"io task","client":"2","":"downstairs","session_id":"73be0d74-a0ba-4f61-bbfe-69fd821bf738"} {"msg":"ds_connection connected from Ok(127.0.0.1:36407)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:02.26461595Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1692,"":"io task","client":"0","":"downstairs","session_id":"73be0d74-a0ba-4f61-bbfe-69fd821bf738"} {"msg":"ds_connection connected from Ok(127.0.0.1:55710)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:02.264793193Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1692,"":"io task","client":"1","":"downstairs","session_id":"73be0d74-a0ba-4f61-bbfe-69fd821bf738"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:02.265371267Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1692,"client":"1","":"downstairs","session_id":"73be0d74-a0ba-4f61-bbfe-69fd821bf738"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:02.265399531Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1692,"client":"1","":"downstairs","session_id":"73be0d74-a0ba-4f61-bbfe-69fd821bf738"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:02.265415253Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1692,"client":"0","":"downstairs","session_id":"73be0d74-a0ba-4f61-bbfe-69fd821bf738"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:02.26546671Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1692,"client":"0","":"downstairs","session_id":"73be0d74-a0ba-4f61-bbfe-69fd821bf738"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:02.265507855Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1692,"client":"2","":"downstairs","session_id":"73be0d74-a0ba-4f61-bbfe-69fd821bf738"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:02.265524367Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1692,"client":"2","":"downstairs","session_id":"73be0d74-a0ba-4f61-bbfe-69fd821bf738"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:02.265990267Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1692,"client":"1","":"downstairs","session_id":"73be0d74-a0ba-4f61-bbfe-69fd821bf738"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:02.266149007Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1692,"client":"0","":"downstairs","session_id":"73be0d74-a0ba-4f61-bbfe-69fd821bf738"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:02.266185182Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1692,"client":"2","":"downstairs","session_id":"73be0d74-a0ba-4f61-bbfe-69fd821bf738"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:02.266601876Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1692,"client":"1","":"downstairs","session_id":"73be0d74-a0ba-4f61-bbfe-69fd821bf738"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:02.26663588Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1692,"client":"1","":"downstairs","session_id":"73be0d74-a0ba-4f61-bbfe-69fd821bf738"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:02.266653012Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1692,"session_id":"73be0d74-a0ba-4f61-bbfe-69fd821bf738"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:02.266737093Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1692,"client":"0","":"downstairs","session_id":"73be0d74-a0ba-4f61-bbfe-69fd821bf738"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:02.266757326Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1692,"client":"0","":"downstairs","session_id":"73be0d74-a0ba-4f61-bbfe-69fd821bf738"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:02.266771178Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1692,"session_id":"73be0d74-a0ba-4f61-bbfe-69fd821bf738"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:02.266783689Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1692,"client":"2","":"downstairs","session_id":"73be0d74-a0ba-4f61-bbfe-69fd821bf738"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:02.266797201Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1692,"client":"2","":"downstairs","session_id":"73be0d74-a0ba-4f61-bbfe-69fd821bf738"} {"msg":"[0]R flush_numbers[0..12]: [193, 204, 209, 209, 205, 209, 195, 208, 208, 209, 205, 205]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:02.266811363Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1692,"":"downstairs","":"downstairs","session_id":"73be0d74-a0ba-4f61-bbfe-69fd821bf738"} {"msg":"[0]R generation[0..12]: [119, 123, 126, 126, 124, 126, 120, 125, 125, 126, 124, 124]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:02.266825525Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1692,"":"downstairs","":"downstairs","session_id":"73be0d74-a0ba-4f61-bbfe-69fd821bf738"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:02.266842887Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1692,"":"downstairs","":"downstairs","session_id":"73be0d74-a0ba-4f61-bbfe-69fd821bf738"} {"msg":"[1]R flush_numbers[0..12]: [193, 204, 209, 209, 205, 209, 195, 208, 208, 209, 205, 205]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:02.266857189Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1692,"":"downstairs","":"downstairs","session_id":"73be0d74-a0ba-4f61-bbfe-69fd821bf738"} {"msg":"[1]R generation[0..12]: [119, 123, 126, 126, 124, 126, 120, 125, 125, 126, 124, 124]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:02.26687101Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1692,"":"downstairs","":"downstairs","session_id":"73be0d74-a0ba-4f61-bbfe-69fd821bf738"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:02.266885202Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1692,"":"downstairs","":"downstairs","session_id":"73be0d74-a0ba-4f61-bbfe-69fd821bf738"} {"msg":"[2]R flush_numbers[0..12]: [193, 204, 209, 209, 205, 209, 195, 208, 208, 209, 205, 205]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:02.266898894Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1692,"":"downstairs","":"downstairs","session_id":"73be0d74-a0ba-4f61-bbfe-69fd821bf738"} {"msg":"[2]R generation[0..12]: [119, 123, 126, 126, 124, 126, 120, 125, 125, 126, 124, 124]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:02.266912656Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1692,"":"downstairs","":"downstairs","session_id":"73be0d74-a0ba-4f61-bbfe-69fd821bf738"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:02.266926647Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1692,"":"downstairs","":"downstairs","session_id":"73be0d74-a0ba-4f61-bbfe-69fd821bf738"} {"msg":"Max found gen is 127","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:02.266939649Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1692,"":"downstairs","":"downstairs","session_id":"73be0d74-a0ba-4f61-bbfe-69fd821bf738"} {"msg":"Generation requested: 128 >= found:127","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-01T16:55:02.266952851Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1692,"":"downstairs","":"downstairs","session_id":"73be0d74-a0ba-4f61-bbfe-69fd821bf738"} {"msg":"Next flush: 210","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:02.266971303Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1692,"":"downstairs","":"downstairs","session_id":"73be0d74-a0ba-4f61-bbfe-69fd821bf738"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:02.266984585Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1692,"":"downstairs","":"downstairs","session_id":"73be0d74-a0ba-4f61-bbfe-69fd821bf738"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:02.266997727Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1692,"session_id":"73be0d74-a0ba-4f61-bbfe-69fd821bf738"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:02.267009778Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1692,"client":"0","":"downstairs","session_id":"73be0d74-a0ba-4f61-bbfe-69fd821bf738"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:02.26702322Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1692,"client":"1","":"downstairs","session_id":"73be0d74-a0ba-4f61-bbfe-69fd821bf738"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:02.267049983Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1692,"client":"2","":"downstairs","session_id":"73be0d74-a0ba-4f61-bbfe-69fd821bf738"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:02.267070906Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1692,"session_id":"73be0d74-a0ba-4f61-bbfe-69fd821bf738"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:02.267084008Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1692,"session_id":"73be0d74-a0ba-4f61-bbfe-69fd821bf738"} {"msg":"5902ceb8-012b-49a0-bdfc-98feba5dbba1 is now active with session: 73be0d74-a0ba-4f61-bbfe-69fd821bf738","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:02.267096889Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1692,"session_id":"73be0d74-a0ba-4f61-bbfe-69fd821bf738"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:02.267115052Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1692,"session_id":"73be0d74-a0ba-4f61-bbfe-69fd821bf738"} {"msg":"The guest has finished waiting for activation with:128","v":0,"name":"crucible","level":30,"Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 time":"2024-11-01T16:55:02.267137355Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1692} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Flush 03/30 Write block 566 len 28672 data: 7 8 6 8 6 8 8 04/30 Read block 397 len 40960 05/30 Write block 261 len 24576 data: 13 13 11 11 11 12 06/30 Read block 119 len 24576 07/30 Write block 434 len 20480 data: 5 5 6 5 8 08/30 Read block 205 len 20480 09/30 Flush 10/30 Write block 37 len 4096 data: 12 11/30 Read block 485 len 32768 12/30 Write block 475 len 24576 data: 10 7 11 14 13 13 13/30 Write block 505 len 24576 data: 7 9 9 11 10 9 14/30 Flush 15/30 Write block 243 len 40960 data: 8 9 13 16 18 18 18 20 18 14 16/30 Write block 80 len 32768 data: 9 9 11 11 10 12 12 14 17/30 Write block 187 len 8192 data: 10 10 18/30 Read block 71 len 40960 19/30 Read block 248 len 8192 20/30 Read block 67 len 4096 21/30 Read block 255 len 12288 22/30 Read block 523 len 28672 23/30 Read block 513 len 32768 24/30 Read block 403 len 4096 25/30 Read block 171 len 20480 26/30 Write block 138 len 28672 data: 12 9 8 8 8 5 6 27/30 Write block 213 len 36864 data: 11 11 11 14 15 15 13 13 13 28/30 Read block 57 len 40960 29/30 Read block 244 len 32768 30/30 Read block 363 len 28672 ---------------------------------------------------------------- Crucible gen:128 GIO:true work queues: Upstairs:0 downstairs:16 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1014 Acked Write 10 Done Done Done false 1015 Acked Write 8 Done Done Done false 1016 Acked Write 2 Done Done Done false 1017 Acked Read 10 Done Done Done false 1018 Acked Read 2 Done Done Done false 1019 Acked Read 1 Done Done Done false 1020 Acked Read 3 Done Done Done false 1021 Acked Read 7 Done Done Done false 1022 Acked Read 8 Done Done Done false 1023 Acked Read 1 Done Done Done false 1024 Acked Read 5 Done Done Done false 1025 Acked Write 7 Done Done Done false 1026 Acked Write 9 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Read 7 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 16 16 16 48 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1013 1013 1013 Downstairs last five completed: 1013 1012 1011 1010 1009 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:55:02.290937921Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1692} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 119 119 119 193 193 193 F F F 1 020-039 128 128 128 213 213 213 F F F 2 040-059 126 126 126 209 209 209 F F F 3 060-079 126 126 126 209 209 209 F F F 4 080-099 124 124 124 205 205 205 T T T 5 100-119 126 126 126 209 209 209 F F F 6 120-139 120 120 120 195 195 195 T T T 7 140-159 125 125 125 208 208 208 T T T 8 160-179 125 125 125 208 208 208 F F F 9 180-199 126 126 126 209 209 209 T T T 10 200-219 124 124 124 205 205 205 T T T 11 220-239 124 124 124 205 205 205 T T T 12 240-259 126 126 126 209 209 209 T T T 13 260-279 128 128 128 212 212 212 F F F 14 280-299 125 125 125 208 208 208 F F F 15 300-319 119 119 119 193 193 193 F F F 16 320-339 125 125 125 208 208 208 F F F 17 340-359 121 121 121 198 198 198 F F F 18 360-379 122 122 122 202 202 202 F F F 19 380-399 123 123 123 204 204 204 F F F 20 400-419 126 126 126 209 209 209 F F F 21 420-439 128 128 128 212 212 212 F F F 22 440-459 126 126 126 209 209 209 F F F 23 460-479 128 128 128 213 213 213 F F F 24 480-499 128 128 128 213 213 213 F F F 25 500-519 128 128 128 213 213 213 F F F 26 520-539 126 126 126 209 209 209 F F F 27 540-559 124 124 124 205 205 205 F F F 28 560-579 128 128 128 212 212 212 F F F 29 580-599 126 126 126 209 209 209 F F F Max gen: 128, Max flush: 213 A Difference in extent metadata is expected here On loop 64 Verifying data now No change after verify Loop: 64 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 119 119 119 193 193 193 F F F 1 020-039 128 128 128 213 213 213 F F F 2 040-059 126 126 126 209 209 209 F F F 3 060-079 126 126 126 209 209 209 F F F 4 080-099 129 129 129 214 214 214 F F F 5 100-119 126 126 126 209 209 209 F F F 6 120-139 129 129 129 214 214 214 F F F 7 140-159 129 129 129 214 214 214 F F F 8 160-179 125 125 125 208 208 208 F F F 9 180-199 129 129 129 214 214 214 F F F 10 200-219 129 129 129 214 214 214 F F F 11 220-239 129 129 129 214 214 214 F F F 12 240-259 129 129 129 214 214 214 F F F 13 260-279 128 128 128 212 212 212 F F F 14 280-299 125 125 125 208 208 208 F F F 15 300-319 119 119 119 193 193 193 F F F 16 320-339 125 125 125 208 208 208 F F F 17 340-359 121 121 121 198 198 198 F F F 18 360-379 122 122 122 202 202 202 F F F 19 380-399 123 123 123 204 204 204 F F F 20 400-419 126 126 126 209 209 209 F F F 21 420-439 128 128 128 212 212 212 F F F 22 440-459 126 126 126 209 209 209 F F F 23 460-479 128 128 128 213 213 213 F F F 24 480-499 128 128 128 213 213 213 F F F 25 500-519 128 128 128 213 213 213 F F F 26 520-539 126 126 126 209 209 209 F F F 27 540-559 124 124 124 205 205 205 F F F 28 560-579 128 128 128 212 212 212 F F F 29 580-599 126 126 126 209 209 209 F F F Max gen: 129, Max flush: 214 Begin loop 65 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:06.651103337Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1700,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:06.697051187Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1700,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:06.697652074Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1700} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"3818f0fc76fe787c6cc412af30fd59b930203885\",\n git_commit_timestamp: \"2024-11-01T16:08:30.000000000Z\",\n git_branch: \"renovate/hyper-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:06.697687589Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1700} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:06.697702551Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1700} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:06.697719123Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1700} {"msg":"Crucible 549d2b86-7ff3-4682-99a6-ce88eb7f8c48 has session id: 4f44abfe-ed02-488f-a8c6-3276b7d6444d","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:06.698175422Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1700,"session_id":"4f44abfe-ed02-488f-a8c6-3276b7d6444d"} {"msg":"Upstairs opts: Upstairs UUID: 549d2b86-7ff3-4682-99a6-ce88eb7f8c48, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:06.698224208Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1700,"session_id":"4f44abfe-ed02-488f-a8c6-3276b7d6444d"} {"msg":"Crucible stats registered with UUID: 549d2b86-7ff3-4682-99a6-ce88eb7f8c48","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:06.698251902Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1700,"session_id":"4f44abfe-ed02-488f-a8c6-3276b7d6444d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:06.728789246Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1700,"task":"crutest"} {"msg":"The guest has requested activation with gen:130","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:06.728823221Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1700} {"msg":"549d2b86-7ff3-4682-99a6-ce88eb7f8c48 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:06.728888589Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1700,"session_id":"4f44abfe-ed02-488f-a8c6-3276b7d6444d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:06.728916423Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1700,"client":"0","":"downstairs","session_id":"4f44abfe-ed02-488f-a8c6-3276b7d6444d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:06.728941546Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1700,"client":"0","":"downstairs","session_id":"4f44abfe-ed02-488f-a8c6-3276b7d6444d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:06.72897304Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1700,"client":"1","":"downstairs","session_id":"4f44abfe-ed02-488f-a8c6-3276b7d6444d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:06.729089725Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1700,"client":"1","":"downstairs","session_id":"4f44abfe-ed02-488f-a8c6-3276b7d6444d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:06.729131221Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1700,"client":"2","":"downstairs","session_id":"4f44abfe-ed02-488f-a8c6-3276b7d6444d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:06.729163815Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1700,"client":"2","":"downstairs","session_id":"4f44abfe-ed02-488f-a8c6-3276b7d6444d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:06.729319595Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1700,"":"io task","client":"2","":"downstairs","session_id":"4f44abfe-ed02-488f-a8c6-3276b7d6444d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:06.72936308Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1700,"":"io task","client":"0","":"downstairs","session_id":"4f44abfe-ed02-488f-a8c6-3276b7d6444d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:06.729396065Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1700,"":"io task","client":"1","":"downstairs","session_id":"4f44abfe-ed02-488f-a8c6-3276b7d6444d"} {"msg":"ds_connection connected from Ok(127.0.0.1:42784)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:06.729429429Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1700,"":"io task","client":"2","":"downstairs","session_id":"4f44abfe-ed02-488f-a8c6-3276b7d6444d"} {"msg":"ds_connection connected from Ok(127.0.0.1:44462)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:06.729459223Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1700,"":"io task","client":"1","":"downstairs","session_id":"4f44abfe-ed02-488f-a8c6-3276b7d6444d"} {"msg":"ds_connection connected from Ok(127.0.0.1:32946)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:06.729507879Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1700,"":"io task","client":"0","":"downstairs","session_id":"4f44abfe-ed02-488f-a8c6-3276b7d6444d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:06.729887278Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1700,"client":"0","":"downstairs","session_id":"4f44abfe-ed02-488f-a8c6-3276b7d6444d"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:06.729913411Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1700,"client":"0","":"downstairs","session_id":"4f44abfe-ed02-488f-a8c6-3276b7d6444d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:06.73014066Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1700,"client":"1","":"downstairs","session_id":"4f44abfe-ed02-488f-a8c6-3276b7d6444d"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:06.730172365Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1700,"client":"1","":"downstairs","session_id":"4f44abfe-ed02-488f-a8c6-3276b7d6444d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:06.730192867Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1700,"client":"2","":"downstairs","session_id":"4f44abfe-ed02-488f-a8c6-3276b7d6444d"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:06.73021664Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1700,"client":"2","":"downstairs","session_id":"4f44abfe-ed02-488f-a8c6-3276b7d6444d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:06.730518939Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1700,"client":"0","":"downstairs","session_id":"4f44abfe-ed02-488f-a8c6-3276b7d6444d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:06.730790384Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1700,"client":"1","":"downstairs","session_id":"4f44abfe-ed02-488f-a8c6-3276b7d6444d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:06.730968447Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1700,"client":"2","":"downstairs","session_id":"4f44abfe-ed02-488f-a8c6-3276b7d6444d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:06.731179024Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1700,"client":"0","":"downstairs","session_id":"4f44abfe-ed02-488f-a8c6-3276b7d6444d"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:06.731208198Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1700,"client":"0","":"downstairs","session_id":"4f44abfe-ed02-488f-a8c6-3276b7d6444d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:06.73122356Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1700,"session_id":"4f44abfe-ed02-488f-a8c6-3276b7d6444d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:06.731306321Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1700,"client":"1","":"downstairs","session_id":"4f44abfe-ed02-488f-a8c6-3276b7d6444d"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:06.731325483Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1700,"client":"1","":"downstairs","session_id":"4f44abfe-ed02-488f-a8c6-3276b7d6444d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:06.731339205Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1700,"session_id":"4f44abfe-ed02-488f-a8c6-3276b7d6444d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:06.731564954Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1700,"client":"2","":"downstairs","session_id":"4f44abfe-ed02-488f-a8c6-3276b7d6444d"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:06.731583686Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1700,"client":"2","":"downstairs","session_id":"4f44abfe-ed02-488f-a8c6-3276b7d6444d"} {"msg":"[0]R flush_numbers[0..12]: [193, 213, 209, 209, 214, 209, 214, 214, 208, 214, 214, 214]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:06.731597818Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1700,"":"downstairs","":"downstairs","session_id":"4f44abfe-ed02-488f-a8c6-3276b7d6444d"} {"msg":"[0]R generation[0..12]: [119, 128, 126, 126, 129, 126, 129, 129, 125, 129, 129, 129]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:06.7316119Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1700,"":"downstairs","":"downstairs","session_id":"4f44abfe-ed02-488f-a8c6-3276b7d6444d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:06.731626152Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1700,"":"downstairs","":"downstairs","session_id":"4f44abfe-ed02-488f-a8c6-3276b7d6444d"} {"msg":"[1]R flush_numbers[0..12]: [193, 213, 209, 209, 214, 209, 214, 214, 208, 214, 214, 214]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:06.731644244Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1700,"":"downstairs","":"downstairs","session_id":"4f44abfe-ed02-488f-a8c6-3276b7d6444d"} {"msg":"[1]R generation[0..12]: [119, 128, 126, 126, 129, 126, 129, 129, 125, 129, 129, 129]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:06.731658316Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1700,"":"downstairs","":"downstairs","session_id":"4f44abfe-ed02-488f-a8c6-3276b7d6444d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:06.731672448Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1700,"":"downstairs","":"downstairs","session_id":"4f44abfe-ed02-488f-a8c6-3276b7d6444d"} {"msg":"[2]R flush_numbers[0..12]: [193, 213, 209, 209, 214, 209, 214, 214, 208, 214, 214, 214]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:06.73168632Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1700,"":"downstairs","":"downstairs","session_id":"4f44abfe-ed02-488f-a8c6-3276b7d6444d"} {"msg":"[2]R generation[0..12]: [119, 128, 126, 126, 129, 126, 129, 129, 125, 129, 129, 129]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:06.731700592Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1700,"":"downstairs","":"downstairs","session_id":"4f44abfe-ed02-488f-a8c6-3276b7d6444d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:06.731714793Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1700,"":"downstairs","":"downstairs","session_id":"4f44abfe-ed02-488f-a8c6-3276b7d6444d"} {"msg":"Max found gen is 130","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:06.731727995Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1700,"":"downstairs","":"downstairs","session_id":"4f44abfe-ed02-488f-a8c6-3276b7d6444d"} {"msg":"Generation requested: 130 >= found:130","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:06.731741247Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1700,"":"downstairs","":"downstairs","session_id":"4f44abfe-ed02-488f-a8c6-3276b7d6444d"} {"msg":"Next flush: 215","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:06.731754288Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1700,"":"downstairs","":"downstairs","session_id":"4f44abfe-ed02-488f-a8c6-3276b7d6444d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:06.73176741Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1700,"":"downstairs","":"downstairs","session_id":"4f44abfe-ed02-488f-a8c6-3276b7d6444d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:06.731780682Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1700,"session_id":"4f44abfe-ed02-488f-a8c6-3276b7d6444d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:06.731792753Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1700,"client":"0","":"downstairs","session_id":"4f44abfe-ed02-488f-a8c6-3276b7d6444d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-01T16:55:06.731806285Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1700,"client":"1","":"downstairs","session_id":"4f44abfe-ed02-488f-a8c6-3276b7d6444d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:06.731829558Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1700,"client":"2","":"downstairs","session_id":"4f44abfe-ed02-488f-a8c6-3276b7d6444d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:06.73184297Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1700,"session_id":"4f44abfe-ed02-488f-a8c6-3276b7d6444d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:06.731857662Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1700,"session_id":"4f44abfe-ed02-488f-a8c6-3276b7d6444d"} {"msg":"549d2b86-7ff3-4682-99a6-ce88eb7f8c48 is now active with session: 4f44abfe-ed02-488f-a8c6-3276b7d6444d","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:06.731870654Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1700,"session_id":"4f44abfe-ed02-488f-a8c6-3276b7d6444d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:06.731882865Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1700,"session_id":"4f44abfe-ed02-488f-a8c6-3276b7d6444d"} {"msg":"The guest has finished waiting for activation with:130","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:06.731895187Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1700} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 556 len 4096 data: 12 02/30 Read block 152 len 12288 03/30 Write block 419 len 4096 data: 8 04/30 Read block 245 len 20480 05/30 Read block 516 len 8192 06/30 Read block 586 len 40960 07/30 Read block 304 len 40960 08/30 Read block 155 len 4096 09/30 Write block 459 len 32768 data: 5 3 5 6 5 5 5 8 10/30 Write block 82 len 4096 data: 12 11/30 Read block 455 len 32768 12/30 Read block 482 len 20480 13/30 Read block 283 len 16384 14/30 Write block 135 len 8192 data: 9 7 15/30 Write block 425 len 12288 data: 11 13 13 16/30 Read block 102 len 8192 17/30 Write block 519 len 12288 data: 7 7 6 18/30 Write block 415 len 40960 data: 6 5 7 9 9 9 10 11 11 11 19/30 Write block 510 len 32768 data: 10 9 9 6 5 4 4 5 20/30 Read block 553 len 24576 21/30 Write block 517 len 12288 data: 6 6 8 22/30 Read block 456 len 32768 23/30 Read block 37 len 28672 24/30 Read block 126 len 36864 25/30 Flush 26/30 Write block 199 len 24576 data: 12 10 10 10 11 11 27/30 Write block 487 len 40960 data: 16 18 16 15 14 11 7 6 9 11 28/30 Read block 94 len 20480 29/30 Write block 300 len 16384 data: 6 5 6 6 30/30 Write block 343 len 4096 data: 7 ---------------------------------------------------------------- Crucible gen:130 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 6 Done Done Done false 1026 Acked Write 10 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Write 4 Sent Sent Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:55:06.752636069Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1700} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 119 119 119 193 193 193 F F F 1 020-039 128 128 128 213 213 213 F F F 2 040-059 126 126 126 209 209 209 F F F 3 060-079 126 126 126 209 209 209 F F F 4 080-099 130 130 130 215 215 215 F F F 5 100-119 126 126 126 209 209 209 F F F 6 120-139 130 130 130 215 215 215 F F F 7 140-159 129 129 129 214 214 214 F F F 8 160-179 125 125 125 208 208 208 F F F 9 180-199 129 129 129 214 214 214 T T T 10 200-219 129 129 129 214 214 214 T T T 11 220-239 129 129 129 214 214 214 F F F 12 240-259 129 129 129 214 214 214 F F F 13 260-279 128 128 128 212 212 212 F F F 14 280-299 125 125 125 208 208 208 F F F 15 300-319 119 119 119 193 193 193 T T T 16 320-339 125 125 125 208 208 208 F F F 17 340-359 121 121 121 198 198 198 T T T 18 360-379 122 122 122 202 202 202 F F F 19 380-399 123 123 123 204 204 204 F F F 20 400-419 130 130 130 215 215 215 F F F 21 420-439 130 130 130 215 215 215 F F F 22 440-459 130 130 130 215 215 215 F F F 23 460-479 130 130 130 215 215 215 F F F 24 480-499 128 128 128 213 213 213 T T T 25 500-519 130 130 130 215 215 215 F F F 26 520-539 130 130 130 215 215 215 F F F 27 540-559 130 130 130 215 215 215 F F F 28 560-579 128 128 128 212 212 212 F F F 29 580-599 126 126 126 209 209 209 F F F Max gen: 130, Max flush: 215 A Difference in extent metadata is expected here On loop 65 Verifying data now No change after verify Loop: 65 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 119 119 119 193 193 193 F F F 1 020-039 128 128 128 213 213 213 F F F 2 040-059 126 126 126 209 209 209 F F F 3 060-079 126 126 126 209 209 209 F F F 4 080-099 130 130 130 215 215 215 F F F 5 100-119 126 126 126 209 209 209 F F F 6 120-139 130 130 130 215 215 215 F F F 7 140-159 129 129 129 214 214 214 F F F 8 160-179 125 125 125 208 208 208 F F F 9 180-199 131 131 131 216 216 216 F F F 10 200-219 131 131 131 216 216 216 F F F 11 220-239 129 129 129 214 214 214 F F F 12 240-259 129 129 129 214 214 214 F F F 13 260-279 128 128 128 212 212 212 F F F 14 280-299 125 125 125 208 208 208 F F F 15 300-319 131 131 131 216 216 216 F F F 16 320-339 125 125 125 208 208 208 F F F 17 340-359 131 131 131 216 216 216 F F F 18 360-379 122 122 122 202 202 202 F F F 19 380-399 123 123 123 204 204 204 F F F 20 400-419 130 130 130 215 215 215 F F F 21 420-439 130 130 130 215 215 215 F F F 22 440-459 130 130 130 215 215 215 F F F 23 460-479 130 130 130 215 215 215 F F F 24 480-499 131 131 131 216 216 216 F F F 25 500-519 130 130 130 215 215 215 F F F 26 520-539 130 130 130 215 215 215 F F F 27 540-559 130 130 130 215 215 215 F F F 28 560-579 128 128 128 212 212 212 F F F 29 580-599 126 126 126 209 209 209 F F F Max gen: 131, Max flush: 216 Begin loop 66 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:11.057469994Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1708,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:11.100989782Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1708,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:11.10640852Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1708} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"3818f0fc76fe787c6cc412af30fd59b930203885\",\n git_commit_timestamp: \"2024-11-01T16:08:30.000000000Z\",\n git_branch: \"renovate/hyper-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:11.106459536Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1708} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:11.106481689Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1708} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:11.106500172Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1708} {"msg":"Crucible d87d61a0-664d-4f8c-bbf3-65d105277ad3 has session id: 10a59e1c-6a65-4851-9de7-081e60e54d51","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:11.106867309Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1708,"session_id":"10a59e1c-6a65-4851-9de7-081e60e54d51"} {"msg":"Upstairs opts: Upstairs UUID: d87d61a0-664d-4f8c-bbf3-65d105277ad3, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:11.106915425Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1708,"session_id":"10a59e1c-6a65-4851-9de7-081e60e54d51"} {"msg":"Crucible stats registered with UUID: d87d61a0-664d-4f8c-bbf3-65d105277ad3","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:11.106944369Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1708,"session_id":"10a59e1c-6a65-4851-9de7-081e60e54d51"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:11.137746928Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1708,"task":"crutest"} {"msg":"The guest has requested activation with gen:132","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:11.137778442Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1708} {"msg":"d87d61a0-664d-4f8c-bbf3-65d105277ad3 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:11.137851031Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1708,"session_id":"10a59e1c-6a65-4851-9de7-081e60e54d51"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:11.137874414Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1708,"client":"0","":"downstairs","session_id":"10a59e1c-6a65-4851-9de7-081e60e54d51"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:11.137902348Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1708,"client":"0","":"downstairs","session_id":"10a59e1c-6a65-4851-9de7-081e60e54d51"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:11.137927731Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1708,"client":"1","":"downstairs","session_id":"10a59e1c-6a65-4851-9de7-081e60e54d51"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:11.137959465Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1708,"client":"1","":"downstairs","session_id":"10a59e1c-6a65-4851-9de7-081e60e54d51"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:11.137991299Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1708,"client":"2","":"downstairs","session_id":"10a59e1c-6a65-4851-9de7-081e60e54d51"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:11.138024234Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1708,"client":"2","":"downstairs","session_id":"10a59e1c-6a65-4851-9de7-081e60e54d51"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:11.138061958Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1708,"":"io task","client":"2","":"downstairs","session_id":"10a59e1c-6a65-4851-9de7-081e60e54d51"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:11.138100223Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1708,"":"io task","client":"0","":"downstairs","session_id":"10a59e1c-6a65-4851-9de7-081e60e54d51"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:11.138136258Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1708,"":"io task","client":"1","":"downstairs","session_id":"10a59e1c-6a65-4851-9de7-081e60e54d51"} {"msg":"ds_connection connected from Ok(127.0.0.1:60050)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:11.138168522Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1708,"":"io task","client":"2","":"downstairs","session_id":"10a59e1c-6a65-4851-9de7-081e60e54d51"} {"msg":"ds_connection connected from Ok(127.0.0.1:63498)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:11.13830493Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1708,"":"io task","client":"0","":"downstairs","session_id":"10a59e1c-6a65-4851-9de7-081e60e54d51"} {"msg":"ds_connection connected from Ok(127.0.0.1:37751)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:11.138339564Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1708,"":"io task","client":"1","":"downstairs","session_id":"10a59e1c-6a65-4851-9de7-081e60e54d51"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:11.138857301Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1708,"client":"0","":"downstairs","session_id":"10a59e1c-6a65-4851-9de7-081e60e54d51"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:11.138885494Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1708,"client":"0","":"downstairs","session_id":"10a59e1c-6a65-4851-9de7-081e60e54d51"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:11.138900566Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1708,"client":"1","":"downstairs","session_id":"10a59e1c-6a65-4851-9de7-081e60e54d51"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:11.138914388Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1708,"client":"1","":"downstairs","session_id":"10a59e1c-6a65-4851-9de7-081e60e54d51"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:11.13892781Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1708,"client":"2","":"downstairs","session_id":"10a59e1c-6a65-4851-9de7-081e60e54d51"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:11.138941102Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1708,"client":"2","":"downstairs","session_id":"10a59e1c-6a65-4851-9de7-081e60e54d51"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:11.139517086Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1708,"client":"0","":"downstairs","session_id":"10a59e1c-6a65-4851-9de7-081e60e54d51"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:11.13954838Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1708,"client":"2","":"downstairs","session_id":"10a59e1c-6a65-4851-9de7-081e60e54d51"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:11.139616148Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1708,"client":"1","":"downstairs","session_id":"10a59e1c-6a65-4851-9de7-081e60e54d51"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:11.140056315Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1708,"client":"0","":"downstairs","session_id":"10a59e1c-6a65-4851-9de7-081e60e54d51"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:11.140082719Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1708,"client":"0","":"downstairs","session_id":"10a59e1c-6a65-4851-9de7-081e60e54d51"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:11.14009717Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1708,"session_id":"10a59e1c-6a65-4851-9de7-081e60e54d51"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:11.140109662Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1708,"client":"2","":"downstairs","session_id":"10a59e1c-6a65-4851-9de7-081e60e54d51"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:11.140123104Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1708,"client":"2","":"downstairs","session_id":"10a59e1c-6a65-4851-9de7-081e60e54d51"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:11.140136356Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1708,"session_id":"10a59e1c-6a65-4851-9de7-081e60e54d51"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:11.140148327Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1708,"client":"1","":"downstairs","session_id":"10a59e1c-6a65-4851-9de7-081e60e54d51"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:11.140161629Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1708,"client":"1","":"downstairs","session_id":"10a59e1c-6a65-4851-9de7-081e60e54d51"} {"msg":"[0]R flush_numbers[0..12]: [193, 213, 209, 209, 215, 209, 215, 214, 208, 216, 216, 214]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:11.140175511Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1708,"":"downstairs","":"downstairs","session_id":"10a59e1c-6a65-4851-9de7-081e60e54d51"} {"msg":"[0]R generation[0..12]: [119, 128, 126, 126, 130, 126, 130, 129, 125, 131, 131, 129]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:11.140189433Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1708,"":"downstairs","":"downstairs","session_id":"10a59e1c-6a65-4851-9de7-081e60e54d51"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:11.140203284Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1708,"":"downstairs","":"downstairs","session_id":"10a59e1c-6a65-4851-9de7-081e60e54d51"} {"msg":"[1]R flush_numbers[0..12]: [193, 213, 209, 209, 215, 209, 215, 214, 208, 216, 216, 214]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:11.140217156Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1708,"":"downstairs","":"downstairs","session_id":"10a59e1c-6a65-4851-9de7-081e60e54d51"} {"msg":"[1]R generation[0..12]: [119, 128, 126, 126, 130, 126, 130, 129, 125, 131, 131, 129]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:11.140231218Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1708,"":"downstairs","":"downstairs","session_id":"10a59e1c-6a65-4851-9de7-081e60e54d51"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:11.14024533Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1708,"":"downstairs","":"downstairs","session_id":"10a59e1c-6a65-4851-9de7-081e60e54d51"} {"msg":"[2]R flush_numbers[0..12]: [193, 213, 209, 209, 215, 209, 215, 214, 208, 216, 216, 214]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:11.140259311Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1708,"":"downstairs","":"downstairs","session_id":"10a59e1c-6a65-4851-9de7-081e60e54d51"} {"msg":"[2]R generation[0..12]: [119, 128, 126, 126, 130, 126, 130, 129, 125, 131, 131, 129]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:11.140276104Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1708,"":"downstairs","":"downstairs","session_id":"10a59e1c-6a65-4851-9de7-081e60e54d51"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:11.140290295Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1708,"":"downstairs","":"downstairs","session_id":"10a59e1c-6a65-4851-9de7-081e60e54d51"} {"msg":"Max found gen is 132","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:11.140303497Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1708,"":"downstairs","":"downstairs","session_id":"10a59e1c-6a65-4851-9de7-081e60e54d51"} {"msg":"Generation requested: 132 >= found:132","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:11.140316679Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1708,"":"downstairs","":"downstairs","session_id":"10a59e1c-6a65-4851-9de7-081e60e54d51"} {"msg":"Next flush: 217","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:11.140329771Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1708,"":"downstairs","":"downstairs","session_id":"10a59e1c-6a65-4851-9de7-081e60e54d51"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:11.140343492Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1708,"":"downstairs","":"downstairs","session_id":"10a59e1c-6a65-4851-9de7-081e60e54d51"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:11.140356914Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1708,"session_id":"10a59e1c-6a65-4851-9de7-081e60e54d51"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:11.140369006Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1708,"client":"0","":"downstairs","session_id":"10a59e1c-6a65-4851-9de7-081e60e54d51"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-01T16:55:11.140382347Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1708,"client":"1","":"downstairs","session_id":"10a59e1c-6a65-4851-9de7-081e60e54d51"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:11.14040508Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1708,"client":"2","":"downstairs","session_id":"10a59e1c-6a65-4851-9de7-081e60e54d51"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:11.140418512Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1708,"session_id":"10a59e1c-6a65-4851-9de7-081e60e54d51"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:11.140430904Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1708,"session_id":"10a59e1c-6a65-4851-9de7-081e60e54d51"} {"msg":"d87d61a0-664d-4f8c-bbf3-65d105277ad3 is now active with session: 10a59e1c-6a65-4851-9de7-081e60e54d51","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:11.140443595Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1708,"session_id":"10a59e1c-6a65-4851-9de7-081e60e54d51"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:11.140455447Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1708,"session_id":"10a59e1c-6a65-4851-9de7-081e60e54d51"} {"msg":"The guest has finished waiting for activation with:132","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:11.140467758Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1708} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 466 len 12288 data: 9 8 8 02/30 Read block 9 len 16384 03/30 Write block 467 len 28672 data: 9 9 10 11 13 13 14 04/30 Write block 564 len 24576 data: 15 8 8 9 7 9 05/30 Read block 441 len 4096 06/30 Read block 393 len 12288 07/30 Write block 89 len 16384 data: 16 18 14 12 08/30 Write block 505 len 40960 data: 8 10 10 12 11 11 10 10 7 6 09/30 Read block 84 len 32768 10/30 Read block 179 len 32768 11/30 Write block 374 len 24576 data: 8 6 7 10 11 11 12/30 Write block 562 len 40960 data: 14 17 16 9 9 10 8 10 7 9 13/30 Write block 561 len 32768 data: 14 15 18 17 10 10 11 9 14/30 Read block 485 len 12288 15/30 Read block 100 len 24576 16/30 Write block 351 len 4096 data: 11 17/30 Read block 545 len 32768 18/30 Flush 19/30 Write block 147 len 36864 data: 8 8 11 14 11 13 10 11 12 20/30 Write block 222 len 32768 data: 13 16 15 13 10 8 6 9 21/30 Read block 299 len 36864 22/30 Write block 554 len 16384 data: 14 12 13 13 23/30 Read block 56 len 40960 24/30 Write block 475 len 16384 data: 11 8 12 15 25/30 Write block 126 len 24576 data: 12 12 13 14 16 16 26/30 Read block 53 len 36864 27/30 Write block 380 len 40960 data: 9 10 11 11 10 11 9 8 8 10 28/30 Write block 231 len 4096 data: 6 29/30 Write block 538 len 32768 data: 13 13 12 8 6 8 7 8 30/30 Write block 12 len 36864 data: 14 13 14 13 12 12 11 11 9 ---------------------------------------------------------------- Crucible gen:132 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1018 Acked Write 9 Done Done Done false 1019 Acked Write 8 Done Done Done false 1020 Acked Read 9 Done Done Done false 1021 Acked Write 4 Done Done Done false 1022 Acked Read 10 Done Done Done false 1023 Acked Write 4 Done Done Done false 1024 Acked Write 6 Done Done Done false 1025 Acked Read 9 Done Done Done false 1026 Acked Write 10 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Write 8 Done Sent Done false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 2 1 4 Done 11 10 11 32 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1017 1017 1017 Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:55:11.164104134Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1708} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 119 119 119 193 193 193 T T T 1 020-039 128 128 128 213 213 213 T T T 2 040-059 126 126 126 209 209 209 F F F 3 060-079 126 126 126 209 209 209 F F F 4 080-099 132 132 132 217 217 217 F F F 5 100-119 126 126 126 209 209 209 F F F 6 120-139 130 130 130 215 215 215 T T T 7 140-159 129 129 129 214 214 214 T T T 8 160-179 125 125 125 208 208 208 F F F 9 180-199 131 131 131 216 216 216 F F F 10 200-219 131 131 131 216 216 216 F F F 11 220-239 129 129 129 214 214 214 T T T 12 240-259 129 129 129 214 214 214 F F F 13 260-279 128 128 128 212 212 212 F F F 14 280-299 125 125 125 208 208 208 F F F 15 300-319 131 131 131 216 216 216 F F F 16 320-339 125 125 125 208 208 208 F F F 17 340-359 132 132 132 217 217 217 F F F 18 360-379 132 132 132 217 217 217 F F F 19 380-399 123 123 123 204 204 204 T T T 20 400-419 130 130 130 215 215 215 F F F 21 420-439 130 130 130 215 215 215 F F F 22 440-459 130 130 130 215 215 215 F F F 23 460-479 132 132 132 217 217 217 T T T 24 480-499 131 131 131 216 216 216 F F F 25 500-519 132 132 132 217 217 217 F F F 26 520-539 130 130 130 215 215 215 T T T 27 540-559 130 130 130 215 215 215 T T T 28 560-579 132 132 132 217 217 217 F F F 29 580-599 126 126 126 209 209 209 F F F Max gen: 132, Max flush: 217 A Difference in extent metadata is expected here On loop 66 Verifying data now No change after verify Loop: 66 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 133 133 133 218 218 218 F F F 1 020-039 133 133 133 218 218 218 F F F 2 040-059 126 126 126 209 209 209 F F F 3 060-079 126 126 126 209 209 209 F F F 4 080-099 132 132 132 217 217 217 F F F 5 100-119 126 126 126 209 209 209 F F F 6 120-139 133 133 133 218 218 218 F F F 7 140-159 133 133 133 218 218 218 F F F 8 160-179 125 125 125 208 208 208 F F F 9 180-199 131 131 131 216 216 216 F F F 10 200-219 131 131 131 216 216 216 F F F 11 220-239 133 133 133 218 218 218 F F F 12 240-259 129 129 129 214 214 214 F F F 13 260-279 128 128 128 212 212 212 F F F 14 280-299 125 125 125 208 208 208 F F F 15 300-319 131 131 131 216 216 216 F F F 16 320-339 125 125 125 208 208 208 F F F 17 340-359 132 132 132 217 217 217 F F F 18 360-379 132 132 132 217 217 217 F F F 19 380-399 133 133 133 218 218 218 F F F 20 400-419 130 130 130 215 215 215 F F F 21 420-439 130 130 130 215 215 215 F F F 22 440-459 130 130 130 215 215 215 F F F 23 460-479 133 133 133 218 218 218 F F F 24 480-499 131 131 131 216 216 216 F F F 25 500-519 132 132 132 217 217 217 F F F 26 520-539 133 133 133 218 218 218 F F F 27 540-559 133 133 133 218 218 218 F F F 28 560-579 132 132 132 217 217 217 F F F 29 580-599 126 126 126 209 209 209 F F F Max gen: 133, Max flush: 218 Begin loop 67 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:15.518155852Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1716,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:15.561960946Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1716,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:15.56253384Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1716} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"3818f0fc76fe787c6cc412af30fd59b930203885\",\n git_commit_timestamp: \"2024-11-01T16:08:30.000000000Z\",\n git_branch: \"renovate/hyper-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:15.562575735Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1716} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:15.562593797Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1716} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:15.562608669Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1716} {"msg":"Crucible 3f0efa1a-0946-4bc6-bd65-e32b9724c16a has session id: 330d9a3d-ea9e-4c7e-8493-71b017011aa6","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:15.562964815Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1716,"session_id":"330d9a3d-ea9e-4c7e-8493-71b017011aa6"} {"msg":"Upstairs opts: Upstairs UUID: 3f0efa1a-0946-4bc6-bd65-e32b9724c16a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:15.563009831Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1716,"session_id":"330d9a3d-ea9e-4c7e-8493-71b017011aa6"} {"msg":"Crucible stats registered with UUID: 3f0efa1a-0946-4bc6-bd65-e32b9724c16a","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:15.563040325Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1716,"session_id":"330d9a3d-ea9e-4c7e-8493-71b017011aa6"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:15.594032648Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1716,"task":"crutest"} {"msg":"The guest has requested activation with gen:134","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:15.594071783Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1716} {"msg":"3f0efa1a-0946-4bc6-bd65-e32b9724c16a active request set","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:15.59420062Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1716,"session_id":"330d9a3d-ea9e-4c7e-8493-71b017011aa6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:15.594228773Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1716,"client":"0","":"downstairs","session_id":"330d9a3d-ea9e-4c7e-8493-71b017011aa6"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:15.594263348Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1716,"client":"0","":"downstairs","session_id":"330d9a3d-ea9e-4c7e-8493-71b017011aa6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:15.594284751Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1716,"client":"1","":"downstairs","session_id":"330d9a3d-ea9e-4c7e-8493-71b017011aa6"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:15.594305353Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1716,"client":"1","":"downstairs","session_id":"330d9a3d-ea9e-4c7e-8493-71b017011aa6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:15.594325616Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1716,"client":"2","":"downstairs","session_id":"330d9a3d-ea9e-4c7e-8493-71b017011aa6"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:15.594346039Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1716,"client":"2","":"downstairs","session_id":"330d9a3d-ea9e-4c7e-8493-71b017011aa6"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:15.594365941Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1716,"":"io task","client":"2","":"downstairs","session_id":"330d9a3d-ea9e-4c7e-8493-71b017011aa6"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:15.594395835Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1716,"":"io task","client":"0","":"downstairs","session_id":"330d9a3d-ea9e-4c7e-8493-71b017011aa6"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:15.594439471Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1716,"":"io task","client":"1","":"downstairs","session_id":"330d9a3d-ea9e-4c7e-8493-71b017011aa6"} {"msg":"ds_connection connected from Ok(127.0.0.1:55688)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:15.594887688Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1716,"":"io task","client":"1","":"downstairs","session_id":"330d9a3d-ea9e-4c7e-8493-71b017011aa6"} {"msg":"ds_connection connected from Ok(127.0.0.1:57879)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:15.594968529Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1716,"":"io task","client":"2","":"downstairs","session_id":"330d9a3d-ea9e-4c7e-8493-71b017011aa6"} {"msg":"ds_connection connected from Ok(127.0.0.1:47710)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:15.595037408Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1716,"":"io task","client":"0","":"downstairs","session_id":"330d9a3d-ea9e-4c7e-8493-71b017011aa6"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:15.595565246Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1716,"client":"0","":"downstairs","session_id":"330d9a3d-ea9e-4c7e-8493-71b017011aa6"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:15.595608211Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1716,"client":"0","":"downstairs","session_id":"330d9a3d-ea9e-4c7e-8493-71b017011aa6"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:15.595642226Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1716,"client":"2","":"downstairs","session_id":"330d9a3d-ea9e-4c7e-8493-71b017011aa6"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:15.59567316Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1716,"client":"2","":"downstairs","session_id":"330d9a3d-ea9e-4c7e-8493-71b017011aa6"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:15.595689912Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1716,"client":"1","":"downstairs","session_id":"330d9a3d-ea9e-4c7e-8493-71b017011aa6"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:15.595703783Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1716,"client":"1","":"downstairs","session_id":"330d9a3d-ea9e-4c7e-8493-71b017011aa6"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:15.596240543Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1716,"client":"2","":"downstairs","session_id":"330d9a3d-ea9e-4c7e-8493-71b017011aa6"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:15.596271847Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1716,"client":"0","":"downstairs","session_id":"330d9a3d-ea9e-4c7e-8493-71b017011aa6"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:15.59630007Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1716,"client":"1","":"downstairs","session_id":"330d9a3d-ea9e-4c7e-8493-71b017011aa6"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:15.596774321Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1716,"client":"2","":"downstairs","session_id":"330d9a3d-ea9e-4c7e-8493-71b017011aa6"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:15.596803415Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1716,"client":"2","":"downstairs","session_id":"330d9a3d-ea9e-4c7e-8493-71b017011aa6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:15.596818277Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1716,"session_id":"330d9a3d-ea9e-4c7e-8493-71b017011aa6"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:15.596897857Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1716,"client":"1","":"downstairs","session_id":"330d9a3d-ea9e-4c7e-8493-71b017011aa6"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:15.59691694Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1716,"client":"1","":"downstairs","session_id":"330d9a3d-ea9e-4c7e-8493-71b017011aa6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:15.596931352Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1716,"session_id":"330d9a3d-ea9e-4c7e-8493-71b017011aa6"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:15.596943733Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1716,"client":"0","":"downstairs","session_id":"330d9a3d-ea9e-4c7e-8493-71b017011aa6"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:15.596958105Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1716,"client":"0","":"downstairs","session_id":"330d9a3d-ea9e-4c7e-8493-71b017011aa6"} {"msg":"[0]R flush_numbers[0..12]: [218, 218, 209, 209, 217, 209, 218, 218, 208, 216, 216, 218]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:15.596972477Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1716,"":"downstairs","":"downstairs","session_id":"330d9a3d-ea9e-4c7e-8493-71b017011aa6"} {"msg":"[0]R generation[0..12]: [133, 133, 126, 126, 132, 126, 133, 133, 125, 131, 131, 133]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:15.596986699Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1716,"":"downstairs","":"downstairs","session_id":"330d9a3d-ea9e-4c7e-8493-71b017011aa6"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:15.597015203Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1716,"":"downstairs","":"downstairs","session_id":"330d9a3d-ea9e-4c7e-8493-71b017011aa6"} {"msg":"[1]R flush_numbers[0..12]: [218, 218, 209, 209, 217, 209, 218, 218, 208, 216, 216, 218]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:15.597030434Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1716,"":"downstairs","":"downstairs","session_id":"330d9a3d-ea9e-4c7e-8493-71b017011aa6"} {"msg":"[1]R generation[0..12]: [133, 133, 126, 126, 132, 126, 133, 133, 125, 131, 131, 133]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:15.597044246Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1716,"":"downstairs","":"downstairs","session_id":"330d9a3d-ea9e-4c7e-8493-71b017011aa6"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:15.597058148Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1716,"":"downstairs","":"downstairs","session_id":"330d9a3d-ea9e-4c7e-8493-71b017011aa6"} {"msg":"[2]R flush_numbers[0..12]: [218, 218, 209, 209, 217, 209, 218, 218, 208, 216, 216, 218]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:15.5970719Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1716,"":"downstairs","":"downstairs","session_id":"330d9a3d-ea9e-4c7e-8493-71b017011aa6"} {"msg":"[2]R generation[0..12]: [133, 133, 126, 126, 132, 126, 133, 133, 125, 131, 131, 133]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:15.597086642Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1716,"":"downstairs","":"downstairs","session_id":"330d9a3d-ea9e-4c7e-8493-71b017011aa6"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:15.597103314Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1716,"":"downstairs","":"downstairs","session_id":"330d9a3d-ea9e-4c7e-8493-71b017011aa6"} {"msg":"Max found gen is 134","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:15.597116676Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1716,"":"downstairs","":"downstairs","session_id":"330d9a3d-ea9e-4c7e-8493-71b017011aa6"Wait for a query_work_queue command to finish before sending IO } {"msg":"Generation requested: 134 >= found:134","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:15.597138028Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1716,"":"downstairs","":"downstairs","session_id":"330d9a3d-ea9e-4c7e-8493-71b017011aa6"} {"msg":"Next flush: 219","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:15.5971513Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1716,"":"downstairs","":"downstairs","session_id":"330d9a3d-ea9e-4c7e-8493-71b017011aa6"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:15.597164422Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1716,"":"downstairs","":"downstairs","session_id":"330d9a3d-ea9e-4c7e-8493-71b017011aa6"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:15.597177704Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1716,"session_id":"330d9a3d-ea9e-4c7e-8493-71b017011aa6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:15.597189765Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1716,"client":"0","":"downstairs","session_id":"330d9a3d-ea9e-4c7e-8493-71b017011aa6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:15.597202977Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1716,"client":"1","":"downstairs","session_id":"330d9a3d-ea9e-4c7e-8493-71b017011aa6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:15.597216149Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1716,"client":"2","":"downstairs","session_id":"330d9a3d-ea9e-4c7e-8493-71b017011aa6"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:15.59722933Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1716,"session_id":"330d9a3d-ea9e-4c7e-8493-71b017011aa6"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:15.597241752Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1716,"session_id":"330d9a3d-ea9e-4c7e-8493-71b017011aa6"} {"msg":"3f0efa1a-0946-4bc6-bd65-e32b9724c16a is now active with session: 330d9a3d-ea9e-4c7e-8493-71b017011aa6","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:15.597254583Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1716,"session_id":"330d9a3d-ea9e-4c7e-8493-71b017011aa6"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:15.597266585Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1716,"session_id":"330d9a3d-ea9e-4c7e-8493-71b017011aa6"} {"msg":"The guest has finished waiting for activation with:134","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:15.597278997Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1716} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 451 len 28672 data: 13 10 10 9 6 7 6 02/30 Flush 03/30 Write block 527 len 20480 data: 9 11 12 14 13 04/30 Flush 05/30 Write block 146 len 28672 data: 6 9 9 12 15 12 14 06/30 Write block 97 len 36864 data: 15 15 15 16 15 13 11 12 13 07/30 Flush 08/30 Write block 357 len 8192 data: 11 10 09/30 Write block 42 len 28672 data: 13 16 16 13 15 11 9 10/30 Read block 248 len 16384 11/30 Write block 178 len 20480 data: 11 10 10 8 8 12/30 Write block 119 len 40960 data: 9 8 11 13 11 10 11 13 13 14 13/30 Read block 290 len 36864 14/30 Read block 79 len 20480 15/30 Read block 394 len 4096 16/30 Read block 65 len 12288 17/30 Read block 152 len 16384 18/30 Read block 85 len 36864 19/30 Write block 165 len 40960 data: 10 8 7 8 8 9 7 6 7 7 20/30 Write block 320 len 12288 data: 13 13 12 21/30 Read block 337 len 16384 22/30 Write block 307 len 24576 data: 13 15 15 13 14 14 23/30 Write block 144 len 16384 data: 7 7 7 10 24/30 Write block 382 len 12288 data: 12 12 11 25/30 Read block 466 len 16384 26/30 Read block 356 len 4096 27/30 Read block 411 len 8192 28/30 Read block 76 len 40960 29/30 Read block 320 len 40960 30/30 Write block 241 len 32768 data: 9 9 9 10 14 17 19 19 ---------------------------------------------------------------- Crucible gen:134 GIO:true work queues: Upstairs:0 downstairs:23 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1007 Acked Write 2 Done Done Done false 1008 Acked Write 7 Done Done Done false 1009 Acked Read 4 Done Done Done false 1010 Acked Write 5 Done Done Done false 1011 Acked Write 10 Done Done Done false 1012 Acked Read 9 Done Done Done false 1013 Acked Read 5 Done Done Done false 1014 Acked Read 1 Done Done Done false 1015 Acked Read 3 Done Done Done false 1016 Acked Read 4 Done Done Done false 1017 Acked Read 9 Done Done Done false 1018 Acked Write 10 Done Done Done false 1019 Acked Write 3 Done Done Done false 1020 Acked Read 4 Done Done Done false 1021 Acked Write 6 Done Done Done false 1022 Acked Write 4 Done Done Done false 1023 Acked Write 3 Done Done Done false 1024 Acked Read 4 Done Done Done false 1025 Acked Read 1 Done Done Done false 1026 Acked Read 2 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Read 10 Done Done Done false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 22 22 22 66 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1006 1006 1006 Downstairs last five completed: 1006 1005 1004 1003 1002 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:55:15.625191243Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1716} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 133 133 133 218 218 218 F F F 1 020-039 133 133 133 218 218 218 F F F 2 040-059 126 126 126 209 209 209 T T T 3 060-079 126 126 126 209 209 209 F F F 4 080-099 134 134 134 221 221 221 F F F 5 100-119 134 134 134 221 221 221 T T T 6 120-139 133 133 133 218 218 218 T T T 7 140-159 134 134 134 221 221 221 T T T 8 160-179 125 125 125 208 208 208 T T T 9 180-199 131 131 131 216 216 216 T T T 10 200-219 131 131 131 216 216 216 F F F 11 220-239 133 133 133 218 218 218 F F F 12 240-259 129 129 129 214 214 214 T T T 13 260-279 128 128 128 212 212 212 F F F 14 280-299 125 125 125 208 208 208 F F F 15 300-319 131 131 131 216 216 216 T T T 16 320-339 125 125 125 208 208 208 T T T 17 340-359 132 132 132 217 217 217 T T T 18 360-379 132 132 132 217 217 217 F F F 19 380-399 133 133 133 218 218 218 T T T 20 400-419 130 130 130 215 215 215 F F F 21 420-439 130 130 130 215 215 215 F F F 22 440-459 134 134 134 219 219 219 F F F 23 460-479 133 133 133 218 218 218 F F F 24 480-499 131 131 131 216 216 216 F F F 25 500-519 132 132 132 217 217 217 F F F 26 520-539 134 134 134 220 220 220 F F F 27 540-559 133 133 133 218 218 218 F F F 28 560-579 132 132 132 217 217 217 F F F 29 580-599 126 126 126 209 209 209 F F F Max gen: 134, Max flush: 221 A Difference in extent metadata is expected here On loop 67 Verifying data now No change after verify Loop: 67 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 133 133 133 218 218 218 F F F 1 020-039 133 133 133 218 218 218 F F F 2 040-059 135 135 135 222 222 222 F F F 3 060-079 126 126 126 209 209 209 F F F 4 080-099 134 134 134 221 221 221 F F F 5 100-119 135 135 135 222 222 222 F F F 6 120-139 135 135 135 222 222 222 F F F 7 140-159 135 135 135 222 222 222 F F F 8 160-179 135 135 135 222 222 222 F F F 9 180-199 135 135 135 222 222 222 F F F 10 200-219 131 131 131 216 216 216 F F F 11 220-239 133 133 133 218 218 218 F F F 12 240-259 135 135 135 222 222 222 F F F 13 260-279 128 128 128 212 212 212 F F F 14 280-299 125 125 125 208 208 208 F F F 15 300-319 135 135 135 222 222 222 F F F 16 320-339 135 135 135 222 222 222 F F F 17 340-359 135 135 135 222 222 222 F F F 18 360-379 132 132 132 217 217 217 F F F 19 380-399 135 135 135 222 222 222 F F F 20 400-419 130 130 130 215 215 215 F F F 21 420-439 130 130 130 215 215 215 F F F 22 440-459 134 134 134 219 219 219 F F F 23 460-479 133 133 133 218 218 218 F F F 24 480-499 131 131 131 216 216 216 F F F 25 500-519 132 132 132 217 217 217 F F F 26 520-539 134 134 134 220 220 220 F F F 27 540-559 133 133 133 218 218 218 F F F 28 560-579 132 132 132 217 217 217 F F F 29 580-599 126 126 126 209 209 209 F F F Max gen: 135, Max flush: 222 Begin loop 68 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:20.033287214Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1724,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:55:20.075623409Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1724,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:20.075992906Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1724,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:20.107852832Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1724,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:20.108371358Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1724} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"3818f0fc76fe787c6cc412af30fd59b930203885\",\n git_commit_timestamp: \"2024-11-01T16:08:30.000000000Z\",\n git_branch: \"renovate/hyper-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:20.108405043Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1724} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:20.108421385Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1724} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:20.108436147Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1724} {"msg":"Crucible 49cce664-4d65-4286-a7c9-e9d63aa99e96 has session id: 26b8bb5a-7367-4d14-bcca-de73d6290aa0","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:20.108827467Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1724,"session_id":"26b8bb5a-7367-4d14-bcca-de73d6290aa0"} {"msg":"Upstairs opts: Upstairs UUID: 49cce664-4d65-4286-a7c9-e9d63aa99e96, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:20.108874913Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1724,"session_id":"26b8bb5a-7367-4d14-bcca-de73d6290aa0"} {"msg":"Crucible stats registered with UUID: 49cce664-4d65-4286-a7c9-e9d63aa99e96","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:20.108900116Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1724,"session_id":"26b8bb5a-7367-4d14-bcca-de73d6290aa0"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:20.139480227Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1724,"task":"crutest"} {"msg":"The guest has requested activation with gen:136","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:20.139511291Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1724} {"msg":"49cce664-4d65-4286-a7c9-e9d63aa99e96 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:20.139573299Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1724,"session_id":"26b8bb5a-7367-4d14-bcca-de73d6290aa0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:20.139600672Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1724,"client":"0","":"downstairs","session_id":"26b8bb5a-7367-4d14-bcca-de73d6290aa0"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:20.139627006Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1724,"client":"0","":"downstairs","session_id":"26b8bb5a-7367-4d14-bcca-de73d6290aa0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:20.139641867Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1724,"client":"1","":"downstairs","session_id":"26b8bb5a-7367-4d14-bcca-de73d6290aa0"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:20.139655899Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1724,"client":"1","":"downstairs","session_id":"26b8bb5a-7367-4d14-bcca-de73d6290aa0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:20.139670341Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1724,"client":"2","":"downstairs","session_id":"26b8bb5a-7367-4d14-bcca-de73d6290aa0"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:20.139684183Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1724,"client":"2","":"downstairs","session_id":"26b8bb5a-7367-4d14-bcca-de73d6290aa0"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:20.139700845Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1724,"":"io task","client":"2","":"downstairs","session_id":"26b8bb5a-7367-4d14-bcca-de73d6290aa0"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:20.139715317Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1724,"":"io task","client":"0","":"downstairs","session_id":"26b8bb5a-7367-4d14-bcca-de73d6290aa0"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:20.13974026Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1724,"":"io task","client":"1","":"downstairs","session_id":"26b8bb5a-7367-4d14-bcca-de73d6290aa0"} {"msg":"ds_connection connected from Ok(127.0.0.1:54926)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:20.139787026Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1724,"":"io task","client":"2","":"downstairs","session_id":"26b8bb5a-7367-4d14-bcca-de73d6290aa0"} {"msg":"ds_connection connected from Ok(127.0.0.1:46625)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:20.13981998Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1724,"":"io task","client":"1","":"downstairs","session_id":"26b8bb5a-7367-4d14-bcca-de73d6290aa0"} {"msg":"ds_connection connected from Ok(127.0.0.1:42730)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:20.139848614Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1724,"":"io task","client":"0","":"downstairs","session_id":"26b8bb5a-7367-4d14-bcca-de73d6290aa0"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:20.140573757Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1724,"client":"0","":"downstairs","session_id":"26b8bb5a-7367-4d14-bcca-de73d6290aa0"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:20.140616963Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1724,"client":"0","":"downstairs","session_id":"26b8bb5a-7367-4d14-bcca-de73d6290aa0"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:20.140647537Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1724,"client":"2","":"downstairs","session_id":"26b8bb5a-7367-4d14-bcca-de73d6290aa0"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:20.140677811Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1724,"client":"2","":"downstairs","session_id":"26b8bb5a-7367-4d14-bcca-de73d6290aa0"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:20.140698584Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1724,"client":"1","":"downstairs","session_id":"26b8bb5a-7367-4d14-bcca-de73d6290aa0"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:20.140712775Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1724,"client":"1","":"downstairs","session_id":"26b8bb5a-7367-4d14-bcca-de73d6290aa0"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:20.141253765Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1724,"client":"1","":"downstairs","session_id":"26b8bb5a-7367-4d14-bcca-de73d6290aa0"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:20.141376021Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1724,"client":"0","":"downstairs","session_id":"26b8bb5a-7367-4d14-bcca-de73d6290aa0"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:20.141502107Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1724,"client":"2","":"downstairs","session_id":"26b8bb5a-7367-4d14-bcca-de73d6290aa0"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:20.141869164Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1724,"client":"1","":"downstairs","session_id":"26b8bb5a-7367-4d14-bcca-de73d6290aa0"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:20.141908059Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1724,"client":"1","":"downstairs","session_id":"26b8bb5a-7367-4d14-bcca-de73d6290aa0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:20.141937963Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1724,"session_id":"26b8bb5a-7367-4d14-bcca-de73d6290aa0"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:20.142001251Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1724,"client":"0","":"downstairs","session_id":"26b8bb5a-7367-4d14-bcca-de73d6290aa0"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:20.142029835Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1724,"client":"0","":"downstairs","session_id":"26b8bb5a-7367-4d14-bcca-de73d6290aa0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:20.142045207Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1724,"session_id":"26b8bb5a-7367-4d14-bcca-de73d6290aa0"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:20.142058029Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1724,"client":"2","":"downstairs","session_id":"26b8bb5a-7367-4d14-bcca-de73d6290aa0"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:20.142072041Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1724,"client":"2","":"downstairs","session_id":"26b8bb5a-7367-4d14-bcca-de73d6290aa0"} {"msg":"[0]R flush_numbers[0..12]: [218, 218, 222, 209, 221, 222, 222, 222, 222, 222, 216, 218]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:20.142086172Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1724,"":"downstairs","":"downstairs","session_id":"26b8bb5a-7367-4d14-bcca-de73d6290aa0"} {"msg":"[0]R generation[0..12]: [133, 133, 135, 126, 134, 135, 135, 135, 135, 135, 131, 133]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:20.142100074Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1724,"":"downstairs","":"downstairs","session_id":"26b8bb5a-7367-4d14-bcca-de73d6290aa0"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:20.142114026Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1724,"":"downstairs","":"downstairs","session_id":"26b8bb5a-7367-4d14-bcca-de73d6290aa0"} {"msg":"[1]R flush_numbers[0..12]: [218, 218, 222, 209, 221, 222, 222, 222, 222, 222, 216, 218]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:20.142128008Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1724,"":"downstairs","":"downstairs","session_id":"26b8bb5a-7367-4d14-bcca-de73d6290aa0"} {"msg":"[1]R generation[0..12]: [133, 133, 135, 126, 134, 135, 135, 135, 135, 135, 131, 133]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:20.142141859Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1724,"":"downstairs","":"downstairs","session_id":"26b8bb5a-7367-4d14-bcca-de73d6290aa0"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:20.142155911Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1724,"":"downstairs","":"downstairs","session_id":"26b8bb5a-7367-4d14-bcca-de73d6290aa0"} {"msg":"[2]R flush_numbers[0..12]: [218, 218, 222, 209, 221, 222, 222, 222, 222, 222, 216, 218]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:20.142176254Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1724,"":"downstairs","":"downstairs","session_id":"26b8bb5a-7367-4d14-bcca-de73d6290aa0"} {"msg":"[2]R generation[0..12]: [133, 133, 135, 126, 134, 135, 135, 135, 135, 135, 131, 133]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:20.142202397Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1724,"":"downstairs","":"downstairs","session_id":"26b8bb5a-7367-4d14-bcca-de73d6290aa0"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:20.142230291Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1724,"":"downstairs","":"downstairs","session_id":"26b8bb5a-7367-4d14-bcca-de73d6290aa0"} {"msg":"Max found gen is 136","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:20.142254144Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1724,"":"downstairs","Wait for a query_work_queue command to finish before sending IO ":"downstairs","session_id":"26b8bb5a-7367-4d14-bcca-de73d6290aa0"} {"msg":"Generation requested: 136 >= found:136","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:20.142289578Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1724,"":"downstairs","":"downstairs","session_id":"26b8bb5a-7367-4d14-bcca-de73d6290aa0"} {"msg":"Next flush: 223","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:20.142315862Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1724,"":"downstairs","":"downstairs","session_id":"26b8bb5a-7367-4d14-bcca-de73d6290aa0"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:20.142334314Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1724,"":"downstairs","":"downstairs","session_id":"26b8bb5a-7367-4d14-bcca-de73d6290aa0"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:20.142358607Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1724,"session_id":"26b8bb5a-7367-4d14-bcca-de73d6290aa0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:20.14238042Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1724,"client":"0","":"downstairs","session_id":"26b8bb5a-7367-4d14-bcca-de73d6290aa0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:20.142405984Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1724,"client":"1","":"downstairs","session_id":"26b8bb5a-7367-4d14-bcca-de73d6290aa0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:20.142432007Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1724,"client":"2","":"downstairs","session_id":"26b8bb5a-7367-4d14-bcca-de73d6290aa0"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:20.14245273Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1724,"session_id":"26b8bb5a-7367-4d14-bcca-de73d6290aa0"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:20.142469392Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1724,"session_id":"26b8bb5a-7367-4d14-bcca-de73d6290aa0"} {"msg":"49cce664-4d65-4286-a7c9-e9d63aa99e96 is now active with session: 26b8bb5a-7367-4d14-bcca-de73d6290aa0","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:20.142492815Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1724,"session_id":"26b8bb5a-7367-4d14-bcca-de73d6290aa0"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:20.142516378Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1724,"session_id":"26b8bb5a-7367-4d14-bcca-de73d6290aa0"} {"msg":"The guest has finished waiting for activation with:136","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:20.142540231Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1724} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 517 len 28672 data: 7 7 9 8 7 8 6 02/30 Flush 03/30 Write block 118 len 20480 data: 8 10 9 12 14 04/30 Read block 492 len 12288 05/30 Read block 515 len 16384 06/30 Write block 474 len 28672 data: 12 12 9 13 16 14 14 07/30 Write block 375 len 16384 data: 7 8 11 12 08/30 Write block 464 len 16384 data: 6 6 10 10 09/30 Write block 590 len 32768 data: 10 9 8 9 8 10 9 8 10/30 Read block 123 len 32768 11/30 Read block 578 len 40960 12/30 Write block 185 len 40960 data: 10 11 11 11 10 12 12 13 15 17 13/30 Write block 310 len 20480 data: 14 15 15 15 13 14/30 Flush 15/30 Write block 260 len 8192 data: 9 14 16/30 Read block 435 len 40960 17/30 Read block 557 len 24576 18/30 Flush 19/30 Write block 140 len 24576 data: 9 9 9 6 8 8 20/30 Write block 411 len 32768 data: 4 5 5 5 7 6 8 10 21/30 Write block 350 len 16384 data: 10 12 13 13 22/30 Read block 326 len 12288 23/30 Flush 24/30 Write block 95 len 36864 data: 12 15 16 16 16 17 16 14 12 25/30 Flush 26/30 Write block 234 len 28672 data: 7 7 9 8 8 8 8 27/30 Write block 95 len 28672 data: 13 16 17 17 17 18 17 28/30 Read block 76 len 8192 29/30 Read block 173 len 4096 30/30 Write block 365 len 36864 data: 8 7 6 5 7 6 5 6 8 ---------------------------------------------------------------- Crucible gen:136 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 7 Done Done Done false 1026 Acked Write 7 Done Done Done false 1027 Acked Read 2 Done Done Done false 1028 Acked Read 1 Done Done Done false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:55:20.170580164Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1724} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 133 133 133 218 218 218 F F F 1 020-039 133 133 133 218 218 218 F F F 2 040-059 135 135 135 222 222 222 F F F 3 060-079 126 126 126 209 209 209 F F F 4 080-099 136 136 136 227 227 227 T T T 5 100-119 136 136 136 227 227 227 T T T 6 120-139 136 136 136 224 224 224 F F F 7 140-159 136 136 136 226 226 226 F F F 8 160-179 135 135 135 222 222 222 F F F 9 180-199 136 136 136 224 224 224 F F F 10 200-219 131 131 131 216 216 216 F F F 11 220-239 133 133 133 218 218 218 T T T 12 240-259 135 135 135 222 222 222 T T T 13 260-279 136 136 136 225 225 225 F F F 14 280-299 125 125 125 208 208 208 F F F 15 300-319 136 136 136 224 224 224 F F F 16 320-339 135 135 135 222 222 222 F F F 17 340-359 136 136 136 226 226 226 F F F 18 360-379 136 136 136 224 224 224 T T T 19 380-399 135 135 135 222 222 222 F F F 20 400-419 136 136 136 226 226 226 F F F 21 420-439 130 130 130 215 215 215 F F F 22 440-459 134 134 134 219 219 219 F F F 23 460-479 136 136 136 224 224 224 F F F 24 480-499 136 136 136 224 224 224 F F F 25 500-519 136 136 136 223 223 223 F F F 26 520-539 136 136 136 223 223 223 F F F 27 540-559 133 133 133 218 218 218 F F F 28 560-579 132 132 132 217 217 217 F F F 29 580-599 136 136 136 224 224 224 F F F Max gen: 136, Max flush: 227 A Difference in extent metadata is expected here On loop 68 Verifying data now No change after verify Loop: 68 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 133 133 133 218 218 218 F F F 1 020-039 133 133 133 218 218 218 F F F 2 040-059 135 135 135 222 222 222 F F F 3 060-079 126 126 126 209 209 209 F F F 4 080-099 137 137 137 228 228 228 F F F 5 100-119 137 137 137 228 228 228 F F F 6 120-139 136 136 136 224 224 224 F F F 7 140-159 136 136 136 226 226 226 F F F 8 160-179 135 135 135 222 222 222 F F F 9 180-199 136 136 136 224 224 224 F F F 10 200-219 131 131 131 216 216 216 F F F 11 220-239 137 137 137 228 228 228 F F F 12 240-259 137 137 137 228 228 228 F F F 13 260-279 136 136 136 225 225 225 F F F 14 280-299 125 125 125 208 208 208 F F F 15 300-319 136 136 136 224 224 224 F F F 16 320-339 135 135 135 222 222 222 F F F 17 340-359 136 136 136 226 226 226 F F F 18 360-379 137 137 137 228 228 228 F F F 19 380-399 135 135 135 222 222 222 F F F 20 400-419 136 136 136 226 226 226 F F F 21 420-439 130 130 130 215 215 215 F F F 22 440-459 134 134 134 219 219 219 F F F 23 460-479 136 136 136 224 224 224 F F F 24 480-499 136 136 136 224 224 224 F F F 25 500-519 136 136 136 223 223 223 F F F 26 520-539 136 136 136 223 223 223 F F F 27 540-559 133 133 133 218 218 218 F F F 28 560-579 132 132 132 217 217 217 F F F 29 580-599 136 136 136 224 224 224 F F F Max gen: 137, Max flush: 228 Begin loop 69 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:24.502303935Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1732,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:24.546014658Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1732,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:24.546675323Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1732} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"3818f0fc76fe787c6cc412af30fd59b930203885\",\n git_commit_timestamp: \"2024-11-01T16:08:30.000000000Z\",\n git_branch: \"renovate/hyper-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:24.546723399Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1732} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:24.546739691Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1732} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:24.546753963Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1732} {"msg":"Crucible eb382204-2c1d-400b-b987-c7dc88926a21 has session id: 75547a58-10c5-48f8-bc46-256c0607cc7e","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:24.547183598Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1732,"session_id":"75547a58-10c5-48f8-bc46-256c0607cc7e"} {"msg":"Upstairs opts: Upstairs UUID: eb382204-2c1d-400b-b987-c7dc88926a21, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:24.547223863Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1732,"session_id":"75547a58-10c5-48f8-bc46-256c0607cc7e"} {"msg":"Crucible stats registered with UUID: eb382204-2c1d-400b-b987-c7dc88926a21","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:24.547246706Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1732,"session_id":"75547a58-10c5-48f8-bc46-256c0607cc7e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:24.578597336Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1732,"task":"crutest"} {"msg":"The guest has requested activation with gen:138","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:24.57862996Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1732} {"msg":"eb382204-2c1d-400b-b987-c7dc88926a21 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:24.578688497Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1732,"session_id":"75547a58-10c5-48f8-bc46-256c0607cc7e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:24.578716171Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1732,"client":"0","":"downstairs","session_id":"75547a58-10c5-48f8-bc46-256c0607cc7e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:24.578741444Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1732,"client":"0","":"downstairs","session_id":"75547a58-10c5-48f8-bc46-256c0607cc7e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:24.578757136Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1732,"client":"1","":"downstairs","session_id":"75547a58-10c5-48f8-bc46-256c0607cc7e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:24.578772078Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1732,"client":"1","":"downstairs","session_id":"75547a58-10c5-48f8-bc46-256c0607cc7e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:24.57878647Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1732,"client":"2","":"downstairs","session_id":"75547a58-10c5-48f8-bc46-256c0607cc7e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:24.578800122Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1732,"client":"2","":"downstairs","session_id":"75547a58-10c5-48f8-bc46-256c0607cc7e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:24.578825815Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1732,"":"io task","client":"2","":"downstairs","session_id":"75547a58-10c5-48f8-bc46-256c0607cc7e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:24.578841557Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1732,"":"io task","client":"0","":"downstairs","session_id":"75547a58-10c5-48f8-bc46-256c0607cc7e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:24.578855459Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1732,"":"io task","client":"1","":"downstairs","session_id":"75547a58-10c5-48f8-bc46-256c0607cc7e"} {"msg":"ds_connection connected from Ok(127.0.0.1:47403)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:24.579145616Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1732,"":"io task","client":"0","":"downstairs","session_id":"75547a58-10c5-48f8-bc46-256c0607cc7e"} {"msg":"ds_connection connected from Ok(127.0.0.1:42819)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:24.579184891Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1732,"":"io task","client":"2","":"downstairs","session_id":"75547a58-10c5-48f8-bc46-256c0607cc7e"} {"msg":"ds_connection connected from Ok(127.0.0.1:63151)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:24.579215565Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1732,"":"io task","client":"1","":"downstairs","session_id":"75547a58-10c5-48f8-bc46-256c0607cc7e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:24.579758525Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1732,"client":"2","":"downstairs","session_id":"75547a58-10c5-48f8-bc46-256c0607cc7e"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:24.579786699Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1732,"client":"2","":"downstairs","session_id":"75547a58-10c5-48f8-bc46-256c0607cc7e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:24.579802221Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1732,"client":"0","":"downstairs","session_id":"75547a58-10c5-48f8-bc46-256c0607cc7e"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:24.579816613Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1732,"client":"0","":"downstairs","session_id":"75547a58-10c5-48f8-bc46-256c0607cc7e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:24.579869309Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1732,"client":"1","":"downstairs","session_id":"75547a58-10c5-48f8-bc46-256c0607cc7e"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:24.579903334Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1732,"client":"1","":"downstairs","session_id":"75547a58-10c5-48f8-bc46-256c0607cc7e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:24.580434562Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1732,"client":"0","":"downstairs","session_id":"75547a58-10c5-48f8-bc46-256c0607cc7e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:24.580486879Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1732,"client":"2","":"downstairs","session_id":"75547a58-10c5-48f8-bc46-256c0607cc7e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:24.580578101Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1732,"client":"1","":"downstairs","session_id":"75547a58-10c5-48f8-bc46-256c0607cc7e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:24.580933507Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1732,"client":"0","":"downstairs","session_id":"75547a58-10c5-48f8-bc46-256c0607cc7e"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:24.580971972Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1732,"client":"0","":"downstairs","session_id":"75547a58-10c5-48f8-bc46-256c0607cc7e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:24.580999735Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1732,"session_id":"75547a58-10c5-48f8-bc46-256c0607cc7e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:24.581029359Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1732,"client":"2","":"downstairs","session_id":"75547a58-10c5-48f8-bc46-256c0607cc7e"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:24.581055773Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1732,"client":"2","":"downstairs","session_id":"75547a58-10c5-48f8-bc46-256c0607cc7e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:24.581080305Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1732,"session_id":"75547a58-10c5-48f8-bc46-256c0607cc7e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:24.581174478Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1732,"client":"1","":"downstairs","session_id":"75547a58-10c5-48f8-bc46-256c0607cc7e"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:24.58119602Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1732,"client":"1","":"downstairs","session_id":"75547a58-10c5-48f8-bc46-256c0607cc7e"} {"msg":"[0]R flush_numbers[0..12]: [218, 218, 222, 209, 228, 228, 224, 226, 222, 224, 216, 228]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:24.581210492Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1732,"":"downstairs","":"downstairs","session_id":"75547a58-10c5-48f8-bc46-256c0607cc7e"} {"msg":"[0]R generation[0..12]: [133, 133, 135, 126, 137, 137, 136, 136, 135, 136, 131, 137]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:24.581224764Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1732,"":"downstairs","":"downstairs","session_id":"75547a58-10c5-48f8-bc46-256c0607cc7e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:24.581238876Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1732,"":"downstairs","":"downstairs","session_id":"75547a58-10c5-48f8-bc46-256c0607cc7e"} {"msg":"[1]R flush_numbers[0..12]: [218, 218, 222, 209, 228, 228, 224, 226, 222, 224, 216, 228]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:24.581252578Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1732,"":"downstairs","":"downstairs","session_id":"75547a58-10c5-48f8-bc46-256c0607cc7e"} {"msg":"[1]R generation[0..12]: [133, 133, 135, 126, 137, 137, 136, 136, 135, 136, 131, 137]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:24.581266109Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1732,"":"downstairs","":"downstairs","session_id":"75547a58-10c5-48f8-bc46-256c0607cc7e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:24.581280011Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1732,"":"downstairs","":"downstairs","session_id":"75547a58-10c5-48f8-bc46-256c0607cc7e"} {"msg":"[2]R flush_numbers[0..12]: [218, 218, 222, 209, 228, 228, 224, 226, 222, 224, 216, 228]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:24.581293683Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1732,"":"downstairs","":"downstairs","session_id":"75547a58-10c5-48f8-bc46-256c0607cc7e"} {"msg":"[2]R generation[0..12]: [133, 133, 135, 126, 137, 137, 136, 136, 135, 136, 131, 137]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:24.581307285Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1732,"":"downstairs","":"downstairs","session_id":"75547a58-10c5-48f8-bc46-256c0607cc7e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:24.581332308Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1732,"":"downstairs","":"downstairs","session_id":"75547a58-10c5-48f8-bc46-256c0607cc7e"} {"msg":"Max found gen is 138","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:24.581362162Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1732,"":"downstairs","":"downstairs","session_id":"75547a58-10c5-48f8-bc46-256c0607cc7e"} {"msg":"Generation requested: 138 >= found:138","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:24.581383555Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1732,"":"downstairs","":"downstairs"Wait for a query_work_queue command to finish before sending IO ,"session_id":"75547a58-10c5-48f8-bc46-256c0607cc7e"} {"msg":"Next flush: 229","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:24.581411568Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1732,"":"downstairs","":"downstairs","session_id":"75547a58-10c5-48f8-bc46-256c0607cc7e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:24.581437062Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1732,"":"downstairs","":"downstairs","session_id":"75547a58-10c5-48f8-bc46-256c0607cc7e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:24.581463525Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1732,"session_id":"75547a58-10c5-48f8-bc46-256c0607cc7e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:24.581488218Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1732,"client":"0","":"downstairs","session_id":"75547a58-10c5-48f8-bc46-256c0607cc7e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:24.581514672Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1732,"client":"1","":"downstairs","session_id":"75547a58-10c5-48f8-bc46-256c0607cc7e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:24.581542125Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1732,"client":"2","":"downstairs","session_id":"75547a58-10c5-48f8-bc46-256c0607cc7e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:24.581569899Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1732,"session_id":"75547a58-10c5-48f8-bc46-256c0607cc7e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:24.581595312Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1732,"session_id":"75547a58-10c5-48f8-bc46-256c0607cc7e"} {"msg":"eb382204-2c1d-400b-b987-c7dc88926a21 is now active with session: 75547a58-10c5-48f8-bc46-256c0607cc7e","v"Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 :0,"name":"crucible","level":30,"time":"2024-11-01T16:55:24.581625486Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1732,"session_id":"75547a58-10c5-48f8-bc46-256c0607cc7e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:24.581651589Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1732,"session_id":"75547a58-10c5-48f8-bc46-256c0607cc7e"} {"msg":"The guest has finished waiting for activation with:138","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:24.581667701Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1732} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 330 len 36864 data: 5 6 6 7 7 9 7 8 8 03/30 Write block 523 len 32768 data: 7 7 8 8 10 12 13 15 04/30 Read block 135 len 20480 05/30 Write block 295 len 16384 data: 11 11 11 10 06/30 Write block 241 len 28672 data: 10 10 10 11 15 18 20 07/30 Write block 559 len 20480 data: 20 15 15 16 19 08/30 Write block 411 len 24576 data: 5 6 6 6 8 7 09/30 Read block 399 len 36864 10/30 Read block 145 len 12288 11/30 Write block 82 len 32768 data: 13 12 11 13 13 15 15 17 12/30 Write block 463 len 40960 data: 6 7 7 11 11 10 11 12 14 14 13/30 Flush 14/30 Write block 532 len 12288 data: 13 14 16 15/30 Write block 411 len 12288 data: 6 7 7 16/30 Read block 325 len 28672 17/30 Write block 212 len 32768 data: 13 12 12 12 15 16 16 14 18/30 Read block 240 len 32768 19/30 Write block 373 len 20480 data: 9 9 8 9 12 20/30 Read block 73 len 36864 21/30 Write block 149 len 24576 data: 13 16 13 15 11 12 22/30 Read block 218 len 32768 23/30 Read block 402 len 40960 24/30 Write block 6 len 28672 data: 12 11 12 12 11 12 15 25/30 Read block 271 len 20480 26/30 Write block 360 len 8192 data: 7 7 27/30 Write block 34 len 16384 data: 10 11 13 13 28/30 Read block 181 len 36864 29/30 Write block 89 len 20480 data: 18 19 15 13 13 30/30 Read block 219 len 36864 ---------------------------------------------------------------- Crucible gen:138 GIO:true work queues: Upstairs:0 downstairs:17 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1013 Acked Write 3 Done Done Done false 1014 Acked Write 3 Done Done Done false 1015 Acked Read 7 Done Done Done false 1016 Acked Write 8 Done Done Done false 1017 Acked Read 8 Done Done Done false 1018 Acked Write 5 Done Done Done false 1019 Acked Read 9 Done Done Done false 1020 Acked Write 6 Done Done Done false 1021 Acked Read 8 Done Done Done false 1022 Acked Read 10 Done Done Done false 1023 Acked Write 7 Done Done Done false 1024 Acked Read 5 Done Done Done false 1025 Acked Write 2 Done Done Done false 1026 Acked Write 4 Done Done Done false 1027 Acked Read 9 Done Done Done false 1028 Acked Write 5 Done Done Sent false 1029 Acked Read 9 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 2 3 Done 17 16 15 48 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1012 1012 1012 Downstairs last five completed: 1012 1011 1010 1009 1008 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:55:24.607069654Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1732} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 133 133 133 218 218 218 T T T 1 020-039 133 133 133 218 218 218 T T T 2 040-059 135 135 135 222 222 222 F F F 3 060-079 126 126 126 209 209 209 F F F 4 080-099 138 138 138 230 230 230 T T T 5 100-119 137 137 137 228 228 228 F F F 6 120-139 136 136 136 224 224 224 F F F 7 140-159 136 136 136 226 226 226 T T T 8 160-179 135 135 135 222 222 222 F F F 9 180-199 136 136 136 224 224 224 F F F 10 200-219 131 131 131 216 216 216 T T T 11 220-239 137 137 137 228 228 228 F F F 12 240-259 138 138 138 230 230 230 F F F 13 260-279 136 136 136 225 225 225 F F F 14 280-299 138 138 138 230 230 230 F F F 15 300-319 136 136 136 224 224 224 F F F 16 320-339 138 138 138 230 230 230 F F F 17 340-359 136 136 136 226 226 226 F F F 18 360-379 137 137 137 228 228 228 T T T 19 380-399 135 135 135 222 222 222 F F F 20 400-419 138 138 138 230 230 230 T T T 21 420-439 130 130 130 215 215 215 F F F 22 440-459 134 134 134 219 219 219 F F F 23 460-479 138 138 138 230 230 230 F F F 24 480-499 136 136 136 224 224 224 F F F 25 500-519 136 136 136 223 223 223 F F F 26 520-539 138 138 138 230 230 230 T T T 27 540-559 138 138 138 230 230 230 F F F 28 560-579 138 138 138 230 230 230 F F F 29 580-599 136 136 136 224 224 224 F F F Max gen: 138, Max flush: 230 A Difference in extent metadata is expected here On loop 69 Verifying data now No change after verify Loop: 69 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 139 139 139 231 231 231 F F F 1 020-039 139 139 139 231 231 231 F F F 2 040-059 135 135 135 222 222 222 F F F 3 060-079 126 126 126 209 209 209 F F F 4 080-099 139 139 139 231 231 231 F F F 5 100-119 137 137 137 228 228 228 F F F 6 120-139 136 136 136 224 224 224 F F F 7 140-159 139 139 139 231 231 231 F F F 8 160-179 135 135 135 222 222 222 F F F 9 180-199 136 136 136 224 224 224 F F F 10 200-219 139 139 139 231 231 231 F F F 11 220-239 137 137 137 228 228 228 F F F 12 240-259 138 138 138 230 230 230 F F F 13 260-279 136 136 136 225 225 225 F F F 14 280-299 138 138 138 230 230 230 F F F 15 300-319 136 136 136 224 224 224 F F F 16 320-339 138 138 138 230 230 230 F F F 17 340-359 136 136 136 226 226 226 F F F 18 360-379 139 139 139 231 231 231 F F F 19 380-399 135 135 135 222 222 222 F F F 20 400-419 139 139 139 231 231 231 F F F 21 420-439 130 130 130 215 215 215 F F F 22 440-459 134 134 134 219 219 219 F F F 23 460-479 138 138 138 230 230 230 F F F 24 480-499 136 136 136 224 224 224 F F F 25 500-519 136 136 136 223 223 223 F F F 26 520-539 139 139 139 231 231 231 F F F 27 540-559 138 138 138 230 230 230 F F F 28 560-579 138 138 138 230 230 230 F F F 29 580-599 136 136 136 224 224 224 F F F Max gen: 139, Max flush: 231 Begin loop 70 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:28.946478586Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1740,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:55:28.989145194Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1740,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:28.98950624Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1740,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:29.021418772Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1740,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:29.021972163Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1740} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"3818f0fc76fe787c6cc412af30fd59b930203885\",\n git_commit_timestamp: \"2024-11-01T16:08:30.000000000Z\",\n git_branch: \"renovate/hyper-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:29.022015019Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1740} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:29.022033371Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1740} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:29.022053414Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1740} {"msg":"Crucible c93bb129-11cd-4978-b6af-be74705c555f has session id: 6845a207-04ab-402f-9fbf-b92e72c5c6c9","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:29.022532226Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1740,"session_id":"6845a207-04ab-402f-9fbf-b92e72c5c6c9"} {"msg":"Upstairs opts: Upstairs UUID: c93bb129-11cd-4978-b6af-be74705c555f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:29.022584702Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1740,"session_id":"6845a207-04ab-402f-9fbf-b92e72c5c6c9"} {"msg":"Crucible stats registered with UUID: c93bb129-11cd-4978-b6af-be74705c555f","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:29.022607875Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1740,"session_id":"6845a207-04ab-402f-9fbf-b92e72c5c6c9"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:29.053270186Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1740,"task":"crutest"} {"msg":"The guest has requested activation with gen:140","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:29.053314682Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1740} {"msg":"c93bb129-11cd-4978-b6af-be74705c555f active request set","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:29.053359487Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1740,"session_id":"6845a207-04ab-402f-9fbf-b92e72c5c6c9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:29.053391622Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1740,"client":"0","":"downstairs","session_id":"6845a207-04ab-402f-9fbf-b92e72c5c6c9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:29.053440708Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1740,"client":"0","":"downstairs","session_id":"6845a207-04ab-402f-9fbf-b92e72c5c6c9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:29.053477113Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1740,"client":"1","":"downstairs","session_id":"6845a207-04ab-402f-9fbf-b92e72c5c6c9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:29.053511577Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1740,"client":"1","":"downstairs","session_id":"6845a207-04ab-402f-9fbf-b92e72c5c6c9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:29.053547082Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1740,"client":"2","":"downstairs","session_id":"6845a207-04ab-402f-9fbf-b92e72c5c6c9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:29.053582156Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1740,"client":"2","":"downstairs","session_id":"6845a207-04ab-402f-9fbf-b92e72c5c6c9"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:29.053615371Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1740,"":"io task","client":"2","":"downstairs","session_id":"6845a207-04ab-402f-9fbf-b92e72c5c6c9"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:29.053648165Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1740,"":"io task","client":"0","":"downstairs","session_id":"6845a207-04ab-402f-9fbf-b92e72c5c6c9"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:29.053685609Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1740,"":"io task","client":"1","":"downstairs","session_id":"6845a207-04ab-402f-9fbf-b92e72c5c6c9"} {"msg":"ds_connection connected from Ok(127.0.0.1:52253)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:29.053719504Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1740,"":"io task","client":"2","":"downstairs","session_id":"6845a207-04ab-402f-9fbf-b92e72c5c6c9"} {"msg":"ds_connection connected from Ok(127.0.0.1:44440)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:29.053838419Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1740,"":"io task","client":"0","":"downstairs","session_id":"6845a207-04ab-402f-9fbf-b92e72c5c6c9"} {"msg":"ds_connection connected from Ok(127.0.0.1:44084)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:29.053877084Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1740,"":"io task","client":"1","":"downstairs","session_id":"6845a207-04ab-402f-9fbf-b92e72c5c6c9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:29.054370848Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1740,"client":"2","":"downstairs","session_id":"6845a207-04ab-402f-9fbf-b92e72c5c6c9"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:29.054398511Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1740,"client":"2","":"downstairs","session_id":"6845a207-04ab-402f-9fbf-b92e72c5c6c9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:29.05446277Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1740,"client":"0","":"downstairs","session_id":"6845a207-04ab-402f-9fbf-b92e72c5c6c9"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:29.054482872Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1740,"client":"0","":"downstairs","session_id":"6845a207-04ab-402f-9fbf-b92e72c5c6c9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:29.054496924Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1740,"client":"1","":"downstairs","session_id":"6845a207-04ab-402f-9fbf-b92e72c5c6c9"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:29.054510586Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1740,"client":"1","":"downstairs","session_id":"6845a207-04ab-402f-9fbf-b92e72c5c6c9"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:29.055022482Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1740,"client":"2","":"downstairs","session_id":"6845a207-04ab-402f-9fbf-b92e72c5c6c9"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:29.055064497Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1740,"client":"1","":"downstairs","session_id":"6845a207-04ab-402f-9fbf-b92e72c5c6c9"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:29.055156159Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1740,"client":"0","":"downstairs","session_id":"6845a207-04ab-402f-9fbf-b92e72c5c6c9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:29.055577173Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1740,"client":"2","":"downstairs","session_id":"6845a207-04ab-402f-9fbf-b92e72c5c6c9"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:29.055599826Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1740,"client":"2","":"downstairs","session_id":"6845a207-04ab-402f-9fbf-b92e72c5c6c9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:29.055617589Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1740,"session_id":"6845a207-04ab-402f-9fbf-b92e72c5c6c9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:29.055656554Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1740,"client":"1","":"downstairs","session_id":"6845a207-04ab-402f-9fbf-b92e72c5c6c9"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:29.055673296Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1740,"client":"1","":"downstairs","session_id":"6845a207-04ab-402f-9fbf-b92e72c5c6c9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:29.055686827Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1740,"session_id":"6845a207-04ab-402f-9fbf-b92e72c5c6c9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:29.055722632Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1740,"client":"0","":"downstairs","session_id":"6845a207-04ab-402f-9fbf-b92e72c5c6c9"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:29.055738934Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1740,"client":"0","":"downstairs","session_id":"6845a207-04ab-402f-9fbf-b92e72c5c6c9"} {"msg":"[0]R flush_numbers[0..12]: [231, 231, 222, 209, 231, 228, 224, 231, 222, 224, 231, 228]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:29.055753186Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1740,"":"downstairs","":"downstairs","session_id":"6845a207-04ab-402f-9fbf-b92e72c5c6c9"} {"msg":"[0]R generation[0..12]: [139, 139, 135, 126, 139, 137, 136, 139, 135, 136, 139, 137]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:29.055767268Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1740,"":"downstairs","":"downstairs","session_id":"6845a207-04ab-402f-9fbf-b92e72c5c6c9"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:29.055781299Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1740,"":"downstairs","":"downstairs","session_id":"6845a207-04ab-402f-9fbf-b92e72c5c6c9"} {"msg":"[1]R flush_numbers[0..12]: [231, 231, 222, 209, 231, 228, 224, 231, 222, 224, 231, 228]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:29.055795121Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1740,"":"downstairs","":"downstairs","session_id":"6845a207-04ab-402f-9fbf-b92e72c5c6c9"} {"msg":"[1]R generation[0..12]: [139, 139, 135, 126, 139, 137, 136, 139, 135, 136, 139, 137]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:29.055809033Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1740,"":"downstairs","":"downstairs","session_id":"6845a207-04ab-402f-9fbf-b92e72c5c6c9"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:29.055822965Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1740,"":"downstairs","":"downstairs","session_id":"6845a207-04ab-402f-9fbf-b92e72c5c6c9"} {"msg":"[2]R flush_numbers[0..12]: [231, 231, 222, 209, 231, 228, 224, 231, 222, 224, 231, 228]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:29.055837437Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1740,"":"downstairs","":"downstairs","session_id":"6845a207-04ab-402f-9fbf-b92e72c5c6c9"} {"msg":"[2]R generation[0..12]: [139, 139, 135, 126, 139, 137, 136, 139, 135, 136, 139, 137]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:29.055851219Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1740,"":"downstairs","":"downstairs","session_id":"6845a207-04ab-402f-9fbf-b92e72c5c6c9"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:29.055867741Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1740,"":"downstairs","":"downstairs","session_id":"6845a207-04ab-402f-9fbf-b92e72c5c6c9"} {"msg":"Max found gen is 140","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:29.055881072Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1740,"":"downstairs","":"downstairs","session_id":"6845a207-04ab-402f-9fbf-b92e72c5c6c9"} {"msg":"Generation requested: 140 >= found:140","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:29.055894244Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1740,"":"downstairs","":"downstairs","session_id":"6845a207-04ab-402f-9fbf-b92e72c5c6c9"} {"msg":"Next flush: 232","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:29.055907426Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1740,"":"downstairs","":"downstairs","session_id":"6845a207-04ab-402f-9fbf-b92e72c5c6c9"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:29.055920348Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1740,"":"downstairs","":"downstairs","session_id":"6845a207-04ab-402f-9fbf-b92e72c5c6c9"} Wait for a query_work_queue command to finish before sending IO {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:29.05594119Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1740,"session_id":"6845a207-04ab-402f-9fbf-b92e72c5c6c9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:29.055953692Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1740,"client":"0","":"downstairs","session_id":"6845a207-04ab-402f-9fbf-b92e72c5c6c9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:29.055967314Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1740,"client":"1","":"downstairs","session_id":"6845a207-04ab-402f-9fbf-b92e72c5c6c9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:29.055980665Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1740,"client":"2","":"downstairs","session_id":"6845a207-04ab-402f-9fbf-b92e72c5c6c9"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:29.055994237Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1740,"session_id":"6845a207-04ab-402f-9fbf-b92e72c5c6c9"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:29.056006588Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1740,"session_id":"6845a207-04ab-402f-9fbf-b92e72c5c6c9"} {"msg":"c93bb129-11cd-4978-b6af-be74705c555f is now active with session: 6845a207-04ab-402f-9fbf-b92e72c5c6c9","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:29.05601947Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1740,"session_id":"6845a207-04ab-402f-9fbf-b92e72c5c6c9"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:29.056031762Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1740,"session_id":"6845a207-04ab-402f-9fbf-b92e72c5c6c9"} {"msg":"The guest has finished waiting for activation with:140","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:29.056044073Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1740} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 117 len 40960 data: 8 9 11 10 13 15 12 11 12 14 02/30 Read block 556 len 8192 03/30 Write block 454 len 40960 data: 10 7 8 7 6 6 4 6 7 7 04/30 Read block 535 len 16384 05/30 Write block 490 len 24576 data: 16 15 12 8 7 10 06/30 Read block 213 len 28672 07/30 Flush 08/30 Write block 557 len 24576 data: 14 17 21 16 16 17 09/30 Read block 459 len 24576 10/30 Write block 91 len 20480 data: 16 14 14 12 14 11/30 Read block 567 len 36864 12/30 Read block 420 len 28672 13/30 Flush 14/30 Flush 15/30 Write block 347 len 20480 data: 11 11 11 11 13 16/30 Write block 394 len 8192 data: 9 11 17/30 Read block 288 len 28672 18/30 Read block 289 len 24576 19/30 Read block 513 len 28672 20/30 Write block 461 len 8192 data: 7 8 21/30 Write block 57 len 24576 data: 9 8 8 7 8 6 22/30 Write block 1 len 16384 data: 5 8 7 9 23/30 Read block 395 len 8192 24/30 Write block 421 len 24576 data: 11 12 12 12 12 14 25/30 Write block 270 len 20480 data: 14 17 15 16 19 26/30 Read block 292 len 28672 27/30 Read block 368 len 16384 28/30 Read block 125 len 32768 29/30 Read block 539 len 16384 30/30 Read block 71 len 32768 ---------------------------------------------------------------- Crucible gen:140 GIO:true work queues: Upstairs:0 downstairs:16 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1014 Acked Write 5 Done Done Done false 1015 Acked Write 2 Done Done Done false 1016 Acked Read 7 Done Done Done false 1017 Acked Read 6 Done Done Done false 1018 Acked Read 7 Done Done Done false 1019 Acked Write 2 Done Done Done false 1020 Acked Write 6 Done Done Done false 1021 Acked Write 4 Done Done Done false 1022 Acked Read 2 Done Done Done false 1023 Acked Write 6 Done Done Done false 1024 Acked Write 5 Done Done Done false 1025 Acked Read 7 Done Done Done false 1026 Acked Read 4 Done Done Done false 1027 Acked Read 8 Done Done Done false 1028 Acked Read 4 Done Done Done false 1029 Acked Read 8 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 16 16 16 48 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1013 1013 1013 Downstairs last five completed: 1013 1012 1011 1010 1009 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:55:29.079786023Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1740} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 139 139 139 231 231 231 T T T 1 020-039 139 139 139 231 231 231 F F F 2 040-059 135 135 135 222 222 222 T T T 3 060-079 126 126 126 209 209 209 T T T 4 080-099 140 140 140 233 233 233 F F F 5 100-119 140 140 140 232 232 232 F F F 6 120-139 140 140 140 232 232 232 F F F 7 140-159 139 139 139 231 231 231 F F F 8 160-179 135 135 135 222 222 222 F F F 9 180-199 136 136 136 224 224 224 F F F 10 200-219 139 139 139 231 231 231 F F F 11 220-239 137 137 137 228 228 228 F F F 12 240-259 138 138 138 230 230 230 F F F 13 260-279 136 136 136 225 225 225 T T T 14 280-299 138 138 138 230 230 230 F F F 15 300-319 136 136 136 224 224 224 F F F 16 320-339 138 138 138 230 230 230 F F F 17 340-359 136 136 136 226 226 226 T T T 18 360-379 139 139 139 231 231 231 F F F 19 380-399 135 135 135 222 222 222 T T T 20 400-419 139 139 139 231 231 231 F F F 21 420-439 130 130 130 215 215 215 T T T 22 440-459 140 140 140 232 232 232 F F F 23 460-479 140 140 140 232 232 232 T T T 24 480-499 140 140 140 232 232 232 F F F 25 500-519 136 136 136 223 223 223 F F F 26 520-539 139 139 139 231 231 231 F F F 27 540-559 140 140 140 233 233 233 F F F 28 560-579 140 140 140 233 233 233 F F F 29 580-599 136 136 136 224 224 224 F F F Max gen: 140, Max flush: 233 A Difference in extent metadata is expected here On loop 70 Verifying data now No change after verify Loop: 70 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 141 141 141 234 234 234 F F F 1 020-039 139 139 139 231 231 231 F F F 2 040-059 141 141 141 234 234 234 F F F 3 060-079 141 141 141 234 234 234 F F F 4 080-099 140 140 140 233 233 233 F F F 5 100-119 140 140 140 232 232 232 F F F 6 120-139 140 140 140 232 232 232 F F F 7 140-159 139 139 139 231 231 231 F F F 8 160-179 135 135 135 222 222 222 F F F 9 180-199 136 136 136 224 224 224 F F F 10 200-219 139 139 139 231 231 231 F F F 11 220-239 137 137 137 228 228 228 F F F 12 240-259 138 138 138 230 230 230 F F F 13 260-279 141 141 141 234 234 234 F F F 14 280-299 138 138 138 230 230 230 F F F 15 300-319 136 136 136 224 224 224 F F F 16 320-339 138 138 138 230 230 230 F F F 17 340-359 141 141 141 234 234 234 F F F 18 360-379 139 139 139 231 231 231 F F F 19 380-399 141 141 141 234 234 234 F F F 20 400-419 139 139 139 231 231 231 F F F 21 420-439 141 141 141 234 234 234 F F F 22 440-459 140 140 140 232 232 232 F F F 23 460-479 141 141 141 234 234 234 F F F 24 480-499 140 140 140 232 232 232 F F F 25 500-519 136 136 136 223 223 223 F F F 26 520-539 139 139 139 231 231 231 F F F 27 540-559 140 140 140 233 233 233 F F F 28 560-579 140 140 140 233 233 233 F F F 29 580-599 136 136 136 224 224 224 F F F Max gen: 141, Max flush: 234 Begin loop 71 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:33.448866066Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1748,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:33.492743729Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1748,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:33.493424877Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1748} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"3818f0fc76fe787c6cc412af30fd59b930203885\",\n git_commit_timestamp: \"2024-11-01T16:08:30.000000000Z\",\n git_branch: \"renovate/hyper-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:33.493475494Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1748} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:33.493493876Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1748} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:33.493511708Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1748} {"msg":"Crucible 6fe0dfd9-f1a4-46fd-8359-49458a80ff25 has session id: 7077826c-ed3e-49ac-9f1e-f08b2a81df56","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:33.49391068Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1748,"session_id":"7077826c-ed3e-49ac-9f1e-f08b2a81df56"} {"msg":"Upstairs opts: Upstairs UUID: 6fe0dfd9-f1a4-46fd-8359-49458a80ff25, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:33.493945914Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1748,"session_id":"7077826c-ed3e-49ac-9f1e-f08b2a81df56"} {"msg":"Crucible stats registered with UUID: 6fe0dfd9-f1a4-46fd-8359-49458a80ff25","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:33.493972988Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1748,"session_id":"7077826c-ed3e-49ac-9f1e-f08b2a81df56"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:33.52503301Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1748,"task":"crutest"} {"msg":"The guest has requested activation with gen:142","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:33.525066624Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1748} {"msg":"6fe0dfd9-f1a4-46fd-8359-49458a80ff25 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:33.525117901Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1748,"session_id":"7077826c-ed3e-49ac-9f1e-f08b2a81df56"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:33.525138523Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1748,"client":"0","":"downstairs","session_id":"7077826c-ed3e-49ac-9f1e-f08b2a81df56"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:33.525167437Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1748,"client":"0","":"downstairs","session_id":"7077826c-ed3e-49ac-9f1e-f08b2a81df56"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:33.52518544Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1748,"client":"1","":"downstairs","session_id":"7077826c-ed3e-49ac-9f1e-f08b2a81df56"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:33.525202902Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1748,"client":"1","":"downstairs","session_id":"7077826c-ed3e-49ac-9f1e-f08b2a81df56"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:33.525232406Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1748,"client":"2","":"downstairs","session_id":"7077826c-ed3e-49ac-9f1e-f08b2a81df56"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:33.525378104Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1748,"client":"2","":"downstairs","session_id":"7077826c-ed3e-49ac-9f1e-f08b2a81df56"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:33.52542133Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1748,"":"io task","client":"2","":"downstairs","session_id":"7077826c-ed3e-49ac-9f1e-f08b2a81df56"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:33.525520913Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1748,"":"io task","client":"0","":"downstairs","session_id":"7077826c-ed3e-49ac-9f1e-f08b2a81df56"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:33.525560248Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1748,"":"io task","client":"1","":"downstairs","session_id":"7077826c-ed3e-49ac-9f1e-f08b2a81df56"} {"msg":"ds_connection connected from Ok(127.0.0.1:47509)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:33.525588021Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1748,"":"io task","client":"2","":"downstairs","session_id":"7077826c-ed3e-49ac-9f1e-f08b2a81df56"} {"msg":"ds_connection connected from Ok(127.0.0.1:35957)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:33.525603143Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1748,"":"io task","client":"1","":"downstairs","session_id":"7077826c-ed3e-49ac-9f1e-f08b2a81df56"} {"msg":"ds_connection connected from Ok(127.0.0.1:45942)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:33.525617185Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1748,"":"io task","client":"0","":"downstairs","session_id":"7077826c-ed3e-49ac-9f1e-f08b2a81df56"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:33.526208851Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1748,"client":"0","":"downstairs","session_id":"7077826c-ed3e-49ac-9f1e-f08b2a81df56"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:33.526247336Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1748,"client":"0","":"downstairs","session_id":"7077826c-ed3e-49ac-9f1e-f08b2a81df56"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:33.52627547Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1748,"client":"2","":"downstairs","session_id":"7077826c-ed3e-49ac-9f1e-f08b2a81df56"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:33.526304624Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1748,"client":"2","":"downstairs","session_id":"7077826c-ed3e-49ac-9f1e-f08b2a81df56"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:33.526333728Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1748,"client":"1","":"downstairs","session_id":"7077826c-ed3e-49ac-9f1e-f08b2a81df56"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:33.52635772Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1748,"client":"1","":"downstairs","session_id":"7077826c-ed3e-49ac-9f1e-f08b2a81df56"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:33.526866826Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1748,"client":"2","":"downstairs","session_id":"7077826c-ed3e-49ac-9f1e-f08b2a81df56"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:33.526994613Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1748,"client":"1","":"downstairs","session_id":"7077826c-ed3e-49ac-9f1e-f08b2a81df56"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:33.527024597Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1748,"client":"0","":"downstairs","session_id":"7077826c-ed3e-49ac-9f1e-f08b2a81df56"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:33.527371051Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1748,"client":"2","":"downstairs","session_id":"7077826c-ed3e-49ac-9f1e-f08b2a81df56"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:33.527409376Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1748,"client":"2","":"downstairs","session_id":"7077826c-ed3e-49ac-9f1e-f08b2a81df56"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:33.52743661Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1748,"session_id":"7077826c-ed3e-49ac-9f1e-f08b2a81df56"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:33.527522691Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1748,"client":"0","":"downstairs","session_id":"7077826c-ed3e-49ac-9f1e-f08b2a81df56"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:33.527546304Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1748,"client":"0","":"downstairs","session_id":"7077826c-ed3e-49ac-9f1e-f08b2a81df56"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:33.527560535Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1748,"session_id":"7077826c-ed3e-49ac-9f1e-f08b2a81df56"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:33.527584229Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1748,"client":"1","":"downstairs","session_id":"7077826c-ed3e-49ac-9f1e-f08b2a81df56"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:33.52759828Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1748,"client":"1","":"downstairs","session_id":"7077826c-ed3e-49ac-9f1e-f08b2a81df56"} {"msg":"[0]R flush_numbers[0..12]: [234, 231, 234, 234, 233, 232, 232, 231, 222, 224, 231, 228]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:33.527617143Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1748,"":"downstairs","":"downstairs","session_id":"7077826c-ed3e-49ac-9f1e-f08b2a81df56"} {"msg":"[0]R generation[0..12]: [141, 139, 141, 141, 140, 140, 140, 139, 135, 136, 139, 137]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:33.527631325Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1748,"":"downstairs","":"downstairs","session_id":"7077826c-ed3e-49ac-9f1e-f08b2a81df56"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:33.527645357Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1748,"":"downstairs","":"downstairs","session_id":"7077826c-ed3e-49ac-9f1e-f08b2a81df56"} {"msg":"[1]R flush_numbers[0..12]: [234, 231, 234, 234, 233, 232, 232, 231, 222, 224, 231, 228]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:33.527658838Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1748,"":"downstairs","":"downstairs","session_id":"7077826c-ed3e-49ac-9f1e-f08b2a81df56"} {"msg":"[1]R generation[0..12]: [141, 139, 141, 141, 140, 140, 140, 139, 135, 136, 139, 137]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:33.52767231Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1748,"":"downstairs","":"downstairs","session_id":"7077826c-ed3e-49ac-9f1e-f08b2a81df56"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:33.527686102Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1748,"":"downstairs","":"downstairs","session_id":"7077826c-ed3e-49ac-9f1e-f08b2a81df56"} {"msg":"[2]R flush_numbers[0..12]: [234, 231, 234, 234, 233, 232, 232, 231, 222, 224, 231, 228]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:33.527699764Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1748,"":"downstairs","":"downstairs","session_id":"7077826c-ed3e-49ac-9f1e-f08b2a81df56"} {"msg":"[2]R generation[0..12]: [141, 139, 141, 141, 140, 140, 140, 139, 135, 136, 139, 137]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:33.527713255Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1748,"":"downstairs","":"downstairs","session_id":"7077826c-ed3e-49ac-9f1e-f08b2a81df56"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:33.527726877Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1748,"":"downstairs","":"downstairs","session_id":"7077826c-ed3e-49ac-9f1e-f08b2a81df56"} {"msg":"Max found gen is 142","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:33.52775242Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1748,"":"downstairs","":"downstairs","session_id":"7077826c-ed3e-49ac-9f1e-f08b2a81df56"} {"msg":"Generation requested: 142 >= found:142","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:33.527769132Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1748,"":"downstairs","":"downstairs","session_id":"7077826c-ed3e-49ac-9f1e-f08b2a81df56"} {"msg":"Next flush: 235","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:33.527782424Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1748,"":"downstairs","":"downstairs","session_id":"7077826c-ed3e-49ac-9f1e-f08b2a81df56"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:33.527795356Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1748,"":"downstairs","":"downstairs","session_id":"7077826c-ed3e-49ac-9f1e-f08b2a81df56"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:33.527808387Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1748,"session_id":"7077826c-ed3e-49ac-9f1e-f08b2a81df56"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:33.527820379Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1748,"client":"0","":"downstairs","session_id":"7077826c-ed3e-49ac-9f1e-f08b2a81df56"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-01T16:55:33.527833351Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1748,"client":"1","":"downstairs","session_id":"7077826c-ed3e-49ac-9f1e-f08b2a81df56"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:33.527857584Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1748,"client":"2","":"downstairs","session_id":"7077826c-ed3e-49ac-9f1e-f08b2a81df56"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:33.527870806Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1748,"session_id":"7077826c-ed3e-49ac-9f1e-f08b2a81df56"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:33.527883237Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1748,"session_id":"7077826c-ed3e-49ac-9f1e-f08b2a81df56"} {"msg":"6fe0dfd9-f1a4-46fd-8359-49458a80ff25 is now active with session: 7077826c-ed3e-49ac-9f1e-f08b2a81df56","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:33.527895829Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1748,"session_id":"7077826c-ed3e-49ac-9f1e-f08b2a81df56"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:33.52790777Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1748,"session_id":"7077826c-ed3e-49ac-9f1e-f08b2a81df56"} {"msg":"The guest has finished waiting for activation with:142","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:33.527920042Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1748} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 52 len 20480 data: 11 10 9 8 9 02/30 Write block 225 len 12288 data: 14 11 9 03/30 Read block 530 len 40960 04/30 Write block 382 len 12288 data: 13 13 12 05/30 Read block 198 len 8192 06/30 Write block 9 len 12288 data: 13 12 13 07/30 Write block 586 len 12288 data: 14 17 13 08/30 Write block 477 len 12288 data: 14 17 15 09/30 Write block 205 len 24576 data: 11 12 15 13 11 13 10/30 Write block 564 len 16384 data: 18 11 11 12 11/30 Write block 37 len 20480 data: 14 13 13 11 12 12/30 Write block 521 len 36864 data: 8 9 8 8 9 9 11 13 14 13/30 Write block 357 len 12288 data: 12 11 8 14/30 Read block 379 len 28672 15/30 Flush 16/30 Write block 431 len 28672 data: 11 6 6 6 6 7 6 17/30 Write block 28 len 4096 data: 16 18/30 Read block 540 len 28672 19/30 Read block 105 len 32768 20/30 Write block 36 len 40960 data: 14 15 14 14 12 13 14 17 17 14 21/30 Read block 78 len 28672 22/30 Write block 7 len 4096 data: 12 23/30 Write block 333 len 32768 data: 8 8 10 8 9 9 7 8 24/30 Write block 478 len 24576 data: 18 16 15 12 11 12 25/30 Write block 523 len 36864 data: 9 9 10 10 12 14 15 16 14 26/30 Read block 152 len 8192 27/30 Read block 403 len 12288 28/30 Read block 579 len 24576 29/30 Write block 151 len 40960 data: 14 16 12 13 13 13 11 10 9 11 30/30 Read block 218 len 12288 ---------------------------------------------------------------- Crucible gen:142 GIO:true work queues: Upstairs:0 downstairs:15 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1015 Acked Write 7 Done Done Done false 1016 Acked Write 1 Done Done Done false 1017 Acked Read 7 Done Done Done false 1018 Acked Read 8 Done Done Done false 1019 Acked Write 10 Done Done Done false 1020 Acked Read 7 Done Done Done false 1021 Acked Write 1 Done Done Done false 1022 Acked Write 8 Done Done Done false 1023 Acked Write 6 Done Done Done false 1024 Acked Write 9 Done Done Done false 1025 Acked Read 2 Done Done Done false 1026 Acked Read 3 Done Done Done false 1027 Acked Read 6 Done Done Done false 1028 Acked Write 10 Done Done Sent false 1029 Acked Read 3 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 2 3 Done 15 14 13 42 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1014 1014 1014 Downstairs last five completed: 1014 1013 1012 1011 1010 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:55:33.549317619Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1748} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 142 142 142 235 235 235 T T T 1 020-039 142 142 142 235 235 235 T T T 2 040-059 142 142 142 235 235 235 T T T 3 060-079 141 141 141 234 234 234 F F F 4 080-099 140 140 140 233 233 233 F F F 5 100-119 140 140 140 232 232 232 F F F 6 120-139 140 140 140 232 232 232 F F F 7 140-159 139 139 139 231 231 231 T T T 8 160-179 135 135 135 222 222 222 T T T 9 180-199 136 136 136 224 224 224 F F F 10 200-219 142 142 142 235 235 235 F F F 11 220-239 142 142 142 235 235 235 F F F 12 240-259 138 138 138 230 230 230 F F F 13 260-279 141 141 141 234 234 234 F F F 14 280-299 138 138 138 230 230 230 F F F 15 300-319 136 136 136 224 224 224 F F F 16 320-339 138 138 138 230 230 230 T T T 17 340-359 142 142 142 235 235 235 T T T 18 360-379 139 139 139 231 231 231 F F F 19 380-399 142 142 142 235 235 235 F F F 20 400-419 139 139 139 231 231 231 F F F 21 420-439 141 141 141 234 234 234 T T T 22 440-459 140 140 140 232 232 232 F F F 23 460-479 142 142 142 235 235 235 T T T 24 480-499 140 140 140 232 232 232 T T T 25 500-519 136 136 136 223 223 223 F F F 26 520-539 142 142 142 235 235 235 T T T 27 540-559 140 140 140 233 233 233 F F F 28 560-579 142 142 142 235 235 235 F F F 29 580-599 142 142 142 235 235 235 F F F Max gen: 142, Max flush: 235 A Difference in extent metadata is expected here On loop 71 Verifying data now No change after verify Loop: 71 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 143 143 143 236 236 236 F F F 1 020-039 143 143 143 236 236 236 F F F 2 040-059 143 143 143 236 236 236 F F F 3 060-079 141 141 141 234 234 234 F F F 4 080-099 140 140 140 233 233 233 F F F 5 100-119 140 140 140 232 232 232 F F F 6 120-139 140 140 140 232 232 232 F F F 7 140-159 143 143 143 236 236 236 F F F 8 160-179 143 143 143 236 236 236 F F F 9 180-199 136 136 136 224 224 224 F F F 10 200-219 142 142 142 235 235 235 F F F 11 220-239 142 142 142 235 235 235 F F F 12 240-259 138 138 138 230 230 230 F F F 13 260-279 141 141 141 234 234 234 F F F 14 280-299 138 138 138 230 230 230 F F F 15 300-319 136 136 136 224 224 224 F F F 16 320-339 143 143 143 236 236 236 F F F 17 340-359 143 143 143 236 236 236 F F F 18 360-379 139 139 139 231 231 231 F F F 19 380-399 142 142 142 235 235 235 F F F 20 400-419 139 139 139 231 231 231 F F F 21 420-439 143 143 143 236 236 236 F F F 22 440-459 140 140 140 232 232 232 F F F 23 460-479 143 143 143 236 236 236 F F F 24 480-499 143 143 143 236 236 236 F F F 25 500-519 136 136 136 223 223 223 F F F 26 520-539 143 143 143 236 236 236 F F F 27 540-559 140 140 140 233 233 233 F F F 28 560-579 142 142 142 235 235 235 F F F 29 580-599 142 142 142 235 235 235 F F F Max gen: 143, Max flush: 236 Begin loop 72 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:37.932177668Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1756,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:37.975820871Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1756,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:37.976460143Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1756} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"3818f0fc76fe787c6cc412af30fd59b930203885\",\n git_commit_timestamp: \"2024-11-01T16:08:30.000000000Z\",\n git_branch: \"renovate/hyper-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:37.976501889Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1756} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:37.976516831Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1756} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:37.976531452Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1756} {"msg":"Crucible 0f2d9549-a81a-400b-b9a4-3ac965da2d6b has session id: f77d1c85-21d4-453d-9daf-78309b6d5fab","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:37.976891779Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1756,"session_id":"f77d1c85-21d4-453d-9daf-78309b6d5fab"} {"msg":"Upstairs opts: Upstairs UUID: 0f2d9549-a81a-400b-b9a4-3ac965da2d6b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:37.976934444Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1756,"session_id":"f77d1c85-21d4-453d-9daf-78309b6d5fab"} {"msg":"Crucible stats registered with UUID: 0f2d9549-a81a-400b-b9a4-3ac965da2d6b","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:37.976959878Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1756,"session_id":"f77d1c85-21d4-453d-9daf-78309b6d5fab"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:38.007686777Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1756,"task":"crutest"} {"msg":"The guest has requested activation with gen:144","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:38.007731232Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1756} {"msg":"0f2d9549-a81a-400b-b9a4-3ac965da2d6b active request set","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:38.007803122Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1756,"session_id":"f77d1c85-21d4-453d-9daf-78309b6d5fab"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:38.007837216Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1756,"client":"0","":"downstairs","session_id":"f77d1c85-21d4-453d-9daf-78309b6d5fab"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:38.00786844Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1756,"client":"0","":"downstairs","session_id":"f77d1c85-21d4-453d-9daf-78309b6d5fab"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:38.007886823Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1756,"client":"1","":"downstairs","session_id":"f77d1c85-21d4-453d-9daf-78309b6d5fab"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:38.007904165Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1756,"client":"1","":"downstairs","session_id":"f77d1c85-21d4-453d-9daf-78309b6d5fab"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:38.007921017Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1756,"client":"2","":"downstairs","session_id":"f77d1c85-21d4-453d-9daf-78309b6d5fab"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:38.0079471Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1756,"client":"2","":"downstairs","session_id":"f77d1c85-21d4-453d-9daf-78309b6d5fab"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:38.007981105Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1756,"":"io task","client":"2","":"downstairs","session_id":"f77d1c85-21d4-453d-9daf-78309b6d5fab"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:38.008014039Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1756,"":"io task","client":"0","":"downstairs","session_id":"f77d1c85-21d4-453d-9daf-78309b6d5fab"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:38.008049804Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1756,"":"io task","client":"1","":"downstairs","session_id":"f77d1c85-21d4-453d-9daf-78309b6d5fab"} {"msg":"ds_connection connected from Ok(127.0.0.1:36865)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:38.00825019Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1756,"":"io task","client":"2","":"downstairs","session_id":"f77d1c85-21d4-453d-9daf-78309b6d5fab"} {"msg":"ds_connection connected from Ok(127.0.0.1:49206)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:38.008290055Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1756,"":"io task","client":"1","":"downstairs","session_id":"f77d1c85-21d4-453d-9daf-78309b6d5fab"} {"msg":"ds_connection connected from Ok(127.0.0.1:52102)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:38.008361414Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1756,"":"io task","client":"0","":"downstairs","session_id":"f77d1c85-21d4-453d-9daf-78309b6d5fab"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:38.008758995Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1756,"client":"0","":"downstairs","session_id":"f77d1c85-21d4-453d-9daf-78309b6d5fab"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:38.008785368Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1756,"client":"0","":"downstairs","session_id":"f77d1c85-21d4-453d-9daf-78309b6d5fab"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:38.00880066Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1756,"client":"1","":"downstairs","session_id":"f77d1c85-21d4-453d-9daf-78309b6d5fab"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:38.008814692Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1756,"client":"1","":"downstairs","session_id":"f77d1c85-21d4-453d-9daf-78309b6d5fab"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:38.008830684Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1756,"client":"2","":"downstairs","session_id":"f77d1c85-21d4-453d-9daf-78309b6d5fab"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:38.008855677Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1756,"client":"2","":"downstairs","session_id":"f77d1c85-21d4-453d-9daf-78309b6d5fab"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:38.009467006Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1756,"client":"0","":"downstairs","session_id":"f77d1c85-21d4-453d-9daf-78309b6d5fab"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:38.009535185Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1756,"client":"2","":"downstairs","session_id":"f77d1c85-21d4-453d-9daf-78309b6d5fab"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:38.009556378Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1756,"client":"1","":"downstairs","session_id":"f77d1c85-21d4-453d-9daf-78309b6d5fab"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:38.010068514Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1756,"client":"0","":"downstairs","session_id":"f77d1c85-21d4-453d-9daf-78309b6d5fab"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:38.010097017Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1756,"client":"0","":"downstairs","session_id":"f77d1c85-21d4-453d-9daf-78309b6d5fab"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:38.010111949Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1756,"session_id":"f77d1c85-21d4-453d-9daf-78309b6d5fab"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:38.010165566Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1756,"client":"1","":"downstairs","session_id":"f77d1c85-21d4-453d-9daf-78309b6d5fab"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:38.01019335Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1756,"client":"1","":"downstairs","session_id":"f77d1c85-21d4-453d-9daf-78309b6d5fab"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:38.010212112Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1756,"session_id":"f77d1c85-21d4-453d-9daf-78309b6d5fab"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:38.010225664Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1756,"client":"2","":"downstairs","session_id":"f77d1c85-21d4-453d-9daf-78309b6d5fab"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:38.010239426Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1756,"client":"2","":"downstairs","session_id":"f77d1c85-21d4-453d-9daf-78309b6d5fab"} {"msg":"[0]R flush_numbers[0..12]: [236, 236, 236, 234, 233, 232, 232, 236, 236, 224, 235, 235]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:38.010253557Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1756,"":"downstairs","":"downstairs","session_id":"f77d1c85-21d4-453d-9daf-78309b6d5fab"} {"msg":"[0]R generation[0..12]: [143, 143, 143, 141, 140, 140, 140, 143, 143, 136, 142, 142]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:38.01026805Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1756,"":"downstairs","":"downstairs","session_id":"f77d1c85-21d4-453d-9daf-78309b6d5fab"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:38.010282471Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1756,"":"downstairs","":"downstairs","session_id":"f77d1c85-21d4-453d-9daf-78309b6d5fab"} {"msg":"[1]R flush_numbers[0..12]: [236, 236, 236, 234, 233, 232, 232, 236, 236, 224, 235, 235]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:38.010296363Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1756,"":"downstairs","":"downstairs","session_id":"f77d1c85-21d4-453d-9daf-78309b6d5fab"} {"msg":"[1]R generation[0..12]: [143, 143, 143, 141, 140, 140, 140, 143, 143, 136, 142, 142]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:38.010310195Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1756,"":"downstairs","":"downstairs","session_id":"f77d1c85-21d4-453d-9daf-78309b6d5fab"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:38.010324167Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1756,"":"downstairs","":"downstairs","session_id":"f77d1c85-21d4-453d-9daf-78309b6d5fab"} {"msg":"[2]R flush_numbers[0..12]: [236, 236, 236, 234, 233, 232, 232, 236, 236, 224, 235, 235]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:38.010337848Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1756,"":"downstairs","":"downstairs","session_id":"f77d1c85-21d4-453d-9daf-78309b6d5fab"} {"msg":"[2]R generation[0..12]: [143, 143, 143, 141, 140, 140, 140, 143, 143, 136, 142, 142]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:38.01035152Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1756,"":"downstairs","":"downstairs","session_id":"f77d1c85-21d4-453d-9daf-78309b6d5fab"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:38.010365552Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1756,"":"downstairs","":"downstairs","session_id":"f77d1c85-21d4-453d-9daf-78309b6d5fab"} {"msg":"Max found gen is 144","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:38.010378584Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1756,"":"downstairs","":"downstairs","session_id":"f77d1c85-21d4-453d-9daf-78309b6d5fab"} {"msg":"Generation requested: 144 >= found:144","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:38.010391825Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1756,"":"downstairs","":"downstairs","session_id":"f77d1c85-21d4-453d-9daf-78309b6d5fab"} {"msg":"Next flush: 237","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-01T16:55:38.010406637Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1756,"":"downstairs","":"downstairs","session_id":"f77d1c85-21d4-453d-9daf-78309b6d5fab"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:38.01042315Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1756,"":"downstairs","":"downstairs","session_id":"f77d1c85-21d4-453d-9daf-78309b6d5fab"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:38.010436311Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1756,"session_id":"f77d1c85-21d4-453d-9daf-78309b6d5fab"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:38.010448493Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1756,"client":"0","":"downstairs","session_id":"f77d1c85-21d4-453d-9daf-78309b6d5fab"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:38.010461804Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1756,"client":"1","":"downstairs","session_id":"f77d1c85-21d4-453d-9daf-78309b6d5fab"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:38.010474866Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1756,"client":"2","":"downstairs","session_id":"f77d1c85-21d4-453d-9daf-78309b6d5fab"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:38.010488168Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1756,"session_id":"f77d1c85-21d4-453d-9daf-78309b6d5fab"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:38.010500479Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1756,"session_id":"f77d1c85-21d4-453d-9daf-78309b6d5fab"} {"msg":"0f2d9549-a81a-400b-b9a4-3ac965da2d6b is now active with session: f77d1c85-21d4-453d-9daf-78309b6d5fab","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:38.010513291Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1756,"session_id":"f77d1c85-21d4-453d-9daf-78309b6d5fab"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:38.010525423Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1756,"session_id":"f77d1c85-21d4-453d-9daf-78309b6d5fab"} {"msg":"The guest has finished waiting for activation with:144","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:38.010537624Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1756} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 24 len 36864 data: 13 14 12 15 17 14 11 13 14 02/30 Read block 179 len 16384 03/30 Write block 388 len 16384 data: 9 11 9 9 04/30 Read block 444 len 40960 05/30 Flush 06/30 Write block 91 len 28672 data: 17 15 15 13 15 17 18 07/30 Read block 471 len 4096 08/30 Flush 09/30 Write block 156 len 36864 data: 14 12 11 10 12 11 9 11 10 10/30 Read block 529 len 16384 11/30 Flush 12/30 Write block 352 len 40960 data: 14 14 10 10 9 13 12 9 8 8 13/30 Read block 433 len 8192 14/30 Read block 335 len 24576 15/30 Read block 347 len 36864 16/30 Write block 327 len 4096 data: 5 17/30 Write block 187 len 28672 data: 12 12 11 13 13 14 16 18/30 Read block 195 len 36864 19/30 Write block 213 len 40960 data: 13 13 13 16 17 17 15 14 14 14 20/30 Read block 367 len 28672 21/30 Read block 169 len 28672 22/30 Read block 245 len 20480 23/30 Read block 373 len 40960 24/30 Write block 85 len 36864 data: 14 14 16 16 19 20 18 16 16 25/30 Read block 334 len 8192 26/30 Write block 573 len 32768 data: 7 7 10 9 10 12 12 9 27/30 Read block 473 len 24576 28/30 Read block 394 len 24576 29/30 Read block 142 len 12288 30/30 Write block 363 len 8192 data: 5 6 ---------------------------------------------------------------- Crucible gen:144 GIO:true work queues: Upstairs:0 downstairs:19 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1011 Acked Write 10 Done Done Done false 1012 Acked Read 2 Done Done Done false 1013 Acked Read 6 Done Done Done false 1014 Acked Read 9 Done Done Done false 1015 Acked Write 1 Done Done Done false 1016 Acked Write 7 Done Done Done false 1017 Acked Read 9 Done Done Done false 1018 Acked Write 10 Done Done Done false 1019 Acked Read 7 Done Done Done false 1020 Acked Read 7 Done Done Done false 1021 Acked Read 5 Done Done Done false 1022 Acked Read 10 Done Done Done false 1023 Acked Write 9 Done Done Done false 1024 Acked Read 2 Done Done Done false 1025 Acked Write 8 Done Done Done false 1026 Acked Read 6 Done Done Done false 1027 Acked Read 6 Done Done Done false 1028 Acked Read 3 Done Done Done false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 18 18 18 54 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1010 1010 1010 Downstairs last five completed: 1010 1009 1008 1007 1006 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:55:38.037373102Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1756} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 143 143 143 236 236 236 F F F 1 020-039 144 144 144 237 237 237 F F F 2 040-059 143 143 143 236 236 236 F F F 3 060-079 141 141 141 234 234 234 F F F 4 080-099 144 144 144 238 238 238 T T T 5 100-119 140 140 140 232 232 232 F F F 6 120-139 140 140 140 232 232 232 F F F 7 140-159 144 144 144 239 239 239 F F F 8 160-179 144 144 144 239 239 239 F F F 9 180-199 136 136 136 224 224 224 T T T 10 200-219 142 142 142 235 235 235 T T T 11 220-239 142 142 142 235 235 235 T T T 12 240-259 138 138 138 230 230 230 F F F 13 260-279 141 141 141 234 234 234 F F F 14 280-299 138 138 138 230 230 230 F F F 15 300-319 136 136 136 224 224 224 F F F 16 320-339 143 143 143 236 236 236 T T T 17 340-359 143 143 143 236 236 236 T T T 18 360-379 139 139 139 231 231 231 T T T 19 380-399 144 144 144 237 237 237 F F F 20 400-419 139 139 139 231 231 231 F F F 21 420-439 143 143 143 236 236 236 F F F 22 440-459 140 140 140 232 232 232 F F F 23 460-479 143 143 143 236 236 236 F F F 24 480-499 143 143 143 236 236 236 F F F 25 500-519 136 136 136 223 223 223 F F F 26 520-539 143 143 143 236 236 236 F F F 27 540-559 140 140 140 233 233 233 F F F 28 560-579 142 142 142 235 235 235 T T T 29 580-599 142 142 142 235 235 235 T T T Max gen: 144, Max flush: 239 A Difference in extent metadata is expected here On loop 72 Verifying data now No change after verify Loop: 72 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 143 143 143 236 236 236 F F F 1 020-039 144 144 144 237 237 237 F F F 2 040-059 143 143 143 236 236 236 F F F 3 060-079 141 141 141 234 234 234 F F F 4 080-099 145 145 145 240 240 240 F F F 5 100-119 140 140 140 232 232 232 F F F 6 120-139 140 140 140 232 232 232 F F F 7 140-159 144 144 144 239 239 239 F F F 8 160-179 144 144 144 239 239 239 F F F 9 180-199 145 145 145 240 240 240 F F F 10 200-219 145 145 145 240 240 240 F F F 11 220-239 145 145 145 240 240 240 F F F 12 240-259 138 138 138 230 230 230 F F F 13 260-279 141 141 141 234 234 234 F F F 14 280-299 138 138 138 230 230 230 F F F 15 300-319 136 136 136 224 224 224 F F F 16 320-339 145 145 145 240 240 240 F F F 17 340-359 145 145 145 240 240 240 F F F 18 360-379 145 145 145 240 240 240 F F F 19 380-399 144 144 144 237 237 237 F F F 20 400-419 139 139 139 231 231 231 F F F 21 420-439 143 143 143 236 236 236 F F F 22 440-459 140 140 140 232 232 232 F F F 23 460-479 143 143 143 236 236 236 F F F 24 480-499 143 143 143 236 236 236 F F F 25 500-519 136 136 136 223 223 223 F F F 26 520-539 143 143 143 236 236 236 F F F 27 540-559 140 140 140 233 233 233 F F F 28 560-579 145 145 145 240 240 240 F F F 29 580-599 145 145 145 240 240 240 F F F Max gen: 145, Max flush: 240 Begin loop 73 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:42.404725353Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1764,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:42.449255241Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1764,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:42.449962902Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1764} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"3818f0fc76fe787c6cc412af30fd59b930203885\",\n git_commit_timestamp: \"2024-11-01T16:08:30.000000000Z\",\n git_branch: \"renovate/hyper-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:42.45001906Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1764} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:42.450051454Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1764} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:42.450085428Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1764} {"msg":"Crucible f04d5d4b-63d0-4958-b2ad-ae7f252dbe0c has session id: 7d1f7a31-32ba-470c-bd16-a37cab6d93e0","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:42.450484369Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1764,"session_id":"7d1f7a31-32ba-470c-bd16-a37cab6d93e0"} {"msg":"Upstairs opts: Upstairs UUID: f04d5d4b-63d0-4958-b2ad-ae7f252dbe0c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:42.450525765Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1764,"session_id":"7d1f7a31-32ba-470c-bd16-a37cab6d93e0"} {"msg":"Crucible stats registered with UUID: f04d5d4b-63d0-4958-b2ad-ae7f252dbe0c","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:42.450553978Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1764,"session_id":"7d1f7a31-32ba-470c-bd16-a37cab6d93e0"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:42.481107985Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1764,"task":"crutest"} {"msg":"The guest has requested activation with gen:146","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:42.481140399Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1764} {"msg":"f04d5d4b-63d0-4958-b2ad-ae7f252dbe0c active request set","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:42.481189726Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1764,"session_id":"7d1f7a31-32ba-470c-bd16-a37cab6d93e0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:42.481210508Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1764,"client":"0","":"downstairs","session_id":"7d1f7a31-32ba-470c-bd16-a37cab6d93e0"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:42.481238392Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1764,"client":"0","":"downstairs","session_id":"7d1f7a31-32ba-470c-bd16-a37cab6d93e0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:42.481255894Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1764,"client":"1","":"downstairs","session_id":"7d1f7a31-32ba-470c-bd16-a37cab6d93e0"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:42.481276107Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1764,"client":"1","":"downstairs","session_id":"7d1f7a31-32ba-470c-bd16-a37cab6d93e0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:42.481304601Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1764,"client":"2","":"downstairs","session_id":"7d1f7a31-32ba-470c-bd16-a37cab6d93e0"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:42.481334444Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1764,"client":"2","":"downstairs","session_id":"7d1f7a31-32ba-470c-bd16-a37cab6d93e0"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:42.481363598Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1764,"":"io task","client":"2","":"downstairs","session_id":"7d1f7a31-32ba-470c-bd16-a37cab6d93e0"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:42.481395272Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1764,"":"io task","client":"0","":"downstairs","session_id":"7d1f7a31-32ba-470c-bd16-a37cab6d93e0"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:42.481426696Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1764,"":"io task","client":"1","":"downstairs","session_id":"7d1f7a31-32ba-470c-bd16-a37cab6d93e0"} {"msg":"ds_connection connected from Ok(127.0.0.1:50234)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:42.481459421Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1764,"":"io task","client":"2","":"downstairs","session_id":"7d1f7a31-32ba-470c-bd16-a37cab6d93e0"} {"msg":"ds_connection connected from Ok(127.0.0.1:35751)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:42.481581366Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1764,"":"io task","client":"0","":"downstairs","session_id":"7d1f7a31-32ba-470c-bd16-a37cab6d93e0"} {"msg":"ds_connection connected from Ok(127.0.0.1:50119)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:42.481632513Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1764,"":"io task","client":"1","":"downstairs","session_id":"7d1f7a31-32ba-470c-bd16-a37cab6d93e0"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:42.482240111Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1764,"client":"2","":"downstairs","session_id":"7d1f7a31-32ba-470c-bd16-a37cab6d93e0"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:42.482268395Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1764,"client":"2","":"downstairs","session_id":"7d1f7a31-32ba-470c-bd16-a37cab6d93e0"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:42.482283637Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1764,"client":"1","":"downstairs","session_id":"7d1f7a31-32ba-470c-bd16-a37cab6d93e0"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:42.482297639Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1764,"client":"1","":"downstairs","session_id":"7d1f7a31-32ba-470c-bd16-a37cab6d93e0"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:42.482322262Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1764,"client":"0","":"downstairs","session_id":"7d1f7a31-32ba-470c-bd16-a37cab6d93e0"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:42.482340944Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1764,"client":"0","":"downstairs","session_id":"7d1f7a31-32ba-470c-bd16-a37cab6d93e0"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:42.482884974Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1764,"client":"2","":"downstairs","session_id":"7d1f7a31-32ba-470c-bd16-a37cab6d93e0"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:42.483050886Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1764,"client":"0","":"downstairs","session_id":"7d1f7a31-32ba-470c-bd16-a37cab6d93e0"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:42.483221838Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1764,"client":"1","":"downstairs","session_id":"7d1f7a31-32ba-470c-bd16-a37cab6d93e0"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:42.483517386Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1764,"client":"2","":"downstairs","session_id":"7d1f7a31-32ba-470c-bd16-a37cab6d93e0"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:42.483541809Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1764,"client":"2","":"downstairs","session_id":"7d1f7a31-32ba-470c-bd16-a37cab6d93e0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:42.483555841Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1764,"session_id":"7d1f7a31-32ba-470c-bd16-a37cab6d93e0"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:42.483593075Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1764,"client":"0","":"downstairs","session_id":"7d1f7a31-32ba-470c-bd16-a37cab6d93e0"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:42.483609248Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1764,"client":"0","":"downstairs","session_id":"7d1f7a31-32ba-470c-bd16-a37cab6d93e0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:42.48362693Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1764,"session_id":"7d1f7a31-32ba-470c-bd16-a37cab6d93e0"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:42.483666525Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1764,"client":"1","":"downstairs","session_id":"7d1f7a31-32ba-470c-bd16-a37cab6d93e0"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:42.483686627Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1764,"client":"1","":"downstairs","session_id":"7d1f7a31-32ba-470c-bd16-a37cab6d93e0"} {"msg":"[0]R flush_numbers[0..12]: [236, 237, 236, 234, 240, 232, 232, 239, 239, 240, 240, 240]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:42.48370408Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1764,"":"downstairs","":"downstairs","session_id":"7d1f7a31-32ba-470c-bd16-a37cab6d93e0"} {"msg":"[0]R generation[0..12]: [143, 144, 143, 141, 145, 140, 140, 144, 144, 145, 145, 145]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:42.483721652Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1764,"":"downstairs","":"downstairs","session_id":"7d1f7a31-32ba-470c-bd16-a37cab6d93e0"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:42.483739194Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1764,"":"downstairs","":"downstairs","session_id":"7d1f7a31-32ba-470c-bd16-a37cab6d93e0"} {"msg":"[1]R flush_numbers[0..12]: [236, 237, 236, 234, 240, 232, 232, 239, 239, 240, 240, 240]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:42.483756516Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1764,"":"downstairs","":"downstairs","session_id":"7d1f7a31-32ba-470c-bd16-a37cab6d93e0"} {"msg":"[1]R generation[0..12]: [143, 144, 143, 141, 145, 140, 140, 144, 144, 145, 145, 145]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:42.483773499Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1764,"":"downstairs","":"downstairs","session_id":"7d1f7a31-32ba-470c-bd16-a37cab6d93e0"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:42.483790691Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1764,"":"downstairs","":"downstairs","session_id":"7d1f7a31-32ba-470c-bd16-a37cab6d93e0"} {"msg":"[2]R flush_numbers[0..12]: [236, 237, 236, 234, 240, 232, 232, 239, 239, 240, 240, 240]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:42.483807793Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1764,"":"downstairs","":"downstairs","session_id":"7d1f7a31-32ba-470c-bd16-a37cab6d93e0"} {"msg":"[2]R generation[0..12]: [143, 144, 143, 141, 145, 140, 140, 144, 144, 145, 145, 145]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:42.483824965Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1764,"":"downstairs","":"downstairs","session_id":"7d1f7a31-32ba-470c-bd16-a37cab6d93e0"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:42.483842298Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1764,"":"downstairs","":"downstairs","session_id":"7d1f7a31-32ba-470c-bd16-a37cab6d93e0"} {"msg":"Max found gen is 146","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:42.4838585Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1764,"":"downstairs","":"downstairs","session_id":"7d1f7a31-32ba-470c-bd16-a37cab6d93e0"} {"msg":"Generation requested: 146 >= found:146","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-01T16:55:42.483875342Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1764,"":"downstairs","":"downstairs","session_id":"7d1f7a31-32ba-470c-bd16-a37cab6d93e0"} {"msg":"Next flush: 241","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:42.483903195Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1764,"":"downstairs","":"downstairs","session_id":"7d1f7a31-32ba-470c-bd16-a37cab6d93e0"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:42.483919978Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1764,"":"downstairs","":"downstairs","session_id":"7d1f7a31-32ba-470c-bd16-a37cab6d93e0"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:42.48393662Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1764,"session_id":"7d1f7a31-32ba-470c-bd16-a37cab6d93e0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:42.483951682Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1764,"client":"0","":"downstairs","session_id":"7d1f7a31-32ba-470c-bd16-a37cab6d93e0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:42.483968024Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1764,"client":"1","":"downstairs","session_id":"7d1f7a31-32ba-470c-bd16-a37cab6d93e0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:42.483984766Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1764,"client":"2","":"downstairs","session_id":"7d1f7a31-32ba-470c-bd16-a37cab6d93e0"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:42.484001488Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1764,"session_id":"7d1f7a31-32ba-470c-bd16-a37cab6d93e0"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:42.48401675Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1764,"session_id":"7d1f7a31-32ba-470c-bd16-a37cab6d93e0"} {"msg":"f04d5d4b-63d0-4958-b2ad-ae7f252dbe0c is now active with session: 7d1f7a31-32ba-470c-bd16-a37cab6d93e0","v":0,"name":"crucible","level":30,"time":"Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 2024-11-01T16:55:42.484032902Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1764,"session_id":"7d1f7a31-32ba-470c-bd16-a37cab6d93e0"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:42.484052245Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1764,"session_id":"7d1f7a31-32ba-470c-bd16-a37cab6d93e0"} {"msg":"The guest has finished waiting for activation with:146","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:42.484067757Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1764} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 447 len 16384 data: 10 10 10 13 02/30 Flush 03/30 Write block 95 len 8192 data: 16 18 04/30 Write block 212 len 24576 data: 14 14 14 14 17 18 05/30 Read block 96 len 8192 06/30 Read block 188 len 36864 07/30 Write block 416 len 12288 data: 8 9 11 08/30 Read block 106 len 16384 09/30 Read block 53 len 12288 10/30 Write block 86 len 4096 data: 15 11/30 Write block 99 len 12288 data: 18 19 18 12/30 Read block 503 len 8192 13/30 Write block 245 len 40960 data: 16 19 21 20 19 21 19 15 13 12 14/30 Write block 16 len 32768 data: 13 13 12 12 10 5 6 9 15/30 Read block 33 len 36864 16/30 Read block 346 len 28672 17/30 Read block 231 len 24576 18/30 Write block 174 len 28672 data: 8 10 10 11 12 11 11 19/30 Write block 137 len 32768 data: 8 13 10 10 10 10 7 9 20/30 Read block 432 len 8192 21/30 Write block 462 len 8192 data: 9 8 22/30 Write block 109 len 8192 data: 12 13 23/30 Write block 76 len 28672 data: 5 5 6 10 10 10 14 24/30 Read block 454 len 28672 25/30 Write block 477 len 12288 data: 15 19 17 26/30 Write block 271 len 4096 data: 18 27/30 Write block 249 len 36864 data: 20 22 20 16 14 13 10 9 9 28/30 Write block 520 len 12288 data: 9 9 10 29/30 Read block 186 len 16384 30/30 Read block 228 len 8192 ---------------------------------------------------------------- Crucible gen:146 GIO:true work queues: Upstairs:0 downstairs:28 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1002 Acked Write 2 Done Done Done false 1003 Acked Write 6 Done Done Done false 1004 Acked Read 2 Done Done Done false 1005 Acked Read 9 Done Done Done false 1006 Acked Write 3 Done Done Done false 1007 Acked Read 4 Done Done Done false 1008 Acked Read 3 Done Done Done false 1009 Acked Write 1 Done Done Done false 1010 Acked Write 3 Done Done Done false 1011 Acked Read 2 Done Done Done false 1012 Acked Write 10 Done Done Done false 1013 Acked Write 8 Done Done Done false 1014 Acked Read 9 Done Done Done false 1015 Acked Read 7 Done Done Done false 1016 Acked Read 6 Done Done Done false 1017 Acked Write 7 Done Done Done false 1018 Acked Write 8 Done Done Done false 1019 Acked Read 2 Done Done Done false 1020 Acked Write 2 Done Done Done false 1021 Acked Write 2 Done Done Done false 1022 Acked Write 7 Done Done Done false 1023 Acked Read 7 Done Done Done false 1024 Acked Write 3 Done Done Done false 1025 Acked Write 1 Done Done Done false 1026 Acked Write 9 Done Done Done false 1027 Acked Write 3 Done Done Done false 1028 Acked Read 4 Done Done Done false 1029 Acked Read 2 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 28 28 27 83 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1001 1001 1001 Downstairs last five completed: 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:55:42.502941899Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1764} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 143 143 143 236 236 236 T T T 1 020-039 144 144 144 237 237 237 T T T 2 040-059 143 143 143 236 236 236 F F F 3 060-079 141 141 141 234 234 234 T T T 4 080-099 145 145 145 240 240 240 T T T 5 100-119 140 140 140 232 232 232 T T T 6 120-139 140 140 140 232 232 232 T T T 7 140-159 144 144 144 239 239 239 T T T 8 160-179 144 144 144 239 239 239 T T T 9 180-199 145 145 145 240 240 240 T T T 10 200-219 145 145 145 240 240 240 T T T 11 220-239 145 145 145 240 240 240 F F F 12 240-259 138 138 138 230 230 230 T T T 13 260-279 141 141 141 234 234 234 T T T 14 280-299 138 138 138 230 230 230 F F F 15 300-319 136 136 136 224 224 224 F F F 16 320-339 145 145 145 240 240 240 F F F 17 340-359 145 145 145 240 240 240 F F F 18 360-379 145 145 145 240 240 240 F F F 19 380-399 144 144 144 237 237 237 F F F 20 400-419 139 139 139 231 231 231 T T T 21 420-439 143 143 143 236 236 236 F F F 22 440-459 146 146 146 241 241 241 F F F 23 460-479 143 143 143 236 236 236 T T T 24 480-499 143 143 143 236 236 236 F F F 25 500-519 136 136 136 223 223 223 F F F 26 520-539 143 143 143 236 236 236 T T T 27 540-559 140 140 140 233 233 233 F F F 28 560-579 145 145 145 240 240 240 F F F 29 580-599 145 145 145 240 240 240 F F F Max gen: 146, Max flush: 241 A Difference in extent metadata is expected here On loop 73 Verifying data now No change after verify Loop: 73 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 147 147 147 242 242 242 F F F 1 020-039 147 147 147 242 242 242 F F F 2 040-059 143 143 143 236 236 236 F F F 3 060-079 147 147 147 242 242 242 F F F 4 080-099 147 147 147 242 242 242 F F F 5 100-119 147 147 147 242 242 242 F F F 6 120-139 147 147 147 242 242 242 F F F 7 140-159 147 147 147 242 242 242 F F F 8 160-179 147 147 147 242 242 242 F F F 9 180-199 147 147 147 242 242 242 F F F 10 200-219 147 147 147 242 242 242 F F F 11 220-239 145 145 145 240 240 240 F F F 12 240-259 147 147 147 242 242 242 F F F 13 260-279 147 147 147 242 242 242 F F F 14 280-299 138 138 138 230 230 230 F F F 15 300-319 136 136 136 224 224 224 F F F 16 320-339 145 145 145 240 240 240 F F F 17 340-359 145 145 145 240 240 240 F F F 18 360-379 145 145 145 240 240 240 F F F 19 380-399 144 144 144 237 237 237 F F F 20 400-419 147 147 147 242 242 242 F F F 21 420-439 143 143 143 236 236 236 F F F 22 440-459 146 146 146 241 241 241 F F F 23 460-479 147 147 147 242 242 242 F F F 24 480-499 143 143 143 236 236 236 F F F 25 500-519 136 136 136 223 223 223 F F F 26 520-539 147 147 147 242 242 242 F F F 27 540-559 140 140 140 233 233 233 F F F 28 560-579 145 145 145 240 240 240 F F F 29 580-599 145 145 145 240 240 240 F F F Max gen: 147, Max flush: 242 Begin loop 74 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:46.945758763Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1772,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:46.990331226Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1772,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:46.990979669Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1772} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"3818f0fc76fe787c6cc412af30fd59b930203885\",\n git_commit_timestamp: \"2024-11-01T16:08:30.000000000Z\",\n git_branch: \"renovate/hyper-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:46.991017784Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1772} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:46.991032036Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1772} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:46.991046028Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1772} {"msg":"Crucible 6e9b9e31-6578-42c6-93a3-3363483e81f5 has session id: c06a09fb-c871-469e-bb7a-64bcbe64fe66","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:46.991532221Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1772,"session_id":"c06a09fb-c871-469e-bb7a-64bcbe64fe66"} {"msg":"Upstairs opts: Upstairs UUID: 6e9b9e31-6578-42c6-93a3-3363483e81f5, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:46.991575706Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1772,"session_id":"c06a09fb-c871-469e-bb7a-64bcbe64fe66"} {"msg":"Crucible stats registered with UUID: 6e9b9e31-6578-42c6-93a3-3363483e81f5","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:46.991594849Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1772,"session_id":"c06a09fb-c871-469e-bb7a-64bcbe64fe66"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:47.022450814Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1772,"task":"crutest"} {"msg":"The guest has requested activation with gen:148","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:47.022486359Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1772} {"msg":"6e9b9e31-6578-42c6-93a3-3363483e81f5 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:47.022547937Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1772,"session_id":"c06a09fb-c871-469e-bb7a-64bcbe64fe66"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:47.02257507Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1772,"client":"0","":"downstairs","session_id":"c06a09fb-c871-469e-bb7a-64bcbe64fe66"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:47.022600354Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1772,"client":"0","":"downstairs","session_id":"c06a09fb-c871-469e-bb7a-64bcbe64fe66"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:47.022615126Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1772,"client":"1","":"downstairs","session_id":"c06a09fb-c871-469e-bb7a-64bcbe64fe66"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:47.022629347Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1772,"client":"1","":"downstairs","session_id":"c06a09fb-c871-469e-bb7a-64bcbe64fe66"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:47.022643059Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1772,"client":"2","":"downstairs","session_id":"c06a09fb-c871-469e-bb7a-64bcbe64fe66"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:47.022667062Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1772,"client":"2","":"downstairs","session_id":"c06a09fb-c871-469e-bb7a-64bcbe64fe66"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:47.022690585Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1772,"":"io task","client":"2","":"downstairs","session_id":"c06a09fb-c871-469e-bb7a-64bcbe64fe66"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:47.022714668Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1772,"":"io task","client":"0","":"downstairs","session_id":"c06a09fb-c871-469e-bb7a-64bcbe64fe66"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:47.022739362Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1772,"":"io task","client":"1","":"downstairs","session_id":"c06a09fb-c871-469e-bb7a-64bcbe64fe66"} {"msg":"ds_connection connected from Ok(127.0.0.1:50882)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:47.022764135Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1772,"":"io task","client":"2","":"downstairs","session_id":"c06a09fb-c871-469e-bb7a-64bcbe64fe66"} {"msg":"ds_connection connected from Ok(127.0.0.1:63055)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:47.022791448Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1772,"":"io task","client":"0","":"downstairs","session_id":"c06a09fb-c871-469e-bb7a-64bcbe64fe66"} {"msg":"ds_connection connected from Ok(127.0.0.1:45403)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:47.023055242Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1772,"":"io task","client":"1","":"downstairs","session_id":"c06a09fb-c871-469e-bb7a-64bcbe64fe66"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:47.023535134Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1772,"client":"2","":"downstairs","session_id":"c06a09fb-c871-469e-bb7a-64bcbe64fe66"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:47.023566238Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1772,"client":"2","":"downstairs","session_id":"c06a09fb-c871-469e-bb7a-64bcbe64fe66"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:47.023593012Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1772,"client":"1","":"downstairs","session_id":"c06a09fb-c871-469e-bb7a-64bcbe64fe66"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:47.023620905Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1772,"client":"1","":"downstairs","session_id":"c06a09fb-c871-469e-bb7a-64bcbe64fe66"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:47.023647828Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1772,"client":"0","":"downstairs","session_id":"c06a09fb-c871-469e-bb7a-64bcbe64fe66"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:47.023675712Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1772,"client":"0","":"downstairs","session_id":"c06a09fb-c871-469e-bb7a-64bcbe64fe66"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:47.024182868Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1772,"client":"2","":"downstairs","session_id":"c06a09fb-c871-469e-bb7a-64bcbe64fe66"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:47.024220993Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1772,"client":"0","":"downstairs","session_id":"c06a09fb-c871-469e-bb7a-64bcbe64fe66"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:47.024303543Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1772,"client":"1","":"downstairs","session_id":"c06a09fb-c871-469e-bb7a-64bcbe64fe66"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:47.024673581Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1772,"client":"2","":"downstairs","session_id":"c06a09fb-c871-469e-bb7a-64bcbe64fe66"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:47.024712386Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1772,"client":"2","":"downstairs","session_id":"c06a09fb-c871-469e-bb7a-64bcbe64fe66"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:47.024741409Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1772,"session_id":"c06a09fb-c871-469e-bb7a-64bcbe64fe66"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:47.024803957Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1772,"client":"0","":"downstairs","session_id":"c06a09fb-c871-469e-bb7a-64bcbe64fe66"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:47.024835392Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1772,"client":"0","":"downstairs","session_id":"c06a09fb-c871-469e-bb7a-64bcbe64fe66"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:47.024850844Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1772,"session_id":"c06a09fb-c871-469e-bb7a-64bcbe64fe66"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:47.024863545Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1772,"client":"1","":"downstairs","session_id":"c06a09fb-c871-469e-bb7a-64bcbe64fe66"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:47.024876827Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1772,"client":"1","":"downstairs","session_id":"c06a09fb-c871-469e-bb7a-64bcbe64fe66"} {"msg":"[0]R flush_numbers[0..12]: [242, 242, 236, 242, 242, 242, 242, 242, 242, 242, 242, 240]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:47.024890879Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1772,"":"downstairs","":"downstairs","session_id":"c06a09fb-c871-469e-bb7a-64bcbe64fe66"} {"msg":"[0]R generation[0..12]: [147, 147, 143, 147, 147, 147, 147, 147, 147, 147, 147, 145]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:47.02490505Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1772,"":"downstairs","":"downstairs","session_id":"c06a09fb-c871-469e-bb7a-64bcbe64fe66"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:47.024919312Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1772,"":"downstairs","":"downstairs","session_id":"c06a09fb-c871-469e-bb7a-64bcbe64fe66"} {"msg":"[1]R flush_numbers[0..12]: [242, 242, 236, 242, 242, 242, 242, 242, 242, 242, 242, 240]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:47.024933284Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1772,"":"downstairs","":"downstairs","session_id":"c06a09fb-c871-469e-bb7a-64bcbe64fe66"} {"msg":"[1]R generation[0..12]: [147, 147, 143, 147, 147, 147, 147, 147, 147, 147, 147, 145]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:47.024946956Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1772,"":"downstairs","":"downstairs","session_id":"c06a09fb-c871-469e-bb7a-64bcbe64fe66"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:47.024960848Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1772,"":"downstairs","":"downstairs","session_id":"c06a09fb-c871-469e-bb7a-64bcbe64fe66"} {"msg":"[2]R flush_numbers[0..12]: [242, 242, 236, 242, 242, 242, 242, 242, 242, 242, 242, 240]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:47.024974759Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1772,"":"downstairs","":"downstairs","session_id":"c06a09fb-c871-469e-bb7a-64bcbe64fe66"} {"msg":"[2]R generation[0..12]: [147, 147, 143, 147, 147, 147, 147, 147, 147, 147, 147, 145]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:47.024988581Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1772,"":"downstairs","":"downstairs","session_id":"c06a09fb-c871-469e-bb7a-64bcbe64fe66"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:47.025002643Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1772,"":"downstairs","":"downstairs","session_id":"c06a09fb-c871-469e-bb7a-64bcbe64fe66"} {"msg":"Max found gen is 148","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:47.025015825Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1772,"":"downstairs","":"downstairs","session_id":"c06a09fb-c871-469e-bb7a-64bcbe64fe66"} {"msg":"Generation requested: 148 >= found:148","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:47.025031527Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1772,"":"downstairs","":"downstairs","session_id":"c06a09fb-c871-469e-bb7a-64bcbe64fe66"} {"msg":"Next flush: 243","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:47.025044859Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1772,"":"downstairs","":"downstairs","session_id":"c06a09fb-c871-469e-bb7a-64bcbe64fe66"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:47.02505797Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1772,"":"downstairs","":"downstairs","session_id":"c06a09fb-c871-469e-bb7a-64bcbe64fe66"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:47.025071232Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1772,"session_id":"c06a09fb-c871-469e-bb7a-64bcbe64fe66"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:47.025083353Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1772,"client":"0","":"downstairs","session_id":"c06a09fb-c871-469e-bb7a-64bcbe64fe66"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-01T16:55:47.025096525Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1772,"client":"1","":"downstairs","session_id":"c06a09fb-c871-469e-bb7a-64bcbe64fe66"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:47.025114168Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1772,"client":"2","":"downstairs","session_id":"c06a09fb-c871-469e-bb7a-64bcbe64fe66"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:47.025127449Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1772,"session_id":"c06a09fb-c871-469e-bb7a-64bcbe64fe66"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:47.025139771Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1772,"session_id":"c06a09fb-c871-469e-bb7a-64bcbe64fe66"} {"msg":"6e9b9e31-6578-42c6-93a3-3363483e81f5 is now active with session: c06a09fb-c871-469e-bb7a-64bcbe64fe66","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:47.025153013Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1772,"session_id":"c06a09fb-c871-469e-bb7a-64bcbe64fe66"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:47.025165034Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1772,"session_id":"c06a09fb-c871-469e-bb7a-64bcbe64fe66"} {"msg":"The guest has finished waiting for activation with:148","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:47.025177456Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1772} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 187 len 28672 data: 13 13 12 14 14 15 17 03/30 Write block 518 len 40960 data: 8 10 10 10 11 10 10 11 11 13 04/30 Read block 427 len 36864 05/30 Write block 129 len 12288 data: 15 17 17 06/30 Read block 355 len 40960 07/30 Write block 450 len 36864 data: 14 14 11 11 11 8 9 8 7 08/30 Read block 370 len 12288 09/30 Write block 293 len 36864 data: 12 14 12 12 12 11 9 7 6 10/30 Flush 11/30 Write block 498 len 16384 data: 14 14 14 12 12/30 Flush 13/30 Write block 69 len 24576 data: 15 19 18 18 13 12 14/30 Write block 267 len 40960 data: 11 11 13 15 19 16 17 20 18 17 15/30 Read block 207 len 24576 16/30 Read block 542 len 28672 17/30 Read block 477 len 28672 18/30 Read block 197 len 16384 19/30 Write block 481 len 12288 data: 13 12 13 20/30 Write block 121 len 12288 data: 14 16 13 21/30 Flush 22/30 Write block 356 len 40960 data: 10 14 13 10 9 9 6 6 7 9 23/30 Read block 503 len 32768 24/30 Flush 25/30 Write block 458 len 24576 data: 8 7 5 8 10 9 26/30 Write block 384 len 12288 data: 13 12 10 27/30 Write block 67 len 40960 data: 14 14 16 20 19 19 14 13 11 6 28/30 Write block 66 len 8192 data: 15 15 29/30 Read block 455 len 32768 30/30 Read block 277 len 28672 ---------------------------------------------------------------- Crucible gen:148 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 6 Done Done Done false 1025 Acked Write 3 Done Done Done false 1026 Acked Write 10 Done Done Done false 1027 Acked Write 2 Done Done Done false 1028 Acked Read 8 Sent Done Done false 1029 Acked Read 7 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 0 1 3 Done 4 6 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:55:47.051894358Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1772} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 147 147 147 242 242 242 F F F 1 020-039 147 147 147 242 242 242 F F F 2 040-059 143 143 143 236 236 236 F F F 3 060-079 148 148 148 246 246 246 T T T 4 080-099 147 147 147 242 242 242 F F F 5 100-119 147 147 147 242 242 242 F F F 6 120-139 148 148 148 246 246 246 F F F 7 140-159 147 147 147 242 242 242 F F F 8 160-179 147 147 147 242 242 242 F F F 9 180-199 148 148 148 244 244 244 F F F 10 200-219 147 147 147 242 242 242 F F F 11 220-239 145 145 145 240 240 240 F F F 12 240-259 147 147 147 242 242 242 F F F 13 260-279 148 148 148 246 246 246 F F F 14 280-299 148 148 148 244 244 244 F F F 15 300-319 148 148 148 244 244 244 F F F 16 320-339 145 145 145 240 240 240 F F F 17 340-359 148 148 148 247 247 247 F F F 18 360-379 148 148 148 247 247 247 F F F 19 380-399 144 144 144 237 237 237 T T T 20 400-419 147 147 147 242 242 242 F F F 21 420-439 143 143 143 236 236 236 F F F 22 440-459 148 148 148 244 244 244 T T T 23 460-479 147 147 147 242 242 242 T T T 24 480-499 148 148 148 246 246 246 F F F 25 500-519 148 148 148 245 245 245 F F F 26 520-539 148 148 148 244 244 244 F F F 27 540-559 140 140 140 233 233 233 F F F 28 560-579 145 145 145 240 240 240 F F F 29 580-599 145 145 145 240 240 240 F F F Max gen: 148, Max flush: 247 A Difference in extent metadata is expected here On loop 74 Verifying data now No change after verify Loop: 74 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 147 147 147 242 242 242 F F F 1 020-039 147 147 147 242 242 242 F F F 2 040-059 143 143 143 236 236 236 F F F 3 060-079 149 149 149 248 248 248 F F F 4 080-099 147 147 147 242 242 242 F F F 5 100-119 147 147 147 242 242 242 F F F 6 120-139 148 148 148 246 246 246 F F F 7 140-159 147 147 147 242 242 242 F F F 8 160-179 147 147 147 242 242 242 F F F 9 180-199 148 148 148 244 244 244 F F F 10 200-219 147 147 147 242 242 242 F F F 11 220-239 145 145 145 240 240 240 F F F 12 240-259 147 147 147 242 242 242 F F F 13 260-279 148 148 148 246 246 246 F F F 14 280-299 148 148 148 244 244 244 F F F 15 300-319 148 148 148 244 244 244 F F F 16 320-339 145 145 145 240 240 240 F F F 17 340-359 148 148 148 247 247 247 F F F 18 360-379 148 148 148 247 247 247 F F F 19 380-399 149 149 149 248 248 248 F F F 20 400-419 147 147 147 242 242 242 F F F 21 420-439 143 143 143 236 236 236 F F F 22 440-459 149 149 149 248 248 248 F F F 23 460-479 149 149 149 248 248 248 F F F 24 480-499 148 148 148 246 246 246 F F F 25 500-519 148 148 148 245 245 245 F F F 26 520-539 148 148 148 244 244 244 F F F 27 540-559 140 140 140 233 233 233 F F F 28 560-579 145 145 145 240 240 240 F F F 29 580-599 145 145 145 240 240 240 F F F Max gen: 149, Max flush: 248 Begin loop 75 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:51.345997452Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1780,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:55:51.392300418Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1780,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:51.392671897Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1780,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:51.425012464Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1780,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:51.425652746Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1780} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"3818f0fc76fe787c6cc412af30fd59b930203885\",\n git_commit_timestamp: \"2024-11-01T16:08:30.000000000Z\",\n git_branch: \"renovate/hyper-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:51.42568398Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1780} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:51.425697622Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1780} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:51.425722355Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1780} {"msg":"Crucible 924de524-0fa0-4c95-978b-c29bfee81212 has session id: 73b96583-9457-4cce-9247-0e9745a0e479","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:51.426243732Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1780,"session_id":"73b96583-9457-4cce-9247-0e9745a0e479"} {"msg":"Upstairs opts: Upstairs UUID: 924de524-0fa0-4c95-978b-c29bfee81212, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:51.426278196Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1780,"session_id":"73b96583-9457-4cce-9247-0e9745a0e479"} {"msg":"Crucible stats registered with UUID: 924de524-0fa0-4c95-978b-c29bfee81212","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:51.426297259Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1780,"session_id":"73b96583-9457-4cce-9247-0e9745a0e479"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:51.45703762Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1780,"task":"crutest"} {"msg":"The guest has requested activation with gen:150","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:51.457070444Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1780} {"msg":"924de524-0fa0-4c95-978b-c29bfee81212 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:51.457104368Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1780,"session_id":"73b96583-9457-4cce-9247-0e9745a0e479"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:51.457121061Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1780,"client":"0","":"downstairs","session_id":"73b96583-9457-4cce-9247-0e9745a0e479"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:51.457149634Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1780,"client":"0","":"downstairs","session_id":"73b96583-9457-4cce-9247-0e9745a0e479"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:51.457167687Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1780,"client":"1","":"downstairs","session_id":"73b96583-9457-4cce-9247-0e9745a0e479"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:51.457185149Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1780,"client":"1","":"downstairs","session_id":"73b96583-9457-4cce-9247-0e9745a0e479"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:51.457201901Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1780,"client":"2","":"downstairs","session_id":"73b96583-9457-4cce-9247-0e9745a0e479"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:51.457218693Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1780,"client":"2","":"downstairs","session_id":"73b96583-9457-4cce-9247-0e9745a0e479"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:51.457235355Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1780,"":"io task","client":"2","":"downstairs","session_id":"73b96583-9457-4cce-9247-0e9745a0e479"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:51.457329428Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1780,"":"io task","client":"0","":"downstairs","session_id":"73b96583-9457-4cce-9247-0e9745a0e479"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:51.457370543Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1780,"":"io task","client":"1","":"downstairs","session_id":"73b96583-9457-4cce-9247-0e9745a0e479"} {"msg":"ds_connection connected from Ok(127.0.0.1:49535)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:51.457408958Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1780,"":"io task","client":"2","":"downstairs","session_id":"73b96583-9457-4cce-9247-0e9745a0e479"} {"msg":"ds_connection connected from Ok(127.0.0.1:63664)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:51.457445102Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1780,"":"io task","client":"1","":"downstairs","session_id":"73b96583-9457-4cce-9247-0e9745a0e479"} {"msg":"ds_connection connected from Ok(127.0.0.1:42772)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:51.457587941Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1780,"":"io task","client":"0","":"downstairs","session_id":"73b96583-9457-4cce-9247-0e9745a0e479"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:51.4581991Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1780,"client":"2","":"downstairs","session_id":"73b96583-9457-4cce-9247-0e9745a0e479"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:51.458228813Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1780,"client":"2","":"downstairs","session_id":"73b96583-9457-4cce-9247-0e9745a0e479"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:51.458244755Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1780,"client":"1","":"downstairs","session_id":"73b96583-9457-4cce-9247-0e9745a0e479"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:51.458259257Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1780,"client":"1","":"downstairs","session_id":"73b96583-9457-4cce-9247-0e9745a0e479"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:51.458273009Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1780,"client":"0","":"downstairs","session_id":"73b96583-9457-4cce-9247-0e9745a0e479"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:51.458286311Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1780,"client":"0","":"downstairs","session_id":"73b96583-9457-4cce-9247-0e9745a0e479"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:51.458815529Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1780,"client":"2","":"downstairs","session_id":"73b96583-9457-4cce-9247-0e9745a0e479"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:51.458893459Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1780,"client":"1","":"downstairs","session_id":"73b96583-9457-4cce-9247-0e9745a0e479"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:51.458923073Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1780,"client":"0","":"downstairs","session_id":"73b96583-9457-4cce-9247-0e9745a0e479"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:51.459397774Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1780,"client":"2","":"downstairs","session_id":"73b96583-9457-4cce-9247-0e9745a0e479"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:51.459425907Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1780,"client":"2","":"downstairs","session_id":"73b96583-9457-4cce-9247-0e9745a0e479"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:51.459440849Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1780,"session_id":"73b96583-9457-4cce-9247-0e9745a0e479"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:51.459532881Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1780,"client":"1","":"downstairs","session_id":"73b96583-9457-4cce-9247-0e9745a0e479"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:51.459555484Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1780,"client":"1","":"downstairs","session_id":"73b96583-9457-4cce-9247-0e9745a0e479"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:51.459569286Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1780,"session_id":"73b96583-9457-4cce-9247-0e9745a0e479"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:51.459585218Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1780,"client":"0","":"downstairs","session_id":"73b96583-9457-4cce-9247-0e9745a0e479"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:51.45959935Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1780,"client":"0","":"downstairs","session_id":"73b96583-9457-4cce-9247-0e9745a0e479"} {"msg":"[0]R flush_numbers[0..12]: [242, 242, 236, 248, 242, 242, 246, 242, 242, 244, 242, 240]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:51.459613712Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1780,"":"downstairs","":"downstairs","session_id":"73b96583-9457-4cce-9247-0e9745a0e479"} {"msg":"[0]R generation[0..12]: [147, 147, 143, 149, 147, 147, 148, 147, 147, 148, 147, 145]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:51.459627853Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1780,"":"downstairs","":"downstairs","session_id":"73b96583-9457-4cce-9247-0e9745a0e479"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:51.459642246Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1780,"":"downstairs","":"downstairs","session_id":"73b96583-9457-4cce-9247-0e9745a0e479"} {"msg":"[1]R flush_numbers[0..12]: [242, 242, 236, 248, 242, 242, 246, 242, 242, 244, 242, 240]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:51.459656027Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1780,"":"downstairs","":"downstairs","session_id":"73b96583-9457-4cce-9247-0e9745a0e479"} {"msg":"[1]R generation[0..12]: [147, 147, 143, 149, 147, 147, 148, 147, 147, 148, 147, 145]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:51.459669879Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1780,"":"downstairs","":"downstairs","session_id":"73b96583-9457-4cce-9247-0e9745a0e479"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:51.459683891Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1780,"":"downstairs","":"downstairs","session_id":"73b96583-9457-4cce-9247-0e9745a0e479"} {"msg":"[2]R flush_numbers[0..12]: [242, 242, 236, 248, 242, 242, 246, 242, 242, 244, 242, 240]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:51.459697853Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1780,"":"downstairs","":"downstairs","session_id":"73b96583-9457-4cce-9247-0e9745a0e479"} {"msg":"[2]R generation[0..12]: [147, 147, 143, 149, 147, 147, 148, 147, 147, 148, 147, 145]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:51.459711714Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1780,"":"downstairs","":"downstairs","session_id":"73b96583-9457-4cce-9247-0e9745a0e479"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:51.459725696Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1780,"":"downstairs","":"downstairs","session_id":"73b96583-9457-4cce-9247-0e9745a0e479"} {"msg":"Max found gen is 150","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:51.459738758Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1780,"":"downstairs","":"downstairs","session_id":"73b96583-9457-4cce-9247-0e9745a0e479"} {"msg":"Generation requested: 150 >= found:150","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:51.459752049Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1780,"":"downstairs","":"downstairs","session_id":"73b96583-9457-4cce-9247-0e9745a0e479"} {"msg":"Next flush: 249","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:51.459765391Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1780,"":"downstairs","":"downstairs","session_id":"73b96583-9457-4cce-9247-0e9745a0e479"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:51.459780093Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1780,"":"downstairs","":"downstairs","session_id":"73b96583-9457-4cce-9247-0e9745a0e479"} Wait for a query_work_queue command to finish before sending IO {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:51.459801826Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1780,"session_id":"73b96583-9457-4cce-9247-0e9745a0e479"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:51.459814268Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1780,"client":"0","":"downstairs","session_id":"73b96583-9457-4cce-9247-0e9745a0e479"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:51.459827569Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1780,"client":"1","":"downstairs","session_id":"73b96583-9457-4cce-9247-0e9745a0e479"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:51.459840721Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1780,"client":"2","":"downstairs","session_id":"73b96583-9457-4cce-9247-0e9745a0e479"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:51.459854203Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1780,"session_id":"73b96583-9457-4cce-9247-0e9745a0e479"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:51.459866504Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1780,"session_id":"73b96583-9457-4cce-9247-0e9745a0e479"} {"msg":"924de524-0fa0-4c95-978b-c29bfee81212 is now active with session: 73b96583-9457-4cce-9247-0e9745a0e479","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:51.459879296Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1780,"session_id":"73b96583-9457-4cce-9247-0e9745a0e479"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:51.459891378Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1780,"session_id":"73b96583-9457-4cce-9247-0e9745a0e479"} {"msg":"The guest has finished waiting for activation with:150","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:51.459903859Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1780} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 577 len 12288 data: 11 13 13 02/30 Write block 106 len 12288 data: 15 12 12 03/30 Read block 94 len 40960 04/30 Read block 435 len 36864 05/30 Write block 554 len 16384 data: 15 13 14 15 06/30 Write block 499 len 4096 data: 15 07/30 Read block 570 len 20480 08/30 Read block 252 len 32768 09/30 Read block 335 len 16384 10/30 Read block 375 len 4096 11/30 Write block 272 len 36864 data: 17 18 21 19 18 12 10 10 10 12/30 Read block 509 len 12288 13/30 Read block 155 len 24576 14/30 Read block 16 len 4096 15/30 Read block 474 len 4096 16/30 Write block 589 len 8192 data: 13 11 17/30 Read block 105 len 32768 18/30 Read block 26 len 28672 19/30 Write block 470 len 36864 data: 13 15 15 15 13 13 10 16 20 20/30 Write block 449 len 8192 data: 11 15 21/30 Read block 329 len 12288 22/30 Read block 441 len 20480 23/30 Read block 219 len 8192 24/30 Read block 238 len 24576 25/30 Flush 26/30 Write block 320 len 12288 data: 14 14 13 27/30 Read block 86 len 24576 28/30 Read block 3 len 4096 29/30 Write block 3 len 12288 data: 8 10 9 30/30 Write block 452 len 36864 data: 12 12 12 9 10 9 9 8 6 ---------------------------------------------------------------- Crucible gen:150 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 3 Done Done Done false 1026 Acked Read 6 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Write 3 Done Done Done false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:55:51.481294455Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1780} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 147 147 147 242 242 242 T T T 1 020-039 147 147 147 242 242 242 F F F 2 040-059 143 143 143 236 236 236 F F F 3 060-079 149 149 149 248 248 248 F F F 4 080-099 147 147 147 242 242 242 F F F 5 100-119 150 150 150 249 249 249 F F F 6 120-139 148 148 148 246 246 246 F F F 7 140-159 147 147 147 242 242 242 F F F 8 160-179 147 147 147 242 242 242 F F F 9 180-199 148 148 148 244 244 244 F F F 10 200-219 147 147 147 242 242 242 F F F 11 220-239 145 145 145 240 240 240 F F F 12 240-259 147 147 147 242 242 242 F F F 13 260-279 150 150 150 249 249 249 F F F 14 280-299 150 150 150 249 249 249 F F F 15 300-319 148 148 148 244 244 244 F F F 16 320-339 145 145 145 240 240 240 T T T 17 340-359 148 148 148 247 247 247 F F F 18 360-379 148 148 148 247 247 247 F F F 19 380-399 149 149 149 248 248 248 F F F 20 400-419 147 147 147 242 242 242 F F F 21 420-439 143 143 143 236 236 236 F F F 22 440-459 150 150 150 249 249 249 T T T 23 460-479 150 150 150 249 249 249 T T T 24 480-499 150 150 150 249 249 249 F F F 25 500-519 148 148 148 245 245 245 F F F 26 520-539 148 148 148 244 244 244 F F F 27 540-559 150 150 150 249 249 249 F F F 28 560-579 150 150 150 249 249 249 F F F 29 580-599 150 150 150 249 249 249 F F F Max gen: 150, Max flush: 249 A Difference in extent metadata is expected here On loop 75 Verifying data now No change after verify Loop: 75 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 151 151 151 250 250 250 F F F 1 020-039 147 147 147 242 242 242 F F F 2 040-059 143 143 143 236 236 236 F F F 3 060-079 149 149 149 248 248 248 F F F 4 080-099 147 147 147 242 242 242 F F F 5 100-119 150 150 150 249 249 249 F F F 6 120-139 148 148 148 246 246 246 F F F 7 140-159 147 147 147 242 242 242 F F F 8 160-179 147 147 147 242 242 242 F F F 9 180-199 148 148 148 244 244 244 F F F 10 200-219 147 147 147 242 242 242 F F F 11 220-239 145 145 145 240 240 240 F F F 12 240-259 147 147 147 242 242 242 F F F 13 260-279 150 150 150 249 249 249 F F F 14 280-299 150 150 150 249 249 249 F F F 15 300-319 148 148 148 244 244 244 F F F 16 320-339 151 151 151 250 250 250 F F F 17 340-359 148 148 148 247 247 247 F F F 18 360-379 148 148 148 247 247 247 F F F 19 380-399 149 149 149 248 248 248 F F F 20 400-419 147 147 147 242 242 242 F F F 21 420-439 143 143 143 236 236 236 F F F 22 440-459 151 151 151 250 250 250 F F F 23 460-479 151 151 151 250 250 250 F F F 24 480-499 150 150 150 249 249 249 F F F 25 500-519 148 148 148 245 245 245 F F F 26 520-539 148 148 148 244 244 244 F F F 27 540-559 150 150 150 249 249 249 F F F 28 560-579 150 150 150 249 249 249 F F F 29 580-599 150 150 150 249 249 249 F F F Max gen: 151, Max flush: 250 Begin loop 76 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:55.806475712Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1788,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:55.85115848Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1788,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:55.851765658Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1788} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"3818f0fc76fe787c6cc412af30fd59b930203885\",\n git_commit_timestamp: \"2024-11-01T16:08:30.000000000Z\",\n git_branch: \"renovate/hyper-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:55.851805133Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1788} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:55.851819955Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1788} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:55.851834497Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1788} {"msg":"Crucible 82234739-6beb-4ec5-ad67-4d07c038fcd6 has session id: 4bf2357e-6d91-4875-9834-4342663a5a73","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:55.852259902Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1788,"session_id":"4bf2357e-6d91-4875-9834-4342663a5a73"} {"msg":"Upstairs opts: Upstairs UUID: 82234739-6beb-4ec5-ad67-4d07c038fcd6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:55.852299207Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1788,"session_id":"4bf2357e-6d91-4875-9834-4342663a5a73"} {"msg":"Crucible stats registered with UUID: 82234739-6beb-4ec5-ad67-4d07c038fcd6","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:55.852318819Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1788,"session_id":"4bf2357e-6d91-4875-9834-4342663a5a73"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:55.883178366Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1788,"task":"crutest"} {"msg":"The guest has requested activation with gen:152","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:55.883221021Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1788} {"msg":"82234739-6beb-4ec5-ad67-4d07c038fcd6 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:55.88328892Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1788,"session_id":"4bf2357e-6d91-4875-9834-4342663a5a73"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:55.883326085Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1788,"client":"0","":"downstairs","session_id":"4bf2357e-6d91-4875-9834-4342663a5a73"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:55.883360629Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1788,"client":"0","":"downstairs","session_id":"4bf2357e-6d91-4875-9834-4342663a5a73"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:55.883393493Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1788,"client":"1","":"downstairs","session_id":"4bf2357e-6d91-4875-9834-4342663a5a73"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:55.883426307Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1788,"client":"1","":"downstairs","session_id":"4bf2357e-6d91-4875-9834-4342663a5a73"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:55.883451071Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1788,"client":"2","":"downstairs","session_id":"4bf2357e-6d91-4875-9834-4342663a5a73"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:55.883482415Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1788,"client":"2","":"downstairs","session_id":"4bf2357e-6d91-4875-9834-4342663a5a73"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:55.883513799Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1788,"":"io task","client":"2","":"downstairs","session_id":"4bf2357e-6d91-4875-9834-4342663a5a73"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:55.883548553Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1788,"":"io task","client":"0","":"downstairs","session_id":"4bf2357e-6d91-4875-9834-4342663a5a73"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:55.883582518Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1788,"":"io task","client":"1","":"downstairs","session_id":"4bf2357e-6d91-4875-9834-4342663a5a73"} {"msg":"ds_connection connected from Ok(127.0.0.1:45950)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:55.883617672Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1788,"":"io task","client":"2","":"downstairs","session_id":"4bf2357e-6d91-4875-9834-4342663a5a73"} {"msg":"ds_connection connected from Ok(127.0.0.1:53528)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:55.883648966Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1788,"":"io task","client":"0","":"downstairs","session_id":"4bf2357e-6d91-4875-9834-4342663a5a73"} {"msg":"ds_connection connected from Ok(127.0.0.1:40961)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:55.883766241Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1788,"":"io task","client":"1","":"downstairs","session_id":"4bf2357e-6d91-4875-9834-4342663a5a73"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:55.884258975Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1788,"client":"2","":"downstairs","session_id":"4bf2357e-6d91-4875-9834-4342663a5a73"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:55.884290569Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1788,"client":"2","":"downstairs","session_id":"4bf2357e-6d91-4875-9834-4342663a5a73"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:55.884316532Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1788,"client":"0","":"downstairs","session_id":"4bf2357e-6d91-4875-9834-4342663a5a73"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:55.884343806Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1788,"client":"0","":"downstairs","session_id":"4bf2357e-6d91-4875-9834-4342663a5a73"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:55.88437647Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1788,"client":"1","":"downstairs","session_id":"4bf2357e-6d91-4875-9834-4342663a5a73"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:55.884404644Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1788,"client":"1","":"downstairs","session_id":"4bf2357e-6d91-4875-9834-4342663a5a73"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:55.884912309Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1788,"client":"0","":"downstairs","session_id":"4bf2357e-6d91-4875-9834-4342663a5a73"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:55.884954284Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1788,"client":"2","":"downstairs","session_id":"4bf2357e-6d91-4875-9834-4342663a5a73"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:55.885003591Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1788,"client":"1","":"downstairs","session_id":"4bf2357e-6d91-4875-9834-4342663a5a73"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:55.88553899Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1788,"client":"0","":"downstairs","session_id":"4bf2357e-6d91-4875-9834-4342663a5a73"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:55.885566973Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1788,"client":"0","":"downstairs","session_id":"4bf2357e-6d91-4875-9834-4342663a5a73"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:55.885581825Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1788,"session_id":"4bf2357e-6d91-4875-9834-4342663a5a73"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:55.885594367Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1788,"client":"2","":"downstairs","session_id":"4bf2357e-6d91-4875-9834-4342663a5a73"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:55.885607848Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1788,"client":"2","":"downstairs","session_id":"4bf2357e-6d91-4875-9834-4342663a5a73"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:55.88562106Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1788,"session_id":"4bf2357e-6d91-4875-9834-4342663a5a73"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:55.885633332Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1788,"client":"1","":"downstairs","session_id":"4bf2357e-6d91-4875-9834-4342663a5a73"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:55.885646733Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1788,"client":"1","":"downstairs","session_id":"4bf2357e-6d91-4875-9834-4342663a5a73"} {"msg":"[0]R flush_numbers[0..12]: [250, 242, 236, 248, 242, 249, 246, 242, 242, 244, 242, 240]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:55.885660745Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1788,"":"downstairs","":"downstairs","session_id":"4bf2357e-6d91-4875-9834-4342663a5a73"} {"msg":"[0]R generation[0..12]: [151, 147, 143, 149, 147, 150, 148, 147, 147, 148, 147, 145]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:55.885674907Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1788,"":"downstairs","":"downstairs","session_id":"4bf2357e-6d91-4875-9834-4342663a5a73"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:55.885692779Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1788,"":"downstairs","":"downstairs","session_id":"4bf2357e-6d91-4875-9834-4342663a5a73"} {"msg":"[1]R flush_numbers[0..12]: [250, 242, 236, 248, 242, 249, 246, 242, 242, 244, 242, 240]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:55.885707892Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1788,"":"downstairs","":"downstairs","session_id":"4bf2357e-6d91-4875-9834-4342663a5a73"} {"msg":"[1]R generation[0..12]: [151, 147, 143, 149, 147, 150, 148, 147, 147, 148, 147, 145]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:55.885721793Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1788,"":"downstairs","":"downstairs","session_id":"4bf2357e-6d91-4875-9834-4342663a5a73"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:55.885735895Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1788,"":"downstairs","":"downstairs","session_id":"4bf2357e-6d91-4875-9834-4342663a5a73"} {"msg":"[2]R flush_numbers[0..12]: [250, 242, 236, 248, 242, 249, 246, 242, 242, 244, 242, 240]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:55.885755788Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1788,"":"downstairs","":"downstairs","session_id":"4bf2357e-6d91-4875-9834-4342663a5a73"} {"msg":"[2]R generation[0..12]: [151, 147, 143, 149, 147, 150, 148, 147, 147, 148, 147, 145]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-01T16:55:55.885782641Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1788,"":"downstairs","":"downstairs","session_id":"4bf2357e-6d91-4875-9834-4342663a5a73"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:55.885809234Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1788,"":"downstairs","":"downstairs","session_id":"4bf2357e-6d91-4875-9834-4342663a5a73"} {"msg":"Max found gen is 152","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:55.885827527Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1788,"":"downstairs","":"downstairs","session_id":"4bf2357e-6d91-4875-9834-4342663a5a73"} {"msg":"Generation requested: 152 >= found:152","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:55.885841189Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1788,"":"downstairs","":"downstairs","session_id":"4bf2357e-6d91-4875-9834-4342663a5a73"} {"msg":"Next flush: 251","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:55.88585434Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1788,"":"downstairs","":"downstairs","session_id":"4bf2357e-6d91-4875-9834-4342663a5a73"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:55.885867472Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1788,"":"downstairs","":"downstairs","session_id":"4bf2357e-6d91-4875-9834-4342663a5a73"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:55.885880654Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1788,"session_id":"4bf2357e-6d91-4875-9834-4342663a5a73"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:55.885892765Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1788,"client":"0","":"downstairs","session_id":"4bf2357e-6d91-4875-9834-4342663a5a73"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:55.885905927Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1788,"client":"1","":"downstairs","session_id":"4bf2357e-6d91-4875-9834-4342663a5a73"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:55.885922699Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1788,"client":"2","":"downstairs","session_id":"4bf2357e-6d91-4875-9834-4342663a5a73"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:55.885936271Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1788,"session_id":"4bf2357e-6d91-4875-9834-4342663a5a73"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:55.885949452Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1788,"session_id":"4bf2357e-6d91-4875-9834-4342663a5a73"} {"msg":"82234739-6beb-4ec5-ad67-4d07c038fcd6 is now active with session: 4bf2357e-6d91-4875-9834-4342663a5a73","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:55.885962104Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1788,"session_id":"4bf2357e-6d91-4875-9834-4342663a5a73"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:55.885974206Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1788,"session_id":"4bf2357e-6d91-4875-9834-4342663a5a73"} {"msg":"The guest has finished waiting for activation with:152","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:55:55.885986527Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1788} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 196 len 32768 data: 14 13 12 13 11 11 11 12 02/30 Flush 03/30 Write block 252 len 24576 data: 17 15 14 11 10 10 04/30 Read block 196 len 24576 05/30 Write block 514 len 12288 data: 7 5 5 06/30 Write block 559 len 12288 data: 22 17 17 07/30 Read block 278 len 40960 08/30 Write block 519 len 28672 data: 11 11 11 12 11 11 12 09/30 Read block 527 len 8192 10/30 Read block 40 len 28672 11/30 Read block 302 len 28672 12/30 Write block 468 len 24576 data: 11 12 14 16 16 16 13/30 Flush 14/30 Write block 25 len 8192 data: 15 13 15/30 Flush 16/30 Flush 17/30 Write block 52 len 32768 data: 12 11 10 9 10 10 9 9 18/30 Write block 398 len 8192 data: 14 12 19/30 Read block 506 len 20480 20/30 Flush 21/30 Write block 229 len 36864 data: 10 9 7 7 6 8 8 10 9 22/30 Read block 129 len 12288 23/30 Read block 377 len 20480 24/30 Write block 458 len 32768 data: 10 9 7 9 11 10 8 8 25/30 Read block 168 len 20480 26/30 Write block 306 len 20480 data: 10 14 16 16 15 27/30 Read block 576 len 12288 28/30 Read block 256 len 24576 29/30 Read block 290 len 12288 30/30 Write block 189 len 28672 data: 13 15 15 16 18 18 16 ---------------------------------------------------------------- Crucible gen:152 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1020 Acked Write 9 Done Done Done false 1021 Acked Read 3 Done Done Done false 1022 Acked Read 5 Done Done Done false 1023 Acked Write 8 Done Done Done false 1024 Acked Read 5 Done Done Done false 1025 Acked Write 5 Done Done Done false 1026 Acked Read 3 Done Done Done false 1027 Acked Read 6 Done Done Done false 1028 Acked Read 3 Done Done Done false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 9 9 9 27 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1019 1019 1019 Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:55:55.911212188Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1788} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 151 151 151 250 250 250 F F F 1 020-039 152 152 152 253 253 253 F F F 2 040-059 152 152 152 255 255 255 F F F 3 060-079 149 149 149 248 248 248 F F F 4 080-099 147 147 147 242 242 242 F F F 5 100-119 150 150 150 249 249 249 F F F 6 120-139 148 148 148 246 246 246 F F F 7 140-159 147 147 147 242 242 242 F F F 8 160-179 147 147 147 242 242 242 F F F 9 180-199 152 152 152 251 251 251 T T T 10 200-219 152 152 152 251 251 251 F F F 11 220-239 145 145 145 240 240 240 T T T 12 240-259 152 152 152 252 252 252 F F F 13 260-279 150 150 150 249 249 249 F F F 14 280-299 150 150 150 249 249 249 F F F 15 300-319 148 148 148 244 244 244 T T T 16 320-339 151 151 151 250 250 250 F F F 17 340-359 148 148 148 247 247 247 F F F 18 360-379 148 148 148 247 247 247 F F F 19 380-399 152 152 152 255 255 255 F F F 20 400-419 147 147 147 242 242 242 F F F 21 420-439 143 143 143 236 236 236 F F F 22 440-459 151 151 151 250 250 250 T T T 23 460-479 152 152 152 252 252 252 T T T 24 480-499 150 150 150 249 249 249 F F F 25 500-519 152 152 152 252 252 252 F F F 26 520-539 152 152 152 252 252 252 F F F 27 540-559 152 152 152 252 252 252 F F F 28 560-579 152 152 152 252 252 252 F F F 29 580-599 150 150 150 249 249 249 F F F Max gen: 152, Max flush: 255 A Difference in extent metadata is expected here On loop 76 Verifying data now No change after verify Loop: 76 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 151 151 151 250 250 250 F F F 1 020-039 152 152 152 253 253 253 F F F 2 040-059 152 152 152 255 255 255 F F F 3 060-079 149 149 149 248 248 248 F F F 4 080-099 147 147 147 242 242 242 F F F 5 100-119 150 150 150 249 249 249 F F F 6 120-139 148 148 148 246 246 246 F F F 7 140-159 147 147 147 242 242 242 F F F 8 160-179 147 147 147 242 242 242 F F F 9 180-199 153 153 153 256 256 256 F F F 10 200-219 152 152 152 251 251 251 F F F 11 220-239 153 153 153 256 256 256 F F F 12 240-259 152 152 152 252 252 252 F F F 13 260-279 150 150 150 249 249 249 F F F 14 280-299 150 150 150 249 249 249 F F F 15 300-319 153 153 153 256 256 256 F F F 16 320-339 151 151 151 250 250 250 F F F 17 340-359 148 148 148 247 247 247 F F F 18 360-379 148 148 148 247 247 247 F F F 19 380-399 152 152 152 255 255 255 F F F 20 400-419 147 147 147 242 242 242 F F F 21 420-439 143 143 143 236 236 236 F F F 22 440-459 153 153 153 256 256 256 F F F 23 460-479 153 153 153 256 256 256 F F F 24 480-499 150 150 150 249 249 249 F F F 25 500-519 152 152 152 252 252 252 F F F 26 520-539 152 152 152 252 252 252 F F F 27 540-559 152 152 152 252 252 252 F F F 28 560-579 152 152 152 252 252 252 F F F 29 580-599 150 150 150 249 249 249 F F F Max gen: 153, Max flush: 256 Begin loop 77 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:00.215706962Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1796,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:00.259739285Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1796,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:00.260365056Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1796} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"3818f0fc76fe787c6cc412af30fd59b930203885\",\n git_commit_timestamp: \"2024-11-01T16:08:30.000000000Z\",\n git_branch: \"renovate/hyper-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:00.260407011Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1796} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:00.260422303Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1796} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:00.260436835Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1796} {"msg":"Crucible d3822582-d727-45b4-80e5-bd647017ceae has session id: ba9b397b-01d3-4f12-af24-d59af7374baf","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:00.260900775Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1796,"session_id":"ba9b397b-01d3-4f12-af24-d59af7374baf"} {"msg":"Upstairs opts: Upstairs UUID: d3822582-d727-45b4-80e5-bd647017ceae, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:00.260943531Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1796,"session_id":"ba9b397b-01d3-4f12-af24-d59af7374baf"} {"msg":"Crucible stats registered with UUID: d3822582-d727-45b4-80e5-bd647017ceae","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:00.260965343Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1796,"session_id":"ba9b397b-01d3-4f12-af24-d59af7374baf"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:00.292211799Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1796,"task":"crutest"} {"msg":"The guest has requested activation with gen:154","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:00.292238723Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1796} {"msg":"d3822582-d727-45b4-80e5-bd647017ceae active request set","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:00.292311512Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1796,"session_id":"ba9b397b-01d3-4f12-af24-d59af7374baf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:00.292333305Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1796,"client":"0","":"downstairs","session_id":"ba9b397b-01d3-4f12-af24-d59af7374baf"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:00.292361409Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1796,"client":"0","":"downstairs","session_id":"ba9b397b-01d3-4f12-af24-d59af7374baf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:00.292378971Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1796,"client":"1","":"downstairs","session_id":"ba9b397b-01d3-4f12-af24-d59af7374baf"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:00.292411285Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1796,"client":"1","":"downstairs","session_id":"ba9b397b-01d3-4f12-af24-d59af7374baf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:00.292445279Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1796,"client":"2","":"downstairs","session_id":"ba9b397b-01d3-4f12-af24-d59af7374baf"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:00.292475913Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1796,"client":"2","":"downstairs","session_id":"ba9b397b-01d3-4f12-af24-d59af7374baf"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:00.292506557Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1796,"":"io task","client":"2","":"downstairs","session_id":"ba9b397b-01d3-4f12-af24-d59af7374baf"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:00.292540452Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1796,"":"io task","client":"0","":"downstairs","session_id":"ba9b397b-01d3-4f12-af24-d59af7374baf"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:00.292572836Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1796,"":"io task","client":"1","":"downstairs","session_id":"ba9b397b-01d3-4f12-af24-d59af7374baf"} {"msg":"ds_connection connected from Ok(127.0.0.1:38610)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:00.29260499Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1796,"":"io task","client":"1","":"downstairs","session_id":"ba9b397b-01d3-4f12-af24-d59af7374baf"} {"msg":"ds_connection connected from Ok(127.0.0.1:62616)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:00.292638624Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1796,"":"io task","client":"0","":"downstairs","session_id":"ba9b397b-01d3-4f12-af24-d59af7374baf"} {"msg":"ds_connection connected from Ok(127.0.0.1:58159)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:00.292786814Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1796,"":"io task","client":"2","":"downstairs","session_id":"ba9b397b-01d3-4f12-af24-d59af7374baf"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:00.293373149Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1796,"client":"0","":"downstairs","session_id":"ba9b397b-01d3-4f12-af24-d59af7374baf"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:00.293401753Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1796,"client":"0","":"downstairs","session_id":"ba9b397b-01d3-4f12-af24-d59af7374baf"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:00.293417285Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1796,"client":"1","":"downstairs","session_id":"ba9b397b-01d3-4f12-af24-d59af7374baf"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:00.293430926Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1796,"client":"1","":"downstairs","session_id":"ba9b397b-01d3-4f12-af24-d59af7374baf"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:00.293444458Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1796,"client":"2","":"downstairs","session_id":"ba9b397b-01d3-4f12-af24-d59af7374baf"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:00.293522238Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1796,"client":"2","":"downstairs","session_id":"ba9b397b-01d3-4f12-af24-d59af7374baf"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:00.293991399Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1796,"client":"0","":"downstairs","session_id":"ba9b397b-01d3-4f12-af24-d59af7374baf"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:00.294087591Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1796,"client":"1","":"downstairs","session_id":"ba9b397b-01d3-4f12-af24-d59af7374baf"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:00.294319101Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1796,"client":"2","":"downstairs","session_id":"ba9b397b-01d3-4f12-af24-d59af7374baf"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:00.294643903Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1796,"client":"0","":"downstairs","session_id":"ba9b397b-01d3-4f12-af24-d59af7374baf"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:00.294674907Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1796,"client":"0","":"downstairs","session_id":"ba9b397b-01d3-4f12-af24-d59af7374baf"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:00.29470155Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1796,"session_id":"ba9b397b-01d3-4f12-af24-d59af7374baf"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:00.294727033Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1796,"client":"1","":"downstairs","session_id":"ba9b397b-01d3-4f12-af24-d59af7374baf"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:00.294753117Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1796,"client":"1","":"downstairs","session_id":"ba9b397b-01d3-4f12-af24-d59af7374baf"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:00.29478216Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1796,"session_id":"ba9b397b-01d3-4f12-af24-d59af7374baf"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:00.294804723Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1796,"client":"2","":"downstairs","session_id":"ba9b397b-01d3-4f12-af24-d59af7374baf"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:00.294828246Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1796,"client":"2","":"downstairs","session_id":"ba9b397b-01d3-4f12-af24-d59af7374baf"} {"msg":"[0]R flush_numbers[0..12]: [250, 253, 255, 248, 242, 249, 246, 242, 242, 256, 251, 256]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:00.294852209Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1796,"":"downstairs","":"downstairs","session_id":"ba9b397b-01d3-4f12-af24-d59af7374baf"} {"msg":"[0]R generation[0..12]: [151, 152, 152, 149, 147, 150, 148, 147, 147, 153, 152, 153]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:00.294877793Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1796,"":"downstairs","":"downstairs","session_id":"ba9b397b-01d3-4f12-af24-d59af7374baf"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:00.294902856Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1796,"":"downstairs","":"downstairs","session_id":"ba9b397b-01d3-4f12-af24-d59af7374baf"} {"msg":"[1]R flush_numbers[0..12]: [250, 253, 255, 248, 242, 249, 246, 242, 242, 256, 251, 256]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:00.294926089Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1796,"":"downstairs","":"downstairs","session_id":"ba9b397b-01d3-4f12-af24-d59af7374baf"} {"msg":"[1]R generation[0..12]: [151, 152, 152, 149, 147, 150, 148, 147, 147, 153, 152, 153]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:00.294952833Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1796,"":"downstairs","":"downstairs","session_id":"ba9b397b-01d3-4f12-af24-d59af7374baf"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:00.294978826Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1796,"":"downstairs","":"downstairs","session_id":"ba9b397b-01d3-4f12-af24-d59af7374baf"} {"msg":"[2]R flush_numbers[0..12]: [250, 253, 255, 248, 242, 249, 246, 242, 242, 256, 251, 256]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:00.295006239Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1796,"":"downstairs","":"downstairs","session_id":"ba9b397b-01d3-4f12-af24-d59af7374baf"} {"msg":"[2]R generation[0..12]: [151, 152, 152, 149, 147, 150, 148, 147, 147, 153, 152, 153]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:00.295032233Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1796,"":"downstairs","":"downstairs","session_id":"ba9b397b-01d3-4f12-af24-d59af7374baf"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:00.295059546Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1796,"":"downstairs","":"downstairs","session_id":"ba9b397b-01d3-4f12-af24-d59af7374baf"} {"msg":"Max found gen is 154"Wait for a query_work_queue command to finish before sending IO ,"v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:00.295094791Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1796,"":"downstairs","":"downstairs","session_id":"ba9b397b-01d3-4f12-af24-d59af7374baf"} {"msg":"Generation requested: 154 >= found:154","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:00.295125005Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1796,"":"downstairs","":"downstairs","session_id":"ba9b397b-01d3-4f12-af24-d59af7374baf"} {"msg":"Next flush: 257","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:00.295151788Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1796,"":"downstairs","":"downstairs","session_id":"ba9b397b-01d3-4f12-af24-d59af7374baf"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:00.295178051Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1796,"":"downstairs","":"downstairs","session_id":"ba9b397b-01d3-4f12-af24-d59af7374baf"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:00.295204965Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1796,"session_id":"ba9b397b-01d3-4f12-af24-d59af7374baf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:00.295229568Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1796,"client":"0","":"downstairs","session_id":"ba9b397b-01d3-4f12-af24-d59af7374baf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2024-11-01T16:56:00.295256452Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1796,"client":"1","":"downstairs","session_id":"ba9b397b-01d3-4f12-af24-d59af7374baf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:00.295286696Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1796,"client":"2","":"downstairs","session_id":"ba9b397b-01d3-4f12-af24-d59af7374baf"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:00.295314149Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1796,"session_id":"ba9b397b-01d3-4f12-af24-d59af7374baf"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:00.295328411Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1796,"session_id":"ba9b397b-01d3-4f12-af24-d59af7374baf"} {"msg":"d3822582-d727-45b4-80e5-bd647017ceae is now active with session: ba9b397b-01d3-4f12-af24-d59af7374baf","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:00.295344183Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1796,"session_id":"ba9b397b-01d3-4f12-af24-d59af7374baf"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:00.295356685Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1796,"session_id":"ba9b397b-01d3-4f12-af24-d59af7374baf"} {"msg":"The guest has finished waiting for activation with:154","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:00.295368926Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1796} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 286 len 8192 data: 8 8 03/30 Write block 380 len 36864 data: 10 11 14 14 14 13 11 9 10 04/30 Flush 05/30 Write block 141 len 16384 data: 11 11 8 10 06/30 Flush 07/30 Write block 438 len 20480 data: 9 8 10 8 8 08/30 Flush 09/30 Flush 10/30 Write block 201 len 4096 data: 12 11/30 Read block 363 len 24576 12/30 Read block 489 len 32768 13/30 Write block 262 len 8192 data: 14 12 14/30 Write block 194 len 24576 data: 19 17 15 14 13 14 15/30 Write block 403 len 8192 data: 10 9 16/30 Read block 388 len 24576 17/30 Read block 241 len 8192 18/30 Write block 554 len 4096 data: 16 19/30 Flush 20/30 Write block 539 len 8192 data: 14 13 21/30 Write block 558 len 40960 data: 18 23 18 18 18 20 19 12 12 13 22/30 Write block 165 len 32768 data: 11 9 8 9 9 10 8 7 23/30 Read block 318 len 28672 24/30 Read block 180 len 36864 25/30 Read block 254 len 40960 26/30 Write block 446 len 36864 data: 10 11 11 12 16 15 13 13 13 27/30 Write block 552 len 16384 data: 13 13 17 14 28/30 Read block 395 len 16384 29/30 Read block 38 len 32768 30/30 Write block 492 len 16384 data: 13 9 8 11 ---------------------------------------------------------------- Crucible gen:154 GIO:true work queues: Upstairs:0 downstairs:11 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1019 Acked Write 2 Done Done Done false 1020 Acked Write 10 Done Done Done false 1021 Acked Write 8 Done Done Done false 1022 Acked Read 7 Done Done Done false 1023 Acked Read 9 Done Done Done false 1024 Acked Read 10 Done Done Done false 1025 Acked Write 9 Done Done Done false 1026 Acked Write 4 Done Done Done false 1027 Acked Read 4 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 10 10 10 30 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1018 1018 1018 Downstairs last five completed: 1018 1017 1016 1015 1014 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:56:00.320016092Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1796} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 151 151 151 250 250 250 F F F 1 020-039 152 152 152 253 253 253 F F F 2 040-059 152 152 152 255 255 255 F F F 3 060-079 149 149 149 248 248 248 F F F 4 080-099 147 147 147 242 242 242 F F F 5 100-119 150 150 150 249 249 249 F F F 6 120-139 148 148 148 246 246 246 F F F 7 140-159 154 154 154 259 259 259 F F F 8 160-179 147 147 147 242 242 242 T T T 9 180-199 154 154 154 262 262 262 F F F 10 200-219 154 154 154 262 262 262 F F F 11 220-239 153 153 153 256 256 256 F F F 12 240-259 152 152 152 252 252 252 F F F 13 260-279 154 154 154 262 262 262 F F F 14 280-299 154 154 154 258 258 258 F F F 15 300-319 153 153 153 256 256 256 F F F 16 320-339 151 151 151 250 250 250 F F F 17 340-359 148 148 148 247 247 247 F F F 18 360-379 148 148 148 247 247 247 F F F 19 380-399 154 154 154 258 258 258 F F F 20 400-419 154 154 154 262 262 262 F F F 21 420-439 154 154 154 260 260 260 F F F 22 440-459 154 154 154 260 260 260 T T T 23 460-479 153 153 153 256 256 256 F F F 24 480-499 150 150 150 249 249 249 T T T 25 500-519 152 152 152 252 252 252 F F F 26 520-539 152 152 152 252 252 252 T T T 27 540-559 154 154 154 262 262 262 T T T 28 560-579 152 152 152 252 252 252 T T T 29 580-599 150 150 150 249 249 249 F F F Max gen: 154, Max flush: 262 A Difference in extent metadata is expected here On loop 77 Verifying data now No change after verify Loop: 77 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 151 151 151 250 250 250 F F F 1 020-039 152 152 152 253 253 253 F F F 2 040-059 152 152 152 255 255 255 F F F 3 060-079 149 149 149 248 248 248 F F F 4 080-099 147 147 147 242 242 242 F F F 5 100-119 150 150 150 249 249 249 F F F 6 120-139 148 148 148 246 246 246 F F F 7 140-159 154 154 154 259 259 259 F F F 8 160-179 155 155 155 263 263 263 F F F 9 180-199 154 154 154 262 262 262 F F F 10 200-219 154 154 154 262 262 262 F F F 11 220-239 153 153 153 256 256 256 F F F 12 240-259 152 152 152 252 252 252 F F F 13 260-279 154 154 154 262 262 262 F F F 14 280-299 154 154 154 258 258 258 F F F 15 300-319 153 153 153 256 256 256 F F F 16 320-339 151 151 151 250 250 250 F F F 17 340-359 148 148 148 247 247 247 F F F 18 360-379 148 148 148 247 247 247 F F F 19 380-399 154 154 154 258 258 258 F F F 20 400-419 154 154 154 262 262 262 F F F 21 420-439 154 154 154 260 260 260 F F F 22 440-459 155 155 155 263 263 263 F F F 23 460-479 153 153 153 256 256 256 F F F 24 480-499 155 155 155 263 263 263 F F F 25 500-519 152 152 152 252 252 252 F F F 26 520-539 155 155 155 263 263 263 F F F 27 540-559 155 155 155 263 263 263 F F F 28 560-579 155 155 155 263 263 263 F F F 29 580-599 150 150 150 249 249 249 F F F Max gen: 155, Max flush: 263 Begin loop 78 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:04.642230048Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1804,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:56:04.68485854Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1804,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:04.685250461Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1804,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:04.717490275Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1804,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:04.718007571Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1804} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"3818f0fc76fe787c6cc412af30fd59b930203885\",\n git_commit_timestamp: \"2024-11-01T16:08:30.000000000Z\",\n git_branch: \"renovate/hyper-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:04.718053387Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1804} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:04.71807178Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1804} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:04.718090962Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1804} {"msg":"Crucible 340c88f4-5bfe-42b9-84d2-de2d69884faa has session id: 11667966-66b9-4d00-9781-0967bd4a180d","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:04.718548241Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1804,"session_id":"11667966-66b9-4d00-9781-0967bd4a180d"} {"msg":"Upstairs opts: Upstairs UUID: 340c88f4-5bfe-42b9-84d2-de2d69884faa, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:04.718589756Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1804,"session_id":"11667966-66b9-4d00-9781-0967bd4a180d"} {"msg":"Crucible stats registered with UUID: 340c88f4-5bfe-42b9-84d2-de2d69884faa","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:04.71861488Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1804,"session_id":"11667966-66b9-4d00-9781-0967bd4a180d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:04.749767304Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1804,"task":"crutest"} {"msg":"The guest has requested activation with gen:156","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:04.749800978Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1804} {"msg":"340c88f4-5bfe-42b9-84d2-de2d69884faa active request set","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:04.749850494Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1804,"session_id":"11667966-66b9-4d00-9781-0967bd4a180d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:04.749870367Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1804,"client":"0","":"downstairs","session_id":"11667966-66b9-4d00-9781-0967bd4a180d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:04.749899781Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1804,"client":"0","":"downstairs","session_id":"11667966-66b9-4d00-9781-0967bd4a180d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:04.749917923Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1804,"client":"1","":"downstairs","session_id":"11667966-66b9-4d00-9781-0967bd4a180d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:04.749934815Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1804,"client":"1","":"downstairs","session_id":"11667966-66b9-4d00-9781-0967bd4a180d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:04.749955798Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1804,"client":"2","":"downstairs","session_id":"11667966-66b9-4d00-9781-0967bd4a180d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:04.74997315Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1804,"client":"2","":"downstairs","session_id":"11667966-66b9-4d00-9781-0967bd4a180d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:04.749989842Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1804,"":"io task","client":"2","":"downstairs","session_id":"11667966-66b9-4d00-9781-0967bd4a180d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:04.750007404Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1804,"":"io task","client":"0","":"downstairs","session_id":"11667966-66b9-4d00-9781-0967bd4a180d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:04.750037879Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1804,"":"io task","client":"1","":"downstairs","session_id":"11667966-66b9-4d00-9781-0967bd4a180d"} {"msg":"ds_connection connected from Ok(127.0.0.1:34843)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:04.750084294Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1804,"":"io task","client":"2","":"downstairs","session_id":"11667966-66b9-4d00-9781-0967bd4a180d"} {"msg":"ds_connection connected from Ok(127.0.0.1:57797)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:04.750122829Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1804,"":"io task","client":"0","":"downstairs","session_id":"11667966-66b9-4d00-9781-0967bd4a180d"} {"msg":"ds_connection connected from Ok(127.0.0.1:47766)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:04.750159004Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1804,"":"io task","client":"1","":"downstairs","session_id":"11667966-66b9-4d00-9781-0967bd4a180d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:04.750837891Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1804,"client":"1","":"downstairs","session_id":"11667966-66b9-4d00-9781-0967bd4a180d"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:04.750866405Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1804,"client":"1","":"downstairs","session_id":"11667966-66b9-4d00-9781-0967bd4a180d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:04.751027906Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1804,"client":"0","":"downstairs","session_id":"11667966-66b9-4d00-9781-0967bd4a180d"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:04.751050559Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1804,"client":"0","":"downstairs","session_id":"11667966-66b9-4d00-9781-0967bd4a180d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:04.751071212Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1804,"client":"2","":"downstairs","session_id":"11667966-66b9-4d00-9781-0967bd4a180d"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:04.751084753Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1804,"client":"2","":"downstairs","session_id":"11667966-66b9-4d00-9781-0967bd4a180d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:04.751605Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1804,"client":"2","":"downstairs","session_id":"11667966-66b9-4d00-9781-0967bd4a180d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:04.751637854Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1804,"client":"1","":"downstairs","session_id":"11667966-66b9-4d00-9781-0967bd4a180d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:04.751896908Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1804,"client":"0","":"downstairs","session_id":"11667966-66b9-4d00-9781-0967bd4a180d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:04.752171303Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1804,"client":"1","":"downstairs","session_id":"11667966-66b9-4d00-9781-0967bd4a180d"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:04.752194886Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1804,"client":"1","":"downstairs","session_id":"11667966-66b9-4d00-9781-0967bd4a180d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:04.752209498Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1804,"session_id":"11667966-66b9-4d00-9781-0967bd4a180d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:04.752269806Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1804,"client":"2","":"downstairs","session_id":"11667966-66b9-4d00-9781-0967bd4a180d"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:04.752289038Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1804,"client":"2","":"downstairs","session_id":"11667966-66b9-4d00-9781-0967bd4a180d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:04.75230274Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1804,"session_id":"11667966-66b9-4d00-9781-0967bd4a180d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:04.752373349Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1804,"client":"0","":"downstairs","session_id":"11667966-66b9-4d00-9781-0967bd4a180d"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:04.752391982Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1804,"client":"0","":"downstairs","session_id":"11667966-66b9-4d00-9781-0967bd4a180d"} {"msg":"[0]R flush_numbers[0..12]: [250, 253, 255, 248, 242, 249, 246, 259, 263, 262, 262, 256]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:04.752406193Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1804,"":"downstairs","":"downstairs","session_id":"11667966-66b9-4d00-9781-0967bd4a180d"} {"msg":"[0]R generation[0..12]: [151, 152, 152, 149, 147, 150, 148, 154, 155, 154, 154, 153]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:04.752420245Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1804,"":"downstairs","":"downstairs","session_id":"11667966-66b9-4d00-9781-0967bd4a180d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:04.752434397Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1804,"":"downstairs","":"downstairs","session_id":"11667966-66b9-4d00-9781-0967bd4a180d"} {"msg":"[1]R flush_numbers[0..12]: [250, 253, 255, 248, 242, 249, 246, 259, 263, 262, 262, 256]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:04.752448159Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1804,"":"downstairs","":"downstairs","session_id":"11667966-66b9-4d00-9781-0967bd4a180d"} {"msg":"[1]R generation[0..12]: [151, 152, 152, 149, 147, 150, 148, 154, 155, 154, 154, 153]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:04.752461831Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1804,"":"downstairs","":"downstairs","session_id":"11667966-66b9-4d00-9781-0967bd4a180d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:04.752480283Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1804,"":"downstairs","":"downstairs","session_id":"11667966-66b9-4d00-9781-0967bd4a180d"} {"msg":"[2]R flush_numbers[0..12]: [250, 253, 255, 248, 242, 249, 246, 259, 263, 262, 262, 256]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:04.752495625Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1804,"":"downstairs","":"downstairs","session_id":"11667966-66b9-4d00-9781-0967bd4a180d"} {"msg":"[2]R generation[0..12]: [151, 152, 152, 149, 147, 150, 148, 154, 155, 154, 154, 153]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:04.752509617Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1804,"":"downstairs","":"downstairs","session_id":"11667966-66b9-4d00-9781-0967bd4a180d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:04.75253461Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1804,"":"downstairs","":"downstairs","session_id":"11667966-66b9-4d00-9781-0967bd4a180d"} {"msg":"Max found gen is 156","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:04.752560273Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1804,"":"downstairs","":"downstairs","session_id":"11667966-66b9-4d00-9781-0967bd4a180d"} {"msg":"Generation requested: 156 >= found:156","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:04.752584887Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1804,"":"downstairs","":"downstairs","session_id":"11667966-66b9-4d00-9781-0967bd4a180d"} {"msg":"Next flush: 264","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-01T16:56:04.752606069Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1804,"":"downstairs","":"downstairs","session_id":"11667966-66b9-4d00-9781-0967bd4a180d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:04.752641064Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1804,"":"downstairs","":"downstairs","session_id":"11667966-66b9-4d00-9781-0967bd4a180d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:04.752667877Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1804,"session_id":"11667966-66b9-4d00-9781-0967bd4a180d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:04.752681809Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1804,"client":"0","":"downstairs","session_id":"11667966-66b9-4d00-9781-0967bd4a180d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:04.752695421Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1804,"client":"1","":"downstairs","session_id":"11667966-66b9-4d00-9781-0967bd4a180d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:04.752716164Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1804,"client":"2","":"downstairs","session_id":"11667966-66b9-4d00-9781-0967bd4a180d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:04.752741037Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1804,"session_id":"11667966-66b9-4d00-9781-0967bd4a180d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:04.75276641Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1804,"session_id":"11667966-66b9-4d00-9781-0967bd4a180d"} {"msg":"340c88f4-5bfe-42b9-84d2-de2d69884faa is now active with session: 11667966-66b9-4d00-9781-0967bd4a180d","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:04.752785302Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1804,"session_id":"11667966-66b9-4d00-9781-0967bd4a180d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:04.752802525Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1804,"session_id":"11667966-66b9-4d00-9781-0967bd4a180d"} {"msg":"The guest has finished waiting for activation with:156","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:04.752825267Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1804} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 2 len 16384 data: 9 9 11 10 02/30 Read block 44 len 4096 03/30 Read block 364 len 24576 04/30 Write block 497 len 8192 data: 13 15 05/30 Read block 519 len 4096 06/30 Read block 367 len 16384 07/30 Read block 187 len 8192 08/30 Read block 241 len 16384 09/30 Read block 15 len 32768 10/30 Flush 11/30 Write block 586 len 24576 data: 15 18 14 14 12 10 12/30 Read block 369 len 32768 13/30 Write block 398 len 36864 data: 15 13 11 10 11 11 10 7 8 14/30 Read block 118 len 12288 15/30 Read block 114 len 4096 16/30 Write block 334 len 28672 data: 9 11 9 10 10 8 9 17/30 Read block 146 len 8192 18/30 Read block 258 len 12288 19/30 Write block 148 len 12288 data: 10 14 17 20/30 Read block 559 len 40960 21/30 Read block 271 len 32768 22/30 Read block 47 len 28672 23/30 Read block 546 len 36864 24/30 Read block 276 len 36864 25/30 Write block 201 len 36864 data: 13 12 13 12 12 13 16 14 12 26/30 Read block 589 len 28672 27/30 Write block 372 len 8192 data: 7 10 28/30 Read block 484 len 36864 29/30 Write block 326 len 16384 data: 7 6 4 5 30/30 Write block 357 len 32768 data: 15 14 11 10 10 7 7 8 ---------------------------------------------------------------- Crucible gen:156 GIO:true work queues: Upstairs:0 downstairs:20 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1010 Acked Write 6 Done Done Done false 1011 Acked Read 8 Done Done Done false 1012 Acked Write 9 Done Done Done false 1013 Acked Read 3 Done Done Done false 1014 Acked Read 1 Done Done Done false 1015 Acked Write 7 Done Done Done false 1016 Acked Read 2 Done Done Done false 1017 Acked Read 3 Done Done Done false 1018 Acked Write 3 Done Done Done false 1019 Acked Read 10 Done Done Done false 1020 Acked Read 8 Done Done Done false 1021 Acked Read 7 Done Done Done false 1022 Acked Read 9 Done Done Done false 1023 Acked Read 9 Done Done Done false 1024 Acked Write 9 Done Done Done false 1025 Acked Read 7 Done Done Done false 1026 Acked Write 2 Done Done Done false 1027 Acked Read 9 Done Done Done false 1028 Acked Write 4 Done Done Done false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 19 19 19 57 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1009 1009 1009 Downstairs last five completed: 1009 1008 1007 1006 1005 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:56:04.774273111Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1804} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 156 156 156 264 264 264 F F F 1 020-039 152 152 152 253 253 253 F F F 2 040-059 152 152 152 255 255 255 F F F 3 060-079 149 149 149 248 248 248 F F F 4 080-099 147 147 147 242 242 242 F F F 5 100-119 150 150 150 249 249 249 F F F 6 120-139 148 148 148 246 246 246 F F F 7 140-159 154 154 154 259 259 259 T T T 8 160-179 155 155 155 263 263 263 F F F 9 180-199 154 154 154 262 262 262 F F F 10 200-219 154 154 154 262 262 262 T T T 11 220-239 153 153 153 256 256 256 F F F 12 240-259 152 152 152 252 252 252 F F F 13 260-279 154 154 154 262 262 262 F F F 14 280-299 154 154 154 258 258 258 F F F 15 300-319 153 153 153 256 256 256 F F F 16 320-339 151 151 151 250 250 250 T T T 17 340-359 148 148 148 247 247 247 T T T 18 360-379 148 148 148 247 247 247 T T T 19 380-399 154 154 154 258 258 258 T T T 20 400-419 154 154 154 262 262 262 T T T 21 420-439 154 154 154 260 260 260 F F F 22 440-459 155 155 155 263 263 263 F F F 23 460-479 153 153 153 256 256 256 F F F 24 480-499 156 156 156 264 264 264 F F F 25 500-519 152 152 152 252 252 252 F F F 26 520-539 155 155 155 263 263 263 F F F 27 540-559 155 155 155 263 263 263 F F F 28 560-579 155 155 155 263 263 263 F F F 29 580-599 150 150 150 249 249 249 T T T Max gen: 156, Max flush: 264 A Difference in extent metadata is expected here On loop 78 Verifying data now No change after verify Loop: 78 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 156 156 156 264 264 264 F F F 1 020-039 152 152 152 253 253 253 F F F 2 040-059 152 152 152 255 255 255 F F F 3 060-079 149 149 149 248 248 248 F F F 4 080-099 147 147 147 242 242 242 F F F 5 100-119 150 150 150 249 249 249 F F F 6 120-139 148 148 148 246 246 246 F F F 7 140-159 157 157 157 265 265 265 F F F 8 160-179 155 155 155 263 263 263 F F F 9 180-199 154 154 154 262 262 262 F F F 10 200-219 157 157 157 265 265 265 F F F 11 220-239 153 153 153 256 256 256 F F F 12 240-259 152 152 152 252 252 252 F F F 13 260-279 154 154 154 262 262 262 F F F 14 280-299 154 154 154 258 258 258 F F F 15 300-319 153 153 153 256 256 256 F F F 16 320-339 157 157 157 265 265 265 F F F 17 340-359 157 157 157 265 265 265 F F F 18 360-379 157 157 157 265 265 265 F F F 19 380-399 157 157 157 265 265 265 F F F 20 400-419 157 157 157 265 265 265 F F F 21 420-439 154 154 154 260 260 260 F F F 22 440-459 155 155 155 263 263 263 F F F 23 460-479 153 153 153 256 256 256 F F F 24 480-499 156 156 156 264 264 264 F F F 25 500-519 152 152 152 252 252 252 F F F 26 520-539 155 155 155 263 263 263 F F F 27 540-559 155 155 155 263 263 263 F F F 28 560-579 155 155 155 263 263 263 F F F 29 580-599 157 157 157 265 265 265 F F F Max gen: 157, Max flush: 265 Begin loop 79 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:09.153508975Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1812,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:09.197168961Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1812,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:09.197828245Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1812} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"3818f0fc76fe787c6cc412af30fd59b930203885\",\n git_commit_timestamp: \"2024-11-01T16:08:30.000000000Z\",\n git_branch: \"renovate/hyper-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:09.197884553Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1812} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:09.197906616Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1812} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:09.197930179Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1812} {"msg":"Crucible 43ec2235-4f45-46cb-810e-91d199650b24 has session id: 2c741545-2050-4d61-846e-ab484730c6e6","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:09.19833316Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1812,"session_id":"2c741545-2050-4d61-846e-ab484730c6e6"} {"msg":"Upstairs opts: Upstairs UUID: 43ec2235-4f45-46cb-810e-91d199650b24, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:09.198377886Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1812,"session_id":"2c741545-2050-4d61-846e-ab484730c6e6"} {"msg":"Crucible stats registered with UUID: 43ec2235-4f45-46cb-810e-91d199650b24","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:09.198418161Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1812,"session_id":"2c741545-2050-4d61-846e-ab484730c6e6"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:09.229309392Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1812,"task":"crutest"} {"msg":"The guest has requested activation with gen:158","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:09.229340286Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1812} {"msg":"43ec2235-4f45-46cb-810e-91d199650b24 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:09.229404194Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1812,"session_id":"2c741545-2050-4d61-846e-ab484730c6e6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:09.229437338Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1812,"client":"0","":"downstairs","session_id":"2c741545-2050-4d61-846e-ab484730c6e6"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:09.229469452Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1812,"client":"0","":"downstairs","session_id":"2c741545-2050-4d61-846e-ab484730c6e6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:09.229493435Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1812,"client":"1","":"downstairs","session_id":"2c741545-2050-4d61-846e-ab484730c6e6"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:09.22952493Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1812,"client":"1","":"downstairs","session_id":"2c741545-2050-4d61-846e-ab484730c6e6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:09.229553723Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1812,"client":"2","":"downstairs","session_id":"2c741545-2050-4d61-846e-ab484730c6e6"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:09.229575886Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1812,"client":"2","":"downstairs","session_id":"2c741545-2050-4d61-846e-ab484730c6e6"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:09.229593018Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1812,"":"io task","client":"2","":"downstairs","session_id":"2c741545-2050-4d61-846e-ab484730c6e6"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:09.229618001Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1812,"":"io task","client":"0","":"downstairs","session_id":"2c741545-2050-4d61-846e-ab484730c6e6"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:09.229649246Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1812,"":"io task","client":"1","":"downstairs","session_id":"2c741545-2050-4d61-846e-ab484730c6e6"} {"msg":"ds_connection connected from Ok(127.0.0.1:59854)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:09.22968221Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1812,"":"io task","client":"2","":"downstairs","session_id":"2c741545-2050-4d61-846e-ab484730c6e6"} {"msg":"ds_connection connected from Ok(127.0.0.1:46778)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:09.229713894Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1812,"":"io task","client":"0","":"downstairs","session_id":"2c741545-2050-4d61-846e-ab484730c6e6"} {"msg":"ds_connection connected from Ok(127.0.0.1:55729)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:09.229845591Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1812,"":"io task","client":"1","":"downstairs","session_id":"2c741545-2050-4d61-846e-ab484730c6e6"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:09.230429026Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1812,"client":"2","":"downstairs","session_id":"2c741545-2050-4d61-846e-ab484730c6e6"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:09.230457099Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1812,"client":"2","":"downstairs","session_id":"2c741545-2050-4d61-846e-ab484730c6e6"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:09.230471641Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1812,"client":"1","":"downstairs","session_id":"2c741545-2050-4d61-846e-ab484730c6e6"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:09.230493204Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1812,"client":"1","":"downstairs","session_id":"2c741545-2050-4d61-846e-ab484730c6e6"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:09.230520928Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1812,"client":"0","":"downstairs","session_id":"2c741545-2050-4d61-846e-ab484730c6e6"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:09.230587636Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1812,"client":"0","":"downstairs","session_id":"2c741545-2050-4d61-846e-ab484730c6e6"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:09.231030813Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1812,"client":"2","":"downstairs","session_id":"2c741545-2050-4d61-846e-ab484730c6e6"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:09.231177632Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1812,"client":"1","":"downstairs","session_id":"2c741545-2050-4d61-846e-ab484730c6e6"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:09.231364887Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1812,"client":"0","":"downstairs","session_id":"2c741545-2050-4d61-846e-ab484730c6e6"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:09.23162122Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1812,"client":"2","":"downstairs","session_id":"2c741545-2050-4d61-846e-ab484730c6e6"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:09.231643042Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1812,"client":"2","":"downstairs","session_id":"2c741545-2050-4d61-846e-ab484730c6e6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:09.231657474Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1812,"session_id":"2c741545-2050-4d61-846e-ab484730c6e6"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:09.231756237Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1812,"client":"1","":"downstairs","session_id":"2c741545-2050-4d61-846e-ab484730c6e6"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:09.231784161Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1812,"client":"1","":"downstairs","session_id":"2c741545-2050-4d61-846e-ab484730c6e6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:09.231798903Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1812,"session_id":"2c741545-2050-4d61-846e-ab484730c6e6"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:09.231955383Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1812,"client":"0","":"downstairs","session_id":"2c741545-2050-4d61-846e-ab484730c6e6"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:09.231975295Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1812,"client":"0","":"downstairs","session_id":"2c741545-2050-4d61-846e-ab484730c6e6"} {"msg":"[0]R flush_numbers[0..12]: [264, 253, 255, 248, 242, 249, 246, 265, 263, 262, 265, 256]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:09.231989877Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1812,"":"downstairs","":"downstairs","session_id":"2c741545-2050-4d61-846e-ab484730c6e6"} {"msg":"[0]R generation[0..12]: [156, 152, 152, 149, 147, 150, 148, 157, 155, 154, 157, 153]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:09.232003789Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1812,"":"downstairs","":"downstairs","session_id":"2c741545-2050-4d61-846e-ab484730c6e6"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:09.232017841Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1812,"":"downstairs","":"downstairs","session_id":"2c741545-2050-4d61-846e-ab484730c6e6"} {"msg":"[1]R flush_numbers[0..12]: [264, 253, 255, 248, 242, 249, 246, 265, 263, 262, 265, 256]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:09.232032053Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1812,"":"downstairs","":"downstairs","session_id":"2c741545-2050-4d61-846e-ab484730c6e6"} {"msg":"[1]R generation[0..12]: [156, 152, 152, 149, 147, 150, 148, 157, 155, 154, 157, 153]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:09.232046144Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1812,"":"downstairs","":"downstairs","session_id":"2c741545-2050-4d61-846e-ab484730c6e6"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:09.232060176Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1812,"":"downstairs","":"downstairs","session_id":"2c741545-2050-4d61-846e-ab484730c6e6"} {"msg":"[2]R flush_numbers[0..12]: [264, 253, 255, 248, 242, 249, 246, 265, 263, 262, 265, 256]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:09.232077158Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1812,"":"downstairs","":"downstairs","session_id":"2c741545-2050-4d61-846e-ab484730c6e6"} {"msg":"[2]R generation[0..12]: [156, 152, 152, 149, 147, 150, 148, 157, 155, 154, 157, 153]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:09.23209122Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1812,"":"downstairs","":"downstairs","session_id":"2c741545-2050-4d61-846e-ab484730c6e6"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:09.232105522Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1812,"":"downstairs","":"downstairs","session_id":"2c741545-2050-4d61-846e-ab484730c6e6"} {"msg":"Max found gen is 158","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:09.232118634Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1812,"":"downstairs","":"downstairs","session_id":"2c741545-2050-4d61-846e-ab484730c6e6"} {"msg":"Generation requested: 158 >= found:158","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:09.232132055Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1812,"":"downstairs","":"downstairs","session_id":"2c741545-2050-4d61-846e-ab484730c6e6"} {"msg":"Next flush: 266","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:09.232145327Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1812,"":"downstairs","":"downstairs","session_id":"2c741545-2050-4d61-846e-ab484730c6e6"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:09.232158579Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1812,"":"downstairs","":"downstairs","session_id":"2c741545-2050-4d61-846e-ab484730c6e6"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:09.23217184Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1812,"session_id":"2c741545-2050-4d61-846e-ab484730c6e6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:09.232183922Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1812,"client":"0","":"downstairs","session_id":"2c741545-2050-4d61-846e-ab484730c6e6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO 2024-11-01T16:56:09.232197204Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1812,"client":"1","":"downstairs","session_id":"2c741545-2050-4d61-846e-ab484730c6e6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:09.232219517Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1812,"client":"2","":"downstairs","session_id":"2c741545-2050-4d61-846e-ab484730c6e6"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:09.232232828Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1812,"session_id":"2c741545-2050-4d61-846e-ab484730c6e6"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:09.23224516Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1812,"session_id":"2c741545-2050-4d61-846e-ab484730c6e6"} {"msg":"43ec2235-4f45-46cb-810e-91d199650b24 is now active with session: 2c741545-2050-4d61-846e-ab484730c6e6","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:09.232257951Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1812,"session_id":"2c741545-2050-4d61-846e-ab484730c6e6"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:09.232270063Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1812,"session_id":"2c741545-2050-4d61-846e-ab484730c6e6"} {"msg":"The guest has finished waiting for activation with:158","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:09.232284715Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1812} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 16 len 4096 data: 14 02/30 Write block 17 len 40960 data: 14 13 13 11 6 7 10 14 16 14 03/30 Write block 99 len 36864 data: 19 20 19 15 13 13 14 16 13 04/30 Flush 05/30 Write block 560 len 16384 data: 19 19 19 21 06/30 Read block 395 len 32768 07/30 Write block 129 len 28672 data: 16 18 18 16 16 12 10 08/30 Write block 291 len 32768 data: 10 13 13 15 13 13 13 12 09/30 Read block 476 len 16384 10/30 Write block 334 len 40960 data: 10 12 10 11 11 9 10 5 6 8 11/30 Write block 105 len 20480 data: 15 17 14 13 13 12/30 Flush 13/30 Write block 22 len 32768 data: 8 11 15 17 15 16 18 15 14/30 Write block 277 len 16384 data: 13 11 11 11 15/30 Read block 203 len 36864 16/30 Read block 267 len 16384 17/30 Read block 442 len 4096 18/30 Write block 210 len 16384 data: 14 15 15 15 19/30 Write block 330 len 16384 data: 6 7 7 9 20/30 Flush 21/30 Write block 86 len 4096 data: 16 22/30 Read block 446 len 16384 23/30 Read block 95 len 36864 24/30 Write block 464 len 16384 data: 9 9 12 12 25/30 Flush 26/30 Flush 27/30 Write block 26 len 4096 data: 16 28/30 Write block 271 len 4096 data: 20 29/30 Read block 165 len 8192 30/30 Read block 85 len 32768 ---------------------------------------------------------------- Crucible gen:158 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 1 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Read 8 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 0 2 Done 3 3 4 10 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:56:09.261256248Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1812} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 158 158 158 266 266 266 F F F 1 020-039 158 158 158 268 268 268 T T T 2 040-059 152 152 152 255 255 255 F F F 3 060-079 149 149 149 248 248 248 F F F 4 080-099 158 158 158 269 269 269 F F F 5 100-119 158 158 158 267 267 267 F F F 6 120-139 158 158 158 267 267 267 F F F 7 140-159 157 157 157 265 265 265 F F F 8 160-179 155 155 155 263 263 263 F F F 9 180-199 154 154 154 262 262 262 F F F 10 200-219 158 158 158 268 268 268 F F F 11 220-239 153 153 153 256 256 256 F F F 12 240-259 152 152 152 252 252 252 F F F 13 260-279 158 158 158 268 268 268 T T T 14 280-299 158 158 158 268 268 268 F F F 15 300-319 153 153 153 256 256 256 F F F 16 320-339 158 158 158 268 268 268 F F F 17 340-359 158 158 158 267 267 267 F F F 18 360-379 157 157 157 265 265 265 F F F 19 380-399 157 157 157 265 265 265 F F F 20 400-419 157 157 157 265 265 265 F F F 21 420-439 154 154 154 260 260 260 F F F 22 440-459 155 155 155 263 263 263 F F F 23 460-479 158 158 158 269 269 269 F F F 24 480-499 156 156 156 264 264 264 F F F 25 500-519 152 152 152 252 252 252 F F F 26 520-539 155 155 155 263 263 263 F F F 27 540-559 155 155 155 263 263 263 F F F 28 560-579 158 158 158 267 267 267 F F F 29 580-599 157 157 157 265 265 265 F F F Max gen: 158, Max flush: 269 A Difference in extent metadata is expected here On loop 79 Verifying data now No change after verify Loop: 79 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 158 158 158 266 266 266 F F F 1 020-039 159 159 159 270 270 270 F F F 2 040-059 152 152 152 255 255 255 F F F 3 060-079 149 149 149 248 248 248 F F F 4 080-099 158 158 158 269 269 269 F F F 5 100-119 158 158 158 267 267 267 F F F 6 120-139 158 158 158 267 267 267 F F F 7 140-159 157 157 157 265 265 265 F F F 8 160-179 155 155 155 263 263 263 F F F 9 180-199 154 154 154 262 262 262 F F F 10 200-219 158 158 158 268 268 268 F F F 11 220-239 153 153 153 256 256 256 F F F 12 240-259 152 152 152 252 252 252 F F F 13 260-279 159 159 159 270 270 270 F F F 14 280-299 158 158 158 268 268 268 F F F 15 300-319 153 153 153 256 256 256 F F F 16 320-339 158 158 158 268 268 268 F F F 17 340-359 158 158 158 267 267 267 F F F 18 360-379 157 157 157 265 265 265 F F F 19 380-399 157 157 157 265 265 265 F F F 20 400-419 157 157 157 265 265 265 F F F 21 420-439 154 154 154 260 260 260 F F F 22 440-459 155 155 155 263 263 263 F F F 23 460-479 158 158 158 269 269 269 F F F 24 480-499 156 156 156 264 264 264 F F F 25 500-519 152 152 152 252 252 252 F F F 26 520-539 155 155 155 263 263 263 F F F 27 540-559 155 155 155 263 263 263 F F F 28 560-579 158 158 158 267 267 267 F F F 29 580-599 157 157 157 265 265 265 F F F Max gen: 159, Max flush: 270 Begin loop 80 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:13.525659296Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1820,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:13.569723423Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1820,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:13.570287386Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1820} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"3818f0fc76fe787c6cc412af30fd59b930203885\",\n git_commit_timestamp: \"2024-11-01T16:08:30.000000000Z\",\n git_branch: \"renovate/hyper-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:13.570328701Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1820} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:13.570344903Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1820} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:13.570359625Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1820} {"msg":"Crucible 476987d3-1e39-4550-a1bc-15bbbd27a6c5 has session id: f27e64bd-1d73-422a-a570-bcf75080c136","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:13.570729873Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1820,"session_id":"f27e64bd-1d73-422a-a570-bcf75080c136"} {"msg":"Upstairs opts: Upstairs UUID: 476987d3-1e39-4550-a1bc-15bbbd27a6c5, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:13.570768118Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1820,"session_id":"f27e64bd-1d73-422a-a570-bcf75080c136"} {"msg":"Crucible stats registered with UUID: 476987d3-1e39-4550-a1bc-15bbbd27a6c5","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:13.570793901Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1820,"session_id":"f27e64bd-1d73-422a-a570-bcf75080c136"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:13.601312544Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1820,"task":"crutest"} {"msg":"The guest has requested activation with gen:160","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:13.601346678Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1820} {"msg":"476987d3-1e39-4550-a1bc-15bbbd27a6c5 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:13.601397044Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1820,"session_id":"f27e64bd-1d73-422a-a570-bcf75080c136"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:13.601417497Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1820,"client":"0","":"downstairs","session_id":"f27e64bd-1d73-422a-a570-bcf75080c136"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:13.601447291Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1820,"client":"0","":"downstairs","session_id":"f27e64bd-1d73-422a-a570-bcf75080c136"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:13.601465153Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1820,"client":"1","":"downstairs","session_id":"f27e64bd-1d73-422a-a570-bcf75080c136"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:13.601482265Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1820,"client":"1","":"downstairs","session_id":"f27e64bd-1d73-422a-a570-bcf75080c136"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:13.601505748Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1820,"client":"2","":"downstairs","session_id":"f27e64bd-1d73-422a-a570-bcf75080c136"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:13.601536582Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1820,"client":"2","":"downstairs","session_id":"f27e64bd-1d73-422a-a570-bcf75080c136"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:13.601567616Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1820,"":"io task","client":"2","":"downstairs","session_id":"f27e64bd-1d73-422a-a570-bcf75080c136"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:13.601602461Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1820,"":"io task","client":"0","":"downstairs","session_id":"f27e64bd-1d73-422a-a570-bcf75080c136"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:13.601639295Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1820,"":"io task","client":"1","":"downstairs","session_id":"f27e64bd-1d73-422a-a570-bcf75080c136"} {"msg":"ds_connection connected from Ok(127.0.0.1:33348)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:13.601680371Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1820,"":"io task","client":"2","":"downstairs","session_id":"f27e64bd-1d73-422a-a570-bcf75080c136"} {"msg":"ds_connection connected from Ok(127.0.0.1:47052)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:13.601737218Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1820,"":"io task","client":"0","":"downstairs","session_id":"f27e64bd-1d73-422a-a570-bcf75080c136"} {"msg":"ds_connection connected from Ok(127.0.0.1:51456)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:13.601900179Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1820,"":"io task","client":"1","":"downstairs","session_id":"f27e64bd-1d73-422a-a570-bcf75080c136"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:13.60244791Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1820,"client":"1","":"downstairs","session_id":"f27e64bd-1d73-422a-a570-bcf75080c136"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:13.602475893Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1820,"client":"1","":"downstairs","session_id":"f27e64bd-1d73-422a-a570-bcf75080c136"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:13.602490285Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1820,"client":"2","":"downstairs","session_id":"f27e64bd-1d73-422a-a570-bcf75080c136"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:13.602503867Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1820,"client":"2","":"downstairs","session_id":"f27e64bd-1d73-422a-a570-bcf75080c136"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:13.602517169Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1820,"client":"0","":"downstairs","session_id":"f27e64bd-1d73-422a-a570-bcf75080c136"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:13.602531041Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1820,"client":"0","":"downstairs","session_id":"f27e64bd-1d73-422a-a570-bcf75080c136"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:13.603138009Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1820,"client":"0","":"downstairs","session_id":"f27e64bd-1d73-422a-a570-bcf75080c136"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:13.603277567Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1820,"client":"1","":"downstairs","session_id":"f27e64bd-1d73-422a-a570-bcf75080c136"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:13.603319442Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1820,"client":"2","":"downstairs","session_id":"f27e64bd-1d73-422a-a570-bcf75080c136"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:13.603667867Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1820,"client":"0","":"downstairs","session_id":"f27e64bd-1d73-422a-a570-bcf75080c136"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:13.60369463Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1820,"client":"0","":"downstairs","session_id":"f27e64bd-1d73-422a-a570-bcf75080c136"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:13.603709922Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1820,"session_id":"f27e64bd-1d73-422a-a570-bcf75080c136"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:13.603796813Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1820,"client":"1","":"downstairs","session_id":"f27e64bd-1d73-422a-a570-bcf75080c136"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:13.603817356Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1820,"client":"1","":"downstairs","session_id":"f27e64bd-1d73-422a-a570-bcf75080c136"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:13.603831068Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1820,"session_id":"f27e64bd-1d73-422a-a570-bcf75080c136"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:13.603907208Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1820,"client":"2","":"downstairs","session_id":"f27e64bd-1d73-422a-a570-bcf75080c136"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:13.6039267Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1820,"client":"2","":"downstairs","session_id":"f27e64bd-1d73-422a-a570-bcf75080c136"} {"msg":"[0]R flush_numbers[0..12]: [266, 270, 255, 248, 269, 267, 267, 265, 263, 262, 268, 256]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:13.603941072Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1820,"":"downstairs","":"downstairs","session_id":"f27e64bd-1d73-422a-a570-bcf75080c136"} {"msg":"[0]R generation[0..12]: [158, 159, 152, 149, 158, 158, 158, 157, 155, 154, 158, 153]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:13.603955104Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1820,"":"downstairs","":"downstairs","session_id":"f27e64bd-1d73-422a-a570-bcf75080c136"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:13.603969106Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1820,"":"downstairs","":"downstairs","session_id":"f27e64bd-1d73-422a-a570-bcf75080c136"} {"msg":"[1]R flush_numbers[0..12]: [266, 270, 255, 248, 269, 267, 267, 265, 263, 262, 268, 256]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:13.603983108Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1820,"":"downstairs","":"downstairs","session_id":"f27e64bd-1d73-422a-a570-bcf75080c136"} {"msg":"[1]R generation[0..12]: [158, 159, 152, 149, 158, 158, 158, 157, 155, 154, 158, 153]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:13.603996839Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1820,"":"downstairs","":"downstairs","session_id":"f27e64bd-1d73-422a-a570-bcf75080c136"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:13.604011981Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1820,"":"downstairs","":"downstairs","session_id":"f27e64bd-1d73-422a-a570-bcf75080c136"} {"msg":"[2]R flush_numbers[0..12]: [266, 270, 255, 248, 269, 267, 267, 265, 263, 262, 268, 256]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:13.604037304Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1820,"":"downstairs","":"downstairs","session_id":"f27e64bd-1d73-422a-a570-bcf75080c136"} {"msg":"[2]R generation[0..12]: [158, 159, 152, 149, 158, 158, 158, 157, 155, 154, 158, 153]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:13.604052506Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1820,"":"downstairs","":"downstairs","session_id":"f27e64bd-1d73-422a-a570-bcf75080c136"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:13.604066688Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1820,"":"downstairs","":"downstairs","session_id":"f27e64bd-1d73-422a-a570-bcf75080c136"} {"msg":"Max found gen is 160","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:13.604083401Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1820,"":"downstairs","":"downstairs","session_id":"f27e64bd-1d73-422a-a570-bcf75080c136"} {"msg":"Generation requested: 160 >= found:160","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:13.604097002Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1820,"":"downstairs","":"downstairs","session_id":"f27e64bd-1d73-422a-a570-bcf75080c136"} {"msg":"Next flush: 271","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:13.604109964Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1820,"":"downstairs","":"downstairs","session_id":"f27e64bd-1d73-422a-a570-bcf75080c136"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:13.604122846Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1820,"":"downstairs","":"downstairs","session_id":"f27e64bd-1d73-422a-a570-bcf75080c136"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:13.604135917Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1820,"session_id":"f27e64bd-1d73-422a-a570-bcf75080c136"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","levelWait for a query_work_queue command to finish before sending IO ":30,"time":"2024-11-01T16:56:13.60415574Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1820,"client":"0","":"downstairs","session_id":"f27e64bd-1d73-422a-a570-bcf75080c136"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:13.604179793Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1820,"client":"1","":"downstairs","session_id":"f27e64bd-1d73-422a-a570-bcf75080c136"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:13.604205776Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1820,"client":"2","":"downstairs","session_id":"f27e64bd-1d73-422a-a570-bcf75080c136"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:13.604230809Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1820,"session_id":"f27e64bd-1d73-422a-a570-bcf75080c136"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:13.604253603Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1820,"session_id":"f27e64bd-1d73-422a-a570-bcf75080c136"} {"msg":"476987d3-1e39-4550-a1bc-15bbbd27a6c5 is now active with session: f27e64bd-1d73-422a-a570-bcf75080c136","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:13.604277746Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1820,"session_id":"f27e64bd-1d73-422a-a570-bcf75080c136"}Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:13.604303279Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1820,"session_id":"f27e64bd-1d73-422a-a570-bcf75080c136"} {"msg":"The guest has finished waiting for activation with:160","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:13.604326262Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1820} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 440 len 24576 data: 11 9 9 9 11 11 02/30 Write block 212 len 24576 data: 16 16 15 15 18 19 03/30 Read block 436 len 40960 04/30 Write block 231 len 4096 data: 8 05/30 Read block 164 len 4096 06/30 Flush 07/30 Write block 374 len 40960 data: 10 9 10 13 13 12 11 12 15 15 08/30 Write block 192 len 24576 data: 17 19 20 18 16 15 09/30 Write block 8 len 40960 data: 13 14 13 14 16 14 15 14 15 15 10/30 Read block 477 len 36864 11/30 Read block 278 len 40960 12/30 Write block 467 len 40960 data: 13 12 13 15 17 17 17 14 14 11 13/30 Read block 430 len 40960 14/30 Flush 15/30 Write block 288 len 8192 data: 8 9 16/30 Read block 507 len 36864 17/30 Flush 18/30 Write block 60 len 16384 data: 8 9 7 7 19/30 Read block 584 len 28672 20/30 Write block 180 len 12288 data: 12 9 9 21/30 Write block 105 len 20480 data: 16 18 15 14 14 22/30 Write block 23 len 28672 data: 12 16 18 17 17 19 16 23/30 Write block 83 len 40960 data: 13 12 15 17 17 17 20 21 19 17 24/30 Write block 417 len 32768 data: 10 12 10 10 12 13 13 13 25/30 Read block 242 len 16384 26/30 Write block 60 len 20480 data: 9 10 8 8 10 27/30 Read block 490 len 36864 28/30 Write block 320 len 28672 data: 15 15 14 10 10 8 8 29/30 Write block 20 len 20480 data: 12 7 9 13 17 30/30 Read block 528 len 8192 ---------------------------------------------------------------- Crucible gen:160 GIO:true work queues: Upstairs:0 downstairs:13 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1017 Acked Write 4 Done Done Done false 1018 Acked Read 7 Done Done Done false 1019 Acked Write 3 Done Done Done false 1020 Acked Write 5 Done Done Done false 1021 Acked Write 7 Done Done Done false 1022 Acked Write 10 Done Done Done false 1023 Acked Write 8 Done Done Done false 1024 Acked Read 4 Done Done Done false 1025 Acked Write 5 Done Done Done false 1026 Acked Read 9 Done Done Done false 1027 Acked Write 7 Done Done Done false 1028 Acked Write 5 Sent Done Sent false 1029 Acked Read 2 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 0 2 4 Done 11 13 11 35 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1016 1016 1016 Downstairs last five completed: 1016 1015 1014 1013 1012 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:56:13.629342635Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1820} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 160 160 160 272 272 272 F F F 1 020-039 159 159 159 270 270 270 T T T 2 040-059 152 152 152 255 255 255 F F F 3 060-079 149 149 149 248 248 248 T T T 4 080-099 158 158 158 269 269 269 T T T 5 100-119 158 158 158 267 267 267 T T T 6 120-139 158 158 158 267 267 267 F F F 7 140-159 157 157 157 265 265 265 F F F 8 160-179 155 155 155 263 263 263 F F F 9 180-199 160 160 160 272 272 272 T T T 10 200-219 160 160 160 271 271 271 F F F 11 220-239 160 160 160 271 271 271 F F F 12 240-259 152 152 152 252 252 252 F F F 13 260-279 159 159 159 270 270 270 F F F 14 280-299 160 160 160 273 273 273 F F F 15 300-319 153 153 153 256 256 256 F F F 16 320-339 158 158 158 268 268 268 T T T 17 340-359 158 158 158 267 267 267 F F F 18 360-379 160 160 160 272 272 272 F F F 19 380-399 160 160 160 272 272 272 F F F 20 400-419 157 157 157 265 265 265 T T T 21 420-439 154 154 154 260 260 260 T T T 22 440-459 160 160 160 271 271 271 F F F 23 460-479 160 160 160 272 272 272 F F F 24 480-499 156 156 156 264 264 264 F F F 25 500-519 152 152 152 252 252 252 F F F 26 520-539 155 155 155 263 263 263 F F F 27 540-559 155 155 155 263 263 263 F F F 28 560-579 158 158 158 267 267 267 F F F 29 580-599 157 157 157 265 265 265 F F F Max gen: 160, Max flush: 273 A Difference in extent metadata is expected here On loop 80 Verifying data now No change after verify Loop: 80 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 160 160 160 272 272 272 F F F 1 020-039 161 161 161 274 274 274 F F F 2 040-059 152 152 152 255 255 255 F F F 3 060-079 161 161 161 274 274 274 F F F 4 080-099 161 161 161 274 274 274 F F F 5 100-119 161 161 161 274 274 274 F F F 6 120-139 158 158 158 267 267 267 F F F 7 140-159 157 157 157 265 265 265 F F F 8 160-179 155 155 155 263 263 263 F F F 9 180-199 161 161 161 274 274 274 F F F 10 200-219 160 160 160 271 271 271 F F F 11 220-239 160 160 160 271 271 271 F F F 12 240-259 152 152 152 252 252 252 F F F 13 260-279 159 159 159 270 270 270 F F F 14 280-299 160 160 160 273 273 273 F F F 15 300-319 153 153 153 256 256 256 F F F 16 320-339 161 161 161 274 274 274 F F F 17 340-359 158 158 158 267 267 267 F F F 18 360-379 160 160 160 272 272 272 F F F 19 380-399 160 160 160 272 272 272 F F F 20 400-419 161 161 161 274 274 274 F F F 21 420-439 161 161 161 274 274 274 F F F 22 440-459 160 160 160 271 271 271 F F F 23 460-479 160 160 160 272 272 272 F F F 24 480-499 156 156 156 264 264 264 F F F 25 500-519 152 152 152 252 252 252 F F F 26 520-539 155 155 155 263 263 263 F F F 27 540-559 155 155 155 263 263 263 F F F 28 560-579 158 158 158 267 267 267 F F F 29 580-599 157 157 157 265 265 265 F F F Max gen: 161, Max flush: 274 Begin loop 81 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:17.984700881Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1828,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:56:18.027497536Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1828,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:18.027899487Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1828,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:18.060187318Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1828,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:18.060807607Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1828} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"3818f0fc76fe787c6cc412af30fd59b930203885\",\n git_commit_timestamp: \"2024-11-01T16:08:30.000000000Z\",\n git_branch: \"renovate/hyper-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:18.060850083Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1828} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:18.060866945Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1828} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:18.060884857Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1828} {"msg":"Crucible 800f814a-2743-4b94-a8a6-bd6c2a2857cc has session id: db866bac-b2bb-49eb-af09-6fa4ba3289c8","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:18.061209339Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1828,"session_id":"db866bac-b2bb-49eb-af09-6fa4ba3289c8"} {"msg":"Upstairs opts: Upstairs UUID: 800f814a-2743-4b94-a8a6-bd6c2a2857cc, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:18.061261846Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1828,"session_id":"db866bac-b2bb-49eb-af09-6fa4ba3289c8"} {"msg":"Crucible stats registered with UUID: 800f814a-2743-4b94-a8a6-bd6c2a2857cc","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:18.0612923Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1828,"session_id":"db866bac-b2bb-49eb-af09-6fa4ba3289c8"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:18.091930608Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1828,"task":"crutest"} {"msg":"The guest has requested activation with gen:162","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:18.091957311Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1828} {"msg":"800f814a-2743-4b94-a8a6-bd6c2a2857cc active request set","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:18.09202751Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1828,"session_id":"db866bac-b2bb-49eb-af09-6fa4ba3289c8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:18.092049503Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1828,"client":"0","":"downstairs","session_id":"db866bac-b2bb-49eb-af09-6fa4ba3289c8"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:18.092079397Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1828,"client":"0","":"downstairs","session_id":"db866bac-b2bb-49eb-af09-6fa4ba3289c8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:18.092097439Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1828,"client":"1","":"downstairs","session_id":"db866bac-b2bb-49eb-af09-6fa4ba3289c8"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:18.092114561Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1828,"client":"1","":"downstairs","session_id":"db866bac-b2bb-49eb-af09-6fa4ba3289c8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:18.092131183Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1828,"client":"2","":"downstairs","session_id":"db866bac-b2bb-49eb-af09-6fa4ba3289c8"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:18.092148286Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1828,"client":"2","":"downstairs","session_id":"db866bac-b2bb-49eb-af09-6fa4ba3289c8"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:18.09217698Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1828,"":"io task","client":"2","":"downstairs","session_id":"db866bac-b2bb-49eb-af09-6fa4ba3289c8"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:18.092211704Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1828,"":"io task","client":"0","":"downstairs","session_id":"db866bac-b2bb-49eb-af09-6fa4ba3289c8"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:18.092234217Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1828,"":"io task","client":"1","":"downstairs","session_id":"db866bac-b2bb-49eb-af09-6fa4ba3289c8"} {"msg":"ds_connection connected from Ok(127.0.0.1:41246)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:18.092296595Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1828,"":"io task","client":"2","":"downstairs","session_id":"db866bac-b2bb-49eb-af09-6fa4ba3289c8"} {"msg":"ds_connection connected from Ok(127.0.0.1:34390)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:18.092428222Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1828,"":"io task","client":"0","":"downstairs","session_id":"db866bac-b2bb-49eb-af09-6fa4ba3289c8"} {"msg":"ds_connection connected from Ok(127.0.0.1:40448)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:18.092476118Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1828,"":"io task","client":"1","":"downstairs","session_id":"db866bac-b2bb-49eb-af09-6fa4ba3289c8"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:18.093029149Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1828,"client":"2","":"downstairs","session_id":"db866bac-b2bb-49eb-af09-6fa4ba3289c8"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:18.093057973Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1828,"client":"2","":"downstairs","session_id":"db866bac-b2bb-49eb-af09-6fa4ba3289c8"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:18.093073715Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1828,"client":"0","":"downstairs","session_id":"db866bac-b2bb-49eb-af09-6fa4ba3289c8"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:18.093087617Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1828,"client":"0","":"downstairs","session_id":"db866bac-b2bb-49eb-af09-6fa4ba3289c8"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:18.093101248Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1828,"client":"1","":"downstairs","session_id":"db866bac-b2bb-49eb-af09-6fa4ba3289c8"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:18.0931149Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1828,"client":"1","":"downstairs","session_id":"db866bac-b2bb-49eb-af09-6fa4ba3289c8"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:18.093717068Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1828,"client":"1","":"downstairs","session_id":"db866bac-b2bb-49eb-af09-6fa4ba3289c8"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:18.093748292Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1828,"client":"2","":"downstairs","session_id":"db866bac-b2bb-49eb-af09-6fa4ba3289c8"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:18.09381442Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1828,"client":"0","":"downstairs","session_id":"db866bac-b2bb-49eb-af09-6fa4ba3289c8"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:18.094400346Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1828,"client":"1","":"downstairs","session_id":"db866bac-b2bb-49eb-af09-6fa4ba3289c8"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:18.094439591Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1828,"client":"1","":"downstairs","session_id":"db866bac-b2bb-49eb-af09-6fa4ba3289c8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:18.094467515Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1828,"session_id":"db866bac-b2bb-49eb-af09-6fa4ba3289c8"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:18.094492768Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1828,"client":"2","":"downstairs","session_id":"db866bac-b2bb-49eb-af09-6fa4ba3289c8"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:18.094516051Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1828,"client":"2","":"downstairs","session_id":"db866bac-b2bb-49eb-af09-6fa4ba3289c8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:18.094540684Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1828,"session_id":"db866bac-b2bb-49eb-af09-6fa4ba3289c8"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:18.094562837Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1828,"client":"0","":"downstairs","session_id":"db866bac-b2bb-49eb-af09-6fa4ba3289c8"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:18.09458563Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1828,"client":"0","":"downstairs","session_id":"db866bac-b2bb-49eb-af09-6fa4ba3289c8"} {"msg":"[0]R flush_numbers[0..12]: [272, 274, 255, 274, 274, 274, 267, 265, 263, 274, 271, 271]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:18.094609633Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1828,"":"downstairs","":"downstairs","session_id":"db866bac-b2bb-49eb-af09-6fa4ba3289c8"} {"msg":"[0]R generation[0..12]: [160, 161, 152, 161, 161, 161, 158, 157, 155, 161, 160, 160]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:18.094633806Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1828,"":"downstairs","":"downstairs","session_id":"db866bac-b2bb-49eb-af09-6fa4ba3289c8"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:18.094661169Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1828,"":"downstairs","":"downstairs","session_id":"db866bac-b2bb-49eb-af09-6fa4ba3289c8"} {"msg":"[1]R flush_numbers[0..12]: [272, 274, 255, 274, 274, 274, 267, 265, 263, 274, 271, 271]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:18.094687553Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1828,"":"downstairs","":"downstairs","session_id":"db866bac-b2bb-49eb-af09-6fa4ba3289c8"} {"msg":"[1]R generation[0..12]: [160, 161, 152, 161, 161, 161, 158, 157, 155, 161, 160, 160]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:18.094713356Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1828,"":"downstairs","":"downstairs","session_id":"db866bac-b2bb-49eb-af09-6fa4ba3289c8"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:18.09474143Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1828,"":"downstairs","":"downstairs","session_id":"db866bac-b2bb-49eb-af09-6fa4ba3289c8"} {"msg":"[2]R flush_numbers[0..12]: [272, 274, 255, 274, 274, 274, 267, 265, 263, 274, 271, 271]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-01T16:56:18.094770974Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1828,"":"downstairs","":"downstairs","session_id":"db866bac-b2bb-49eb-af09-6fa4ba3289c8"} {"msg":"[2]R generation[0..12]: [160, 161, 152, 161, 161, 161, 158, 157, 155, 161, 160, 160]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:18.094801458Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1828,"":"downstairs","":"downstairs","session_id":"db866bac-b2bb-49eb-af09-6fa4ba3289c8"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:18.094831182Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1828,"":"downstairs","":"downstairs","session_id":"db866bac-b2bb-49eb-af09-6fa4ba3289c8"} {"msg":"Max found gen is 162","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:18.094863055Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1828,"":"downstairs","":"downstairs","session_id":"db866bac-b2bb-49eb-af09-6fa4ba3289c8"} {"msg":"Generation requested: 162 >= found:162","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:18.094892179Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1828,"":"downstairs","":"downstairs","session_id":"db866bac-b2bb-49eb-af09-6fa4ba3289c8"} {"msg":"Next flush: 275","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:18.094919783Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1828,"":"downstairs","":"downstairs","session_id":"db866bac-b2bb-49eb-af09-6fa4ba3289c8"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:18.094946276Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1828,"":"downstairs","":"downstairs","session_id":"db866bac-b2bb-49eb-af09-6fa4ba3289c8"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:18.09497392Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP"Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"pid":1828,"session_id":"db866bac-b2bb-49eb-af09-6fa4ba3289c8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:18.095005404Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1828,"client":"0","":"downstairs","session_id":"db866bac-b2bb-49eb-af09-6fa4ba3289c8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:18.095030987Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1828,"client":"1","":"downstairs","session_id":"db866bac-b2bb-49eb-af09-6fa4ba3289c8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:18.095045789Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1828,"client":"2","":"downstairs","session_id":"db866bac-b2bb-49eb-af09-6fa4ba3289c8"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:18.095060161Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1828,"session_id":"db866bac-b2bb-49eb-af09-6fa4ba3289c8"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:18.095072613Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1828,"session_id":"db866bac-b2bb-49eb-af09-6fa4ba3289c8"} {"msg":"800f814a-2743-4b94-a8a6-bd6c2a2857cc is now active with session: db866bac-b2bb-49eb-af09-6fa4ba3289c8","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:18.095085564Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1828,"session_id":"db866bac-b2bb-49eb-af09-6fa4ba3289c8"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:18.095097726Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1828,"session_id":"db866bac-b2bb-49eb-af09-6fa4ba3289c8"} {"msg":"The guest has finished waiting for activation with:162","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:18.095110017Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1828} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 149 len 24576 data: 15 18 15 17 13 14 02/30 Read block 353 len 28672 03/30 Read block 21 len 20480 04/30 Write block 373 len 36864 data: 11 11 10 11 14 14 13 12 13 05/30 Write block 35 len 16384 data: 12 15 16 15 06/30 Read block 435 len 12288 07/30 Read block 210 len 12288 08/30 Flush 09/30 Write block 340 len 24576 data: 11 6 7 9 9 10 10/30 Write block 153 len 36864 data: 14 15 14 15 13 12 11 13 12 11/30 Read block 106 len 20480 12/30 Read block 573 len 4096 13/30 Read block 531 len 20480 14/30 Write block 583 len 12288 data: 11 14 16 15/30 Write block 356 len 28672 data: 11 16 15 12 11 11 8 16/30 Write block 180 len 16384 data: 13 10 10 9 17/30 Read block 552 len 12288 18/30 Flush 19/30 Write block 349 len 4096 data: 12 20/30 Write block 301 len 40960 data: 7 7 7 8 10 11 15 17 17 16 21/30 Write block 103 len 28672 data: 14 14 17 19 16 15 15 22/30 Read block 233 len 24576 23/30 Write block 514 len 16384 data: 8 6 6 8 24/30 Flush 25/30 Write block 197 len 16384 data: 16 14 15 12 26/30 Write block 75 len 8192 data: 12 7 27/30 Write block 386 len 8192 data: 12 10 28/30 Read block 168 len 12288 29/30 Write block 227 len 40960 data: 10 7 11 10 9 8 7 9 9 11 30/30 Read block 587 len 40960 ---------------------------------------------------------------- Crucible gen:162 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 4 Done Done Done false 1025 Acked Write 2 Done Done Done false 1026 Acked Write 2 Done Done Done false 1027 Acked Read 3 Done Done Done false 1028 Acked Write 10 Sent Done Done false 1029 Acked Read 10 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 0 1 3 Done 4 6 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:56:18.120480096Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1828} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 160 160 160 272 272 272 F F F 1 020-039 162 162 162 275 275 275 F F F 2 040-059 152 152 152 255 255 255 F F F 3 060-079 161 161 161 274 274 274 T T T 4 080-099 161 161 161 274 274 274 F F F 5 100-119 162 162 162 277 277 277 F F F 6 120-139 158 158 158 267 267 267 F F F 7 140-159 162 162 162 276 276 276 F F F 8 160-179 162 162 162 276 276 276 F F F 9 180-199 162 162 162 276 276 276 T T T 10 200-219 160 160 160 271 271 271 T T T 11 220-239 160 160 160 271 271 271 T T T 12 240-259 152 152 152 252 252 252 F F F 13 260-279 159 159 159 270 270 270 F F F 14 280-299 160 160 160 273 273 273 F F F 15 300-319 162 162 162 277 277 277 F F F 16 320-339 161 161 161 274 274 274 F F F 17 340-359 162 162 162 277 277 277 F F F 18 360-379 162 162 162 276 276 276 F F F 19 380-399 162 162 162 275 275 275 T T T 20 400-419 161 161 161 274 274 274 F F F 21 420-439 161 161 161 274 274 274 F F F 22 440-459 160 160 160 271 271 271 F F F 23 460-479 160 160 160 272 272 272 F F F 24 480-499 156 156 156 264 264 264 F F F 25 500-519 162 162 162 277 277 277 F F F 26 520-539 155 155 155 263 263 263 F F F 27 540-559 155 155 155 263 263 263 F F F 28 560-579 158 158 158 267 267 267 F F F 29 580-599 162 162 162 276 276 276 F F F Max gen: 162, Max flush: 277 A Difference in extent metadata is expected here On loop 81 Verifying data now No change after verify Loop: 81 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 160 160 160 272 272 272 F F F 1 020-039 162 162 162 275 275 275 F F F 2 040-059 152 152 152 255 255 255 F F F 3 060-079 163 163 163 278 278 278 F F F 4 080-099 161 161 161 274 274 274 F F F 5 100-119 162 162 162 277 277 277 F F F 6 120-139 158 158 158 267 267 267 F F F 7 140-159 162 162 162 276 276 276 F F F 8 160-179 162 162 162 276 276 276 F F F 9 180-199 163 163 163 278 278 278 F F F 10 200-219 163 163 163 278 278 278 F F F 11 220-239 163 163 163 278 278 278 F F F 12 240-259 152 152 152 252 252 252 F F F 13 260-279 159 159 159 270 270 270 F F F 14 280-299 160 160 160 273 273 273 F F F 15 300-319 162 162 162 277 277 277 F F F 16 320-339 161 161 161 274 274 274 F F F 17 340-359 162 162 162 277 277 277 F F F 18 360-379 162 162 162 276 276 276 F F F 19 380-399 163 163 163 278 278 278 F F F 20 400-419 161 161 161 274 274 274 F F F 21 420-439 161 161 161 274 274 274 F F F 22 440-459 160 160 160 271 271 271 F F F 23 460-479 160 160 160 272 272 272 F F F 24 480-499 156 156 156 264 264 264 F F F 25 500-519 162 162 162 277 277 277 F F F 26 520-539 155 155 155 263 263 263 F F F 27 540-559 155 155 155 263 263 263 F F F 28 560-579 158 158 158 267 267 267 F F F 29 580-599 162 162 162 276 276 276 F F F Max gen: 163, Max flush: 278 Begin loop 82 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:22.458967798Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1836,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:56:22.50158768Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1836,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:22.501981531Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1836,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:22.533535056Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1836,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:22.534156176Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1836} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"3818f0fc76fe787c6cc412af30fd59b930203885\",\n git_commit_timestamp: \"2024-11-01T16:08:30.000000000Z\",\n git_branch: \"renovate/hyper-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:22.53418713Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1836} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:22.534200852Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1836} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:22.534215104Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1836} {"msg":"Crucible 1fa705af-a90f-4e23-aa6f-4b302d3728e4 has session id: 460c1427-a01a-48e7-be37-d34762cb9c81","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:22.534673903Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1836,"session_id":"460c1427-a01a-48e7-be37-d34762cb9c81"} {"msg":"Upstairs opts: Upstairs UUID: 1fa705af-a90f-4e23-aa6f-4b302d3728e4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:22.534705377Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1836,"session_id":"460c1427-a01a-48e7-be37-d34762cb9c81"} {"msg":"Crucible stats registered with UUID: 1fa705af-a90f-4e23-aa6f-4b302d3728e4","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:22.534720389Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1836,"session_id":"460c1427-a01a-48e7-be37-d34762cb9c81"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:22.565205657Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1836,"task":"crutest"} {"msg":"The guest has requested activation with gen:164","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:22.565234871Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1836} {"msg":"1fa705af-a90f-4e23-aa6f-4b302d3728e4 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:22.565303359Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1836,"session_id":"460c1427-a01a-48e7-be37-d34762cb9c81"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:22.565324732Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1836,"client":"0","":"downstairs","session_id":"460c1427-a01a-48e7-be37-d34762cb9c81"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:22.565353256Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1836,"client":"0","":"downstairs","session_id":"460c1427-a01a-48e7-be37-d34762cb9c81"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:22.565370818Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1836,"client":"1","":"downstairs","session_id":"460c1427-a01a-48e7-be37-d34762cb9c81"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:22.565399162Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1836,"client":"1","":"downstairs","session_id":"460c1427-a01a-48e7-be37-d34762cb9c81"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:22.565432046Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1836,"client":"2","":"downstairs","session_id":"460c1427-a01a-48e7-be37-d34762cb9c81"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:22.56546391Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1836,"client":"2","":"downstairs","session_id":"460c1427-a01a-48e7-be37-d34762cb9c81"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:22.565499145Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1836,"":"io task","client":"2","":"downstairs","session_id":"460c1427-a01a-48e7-be37-d34762cb9c81"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:22.56553625Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1836,"":"io task","client":"0","":"downstairs","session_id":"460c1427-a01a-48e7-be37-d34762cb9c81"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:22.565727154Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1836,"":"io task","client":"1","":"downstairs","session_id":"460c1427-a01a-48e7-be37-d34762cb9c81"} {"msg":"ds_connection connected from Ok(127.0.0.1:36736)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:22.565750867Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1836,"":"io task","client":"2","":"downstairs","session_id":"460c1427-a01a-48e7-be37-d34762cb9c81"} {"msg":"ds_connection connected from Ok(127.0.0.1:37368)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:22.56577307Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1836,"":"io task","client":"0","":"downstairs","session_id":"460c1427-a01a-48e7-be37-d34762cb9c81"} {"msg":"ds_connection connected from Ok(127.0.0.1:37810)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:22.565791413Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1836,"":"io task","client":"1","":"downstairs","session_id":"460c1427-a01a-48e7-be37-d34762cb9c81"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:22.566418733Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1836,"client":"2","":"downstairs","session_id":"460c1427-a01a-48e7-be37-d34762cb9c81"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:22.566449267Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1836,"client":"2","":"downstairs","session_id":"460c1427-a01a-48e7-be37-d34762cb9c81"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:22.566475981Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1836,"client":"0","":"downstairs","session_id":"460c1427-a01a-48e7-be37-d34762cb9c81"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:22.566504234Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1836,"client":"0","":"downstairs","session_id":"460c1427-a01a-48e7-be37-d34762cb9c81"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:22.566583124Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1836,"client":"1","":"downstairs","session_id":"460c1427-a01a-48e7-be37-d34762cb9c81"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:22.566614248Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1836,"client":"1","":"downstairs","session_id":"460c1427-a01a-48e7-be37-d34762cb9c81"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:22.567113423Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1836,"client":"1","":"downstairs","session_id":"460c1427-a01a-48e7-be37-d34762cb9c81"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:22.567152708Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1836,"client":"2","":"downstairs","session_id":"460c1427-a01a-48e7-be37-d34762cb9c81"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:22.567179121Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1836,"client":"0","":"downstairs","session_id":"460c1427-a01a-48e7-be37-d34762cb9c81"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:22.567663284Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1836,"client":"0","":"downstairs","session_id":"460c1427-a01a-48e7-be37-d34762cb9c81"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:22.567688797Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1836,"client":"0","":"downstairs","session_id":"460c1427-a01a-48e7-be37-d34762cb9c81"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:22.567703359Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1836,"session_id":"460c1427-a01a-48e7-be37-d34762cb9c81"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:22.567777729Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1836,"client":"1","":"downstairs","session_id":"460c1427-a01a-48e7-be37-d34762cb9c81"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:22.567804352Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1836,"client":"1","":"downstairs","session_id":"460c1427-a01a-48e7-be37-d34762cb9c81"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:22.567819494Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1836,"session_id":"460c1427-a01a-48e7-be37-d34762cb9c81"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:22.567832525Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1836,"client":"2","":"downstairs","session_id":"460c1427-a01a-48e7-be37-d34762cb9c81"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:22.567846587Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1836,"client":"2","":"downstairs","session_id":"460c1427-a01a-48e7-be37-d34762cb9c81"} {"msg":"[0]R flush_numbers[0..12]: [272, 275, 255, 278, 274, 277, 267, 276, 276, 278, 278, 278]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:22.567860709Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1836,"":"downstairs","":"downstairs","session_id":"460c1427-a01a-48e7-be37-d34762cb9c81"} {"msg":"[0]R generation[0..12]: [160, 162, 152, 163, 161, 162, 158, 162, 162, 163, 163, 163]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:22.567875271Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1836,"":"downstairs","":"downstairs","session_id":"460c1427-a01a-48e7-be37-d34762cb9c81"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:22.567890053Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1836,"":"downstairs","":"downstairs","session_id":"460c1427-a01a-48e7-be37-d34762cb9c81"} {"msg":"[1]R flush_numbers[0..12]: [272, 275, 255, 278, 274, 277, 267, 276, 276, 278, 278, 278]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:22.567904575Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1836,"":"downstairs","":"downstairs","session_id":"460c1427-a01a-48e7-be37-d34762cb9c81"} {"msg":"[1]R generation[0..12]: [160, 162, 152, 163, 161, 162, 158, 162, 162, 163, 163, 163]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:22.567919137Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1836,"":"downstairs","":"downstairs","session_id":"460c1427-a01a-48e7-be37-d34762cb9c81"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:22.567933018Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1836,"":"downstairs","":"downstairs","session_id":"460c1427-a01a-48e7-be37-d34762cb9c81"} {"msg":"[2]R flush_numbers[0..12]: [272, 275, 255, 278, 274, 277, 267, 276, 276, 278, 278, 278]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:22.56794794Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1836,"":"downstairs","":"downstairs","session_id":"460c1427-a01a-48e7-be37-d34762cb9c81"} {"msg":"[2]R generation[0..12]: [160, 162, 152, 163, 161, 162, 158, 162, 162, 163, 163, 163]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:22.567962402Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1836,"":"downstairs","":"downstairs","session_id":"460c1427-a01a-48e7-be37-d34762cb9c81"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-01T16:56:22.567985505Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1836,"":"downstairs","":"downstairs","session_id":"460c1427-a01a-48e7-be37-d34762cb9c81"} {"msg":"Max found gen is 164","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:22.568034691Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1836,"":"downstairs","":"downstairs","session_id":"460c1427-a01a-48e7-be37-d34762cb9c81"} {"msg":"Generation requested: 164 >= found:164","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:22.568056724Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1836,"":"downstairs","":"downstairs","session_id":"460c1427-a01a-48e7-be37-d34762cb9c81"} {"msg":"Next flush: 279","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:22.568071186Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1836,"":"downstairs","":"downstairs","session_id":"460c1427-a01a-48e7-be37-d34762cb9c81"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:22.568084958Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1836,"":"downstairs","":"downstairs","session_id":"460c1427-a01a-48e7-be37-d34762cb9c81"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:22.56809807Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1836,"session_id":"460c1427-a01a-48e7-be37-d34762cb9c81"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 level":30,"time":"2024-11-01T16:56:22.568114312Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1836,"client":"0","":"downstairs","session_id":"460c1427-a01a-48e7-be37-d34762cb9c81"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:22.568128544Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1836,"client":"1","":"downstairs","session_id":"460c1427-a01a-48e7-be37-d34762cb9c81"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:22.568142215Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1836,"client":"2","":"downstairs","session_id":"460c1427-a01a-48e7-be37-d34762cb9c81"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:22.568155647Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1836,"session_id":"460c1427-a01a-48e7-be37-d34762cb9c81"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:22.568168639Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1836,"session_id":"460c1427-a01a-48e7-be37-d34762cb9c81"} {"msg":"1fa705af-a90f-4e23-aa6f-4b302d3728e4 is now active with session: 460c1427-a01a-48e7-be37-d34762cb9c81","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:22.56818207Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1836,"session_id":"460c1427-a01a-48e7-be37-d34762cb9c81"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:22.568194762Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1836,"session_id":"460c1427-a01a-48e7-be37-d34762cb9c81"} {"msg":"The guest has finished waiting for activation with:164","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:22.568207674Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1836} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 195 len 32768 data: 19 17 17 15 16 13 14 13 02/30 Read block 294 len 24576 03/30 Read block 274 len 40960 04/30 Read block 485 len 4096 05/30 Write block 67 len 8192 data: 16 15 06/30 Write block 199 len 12288 data: 17 14 15 07/30 Read block 190 len 24576 08/30 Read block 93 len 20480 09/30 Read block 193 len 36864 10/30 Read block 420 len 8192 11/30 Write block 257 len 28672 data: 11 12 9 10 15 15 13 12/30 Read block 559 len 36864 13/30 Read block 317 len 4096 14/30 Read block 142 len 40960 15/30 Flush 16/30 Write block 5 len 16384 data: 11 13 13 14 17/30 Read block 15 len 36864 18/30 Read block 405 len 24576 19/30 Read block 8 len 36864 20/30 Read block 564 len 40960 21/30 Read block 130 len 8192 22/30 Write block 234 len 32768 data: 10 10 12 10 9 9 9 11 23/30 Read block 399 len 12288 24/30 Read block 512 len 32768 25/30 Write block 508 len 4096 data: 13 26/30 Write block 101 len 28672 data: 20 16 15 15 18 20 17 27/30 Write block 68 len 24576 data: 16 17 21 20 20 15 28/30 Read block 416 len 28672 29/30 Read block 563 len 24576 30/30 Read block 420 len 20480 ---------------------------------------------------------------- Crucible gen:164 GIO:true work queues: Upstairs:0 downstairs:15 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1015 Acked Write 4 Done Done Done false 1016 Acked Read 9 Done Done Done false 1017 Acked Read 6 Done Done Done false 1018 Acked Read 9 Done Done Done false 1019 Acked Read 10 Done Done Done false 1020 Acked Read 2 Done Done Done false 1021 Acked Write 8 Done Done Done false 1022 Acked Read 3 Done Done Done false 1023 Acked Read 8 Done Done Done false 1024 Acked Write 1 Done Done Done false 1025 Acked Write 7 Done Done Done false 1026 Acked Write 6 Done Done Done false 1027 Acked Read 7 Done Done Done false 1028 Acked Read 6 Done Done Done false 1029 Acked Read 5 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 1 2 Done 14 15 14 43 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1014 1014 1014 Downstairs last five completed: 1014 1013 1012 1011 1010 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:56:22.590670198Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1836} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 160 160 160 272 272 272 T T T 1 020-039 162 162 162 275 275 275 F F F 2 040-059 152 152 152 255 255 255 F F F 3 060-079 164 164 164 279 279 279 T T T 4 080-099 161 161 161 274 274 274 F F F 5 100-119 162 162 162 277 277 277 T T T 6 120-139 158 158 158 267 267 267 F F F 7 140-159 162 162 162 276 276 276 F F F 8 160-179 162 162 162 276 276 276 F F F 9 180-199 164 164 164 279 279 279 F F F 10 200-219 164 164 164 279 279 279 F F F 11 220-239 163 163 163 278 278 278 T T T 12 240-259 164 164 164 279 279 279 T T T 13 260-279 164 164 164 279 279 279 F F F 14 280-299 160 160 160 273 273 273 F F F 15 300-319 162 162 162 277 277 277 F F F 16 320-339 161 161 161 274 274 274 F F F 17 340-359 162 162 162 277 277 277 F F F 18 360-379 162 162 162 276 276 276 F F F 19 380-399 163 163 163 278 278 278 F F F 20 400-419 161 161 161 274 274 274 F F F 21 420-439 161 161 161 274 274 274 F F F 22 440-459 160 160 160 271 271 271 F F F 23 460-479 160 160 160 272 272 272 F F F 24 480-499 156 156 156 264 264 264 F F F 25 500-519 162 162 162 277 277 277 T T T 26 520-539 155 155 155 263 263 263 F F F 27 540-559 155 155 155 263 263 263 F F F 28 560-579 158 158 158 267 267 267 F F F 29 580-599 162 162 162 276 276 276 F F F Max gen: 164, Max flush: 279 A Difference in extent metadata is expected here On loop 82 Verifying data now No change after verify Loop: 82 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 165 165 165 280 280 280 F F F 1 020-039 162 162 162 275 275 275 F F F 2 040-059 152 152 152 255 255 255 F F F 3 060-079 165 165 165 280 280 280 F F F 4 080-099 161 161 161 274 274 274 F F F 5 100-119 165 165 165 280 280 280 F F F 6 120-139 158 158 158 267 267 267 F F F 7 140-159 162 162 162 276 276 276 F F F 8 160-179 162 162 162 276 276 276 F F F 9 180-199 164 164 164 279 279 279 F F F 10 200-219 164 164 164 279 279 279 F F F 11 220-239 165 165 165 280 280 280 F F F 12 240-259 165 165 165 280 280 280 F F F 13 260-279 164 164 164 279 279 279 F F F 14 280-299 160 160 160 273 273 273 F F F 15 300-319 162 162 162 277 277 277 F F F 16 320-339 161 161 161 274 274 274 F F F 17 340-359 162 162 162 277 277 277 F F F 18 360-379 162 162 162 276 276 276 F F F 19 380-399 163 163 163 278 278 278 F F F 20 400-419 161 161 161 274 274 274 F F F 21 420-439 161 161 161 274 274 274 F F F 22 440-459 160 160 160 271 271 271 F F F 23 460-479 160 160 160 272 272 272 F F F 24 480-499 156 156 156 264 264 264 F F F 25 500-519 165 165 165 280 280 280 F F F 26 520-539 155 155 155 263 263 263 F F F 27 540-559 155 155 155 263 263 263 F F F 28 560-579 158 158 158 267 267 267 F F F 29 580-599 162 162 162 276 276 276 F F F Max gen: 165, Max flush: 280 Begin loop 83 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:26.944241204Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1844,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:26.988576477Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1844,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:26.989252774Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1844} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"3818f0fc76fe787c6cc412af30fd59b930203885\",\n git_commit_timestamp: \"2024-11-01T16:08:30.000000000Z\",\n git_branch: \"renovate/hyper-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:26.98929585Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1844} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:26.989311701Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1844} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:26.989326843Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1844} {"msg":"Crucible 1389e638-7f61-4644-9c29-4e90f71de7e1 has session id: 4e1c4e43-8175-4cfc-b169-dd414323a583","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:26.989715664Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1844,"session_id":"4e1c4e43-8175-4cfc-b169-dd414323a583"} {"msg":"Upstairs opts: Upstairs UUID: 1389e638-7f61-4644-9c29-4e90f71de7e1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:26.989757619Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1844,"session_id":"4e1c4e43-8175-4cfc-b169-dd414323a583"} {"msg":"Crucible stats registered with UUID: 1389e638-7f61-4644-9c29-4e90f71de7e1","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:26.989786213Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1844,"session_id":"4e1c4e43-8175-4cfc-b169-dd414323a583"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:27.020830733Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1844,"task":"crutest"} {"msg":"The guest has requested activation with gen:166","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:27.020877549Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1844} {"msg":"1389e638-7f61-4644-9c29-4e90f71de7e1 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:27.020904712Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1844,"session_id":"4e1c4e43-8175-4cfc-b169-dd414323a583"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:27.020922454Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1844,"client":"0","":"downstairs","session_id":"4e1c4e43-8175-4cfc-b169-dd414323a583"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:27.02096735Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1844,"client":"0","":"downstairs","session_id":"4e1c4e43-8175-4cfc-b169-dd414323a583"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:27.021001365Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1844,"client":"1","":"downstairs","session_id":"4e1c4e43-8175-4cfc-b169-dd414323a583"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:27.021033649Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1844,"client":"1","":"downstairs","session_id":"4e1c4e43-8175-4cfc-b169-dd414323a583"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:27.021063873Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1844,"client":"2","":"downstairs","session_id":"4e1c4e43-8175-4cfc-b169-dd414323a583"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:27.021094927Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1844,"client":"2","":"downstairs","session_id":"4e1c4e43-8175-4cfc-b169-dd414323a583"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:27.021127141Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1844,"":"io task","client":"2","":"downstairs","session_id":"4e1c4e43-8175-4cfc-b169-dd414323a583"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:27.021160215Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1844,"":"io task","client":"0","":"downstairs","session_id":"4e1c4e43-8175-4cfc-b169-dd414323a583"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:27.021194149Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1844,"":"io task","client":"1","":"downstairs","session_id":"4e1c4e43-8175-4cfc-b169-dd414323a583"} {"msg":"ds_connection connected from Ok(127.0.0.1:54837)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:27.021229074Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1844,"":"io task","client":"2","":"downstairs","session_id":"4e1c4e43-8175-4cfc-b169-dd414323a583"} {"msg":"ds_connection connected from Ok(127.0.0.1:52647)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:27.02143572Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1844,"":"io task","client":"1","":"downstairs","session_id":"4e1c4e43-8175-4cfc-b169-dd414323a583"} {"msg":"ds_connection connected from Ok(127.0.0.1:45643)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:27.021481766Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1844,"":"io task","client":"0","":"downstairs","session_id":"4e1c4e43-8175-4cfc-b169-dd414323a583"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:27.022048779Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1844,"client":"0","":"downstairs","session_id":"4e1c4e43-8175-4cfc-b169-dd414323a583"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:27.022079574Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1844,"client":"0","":"downstairs","session_id":"4e1c4e43-8175-4cfc-b169-dd414323a583"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:27.022106087Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1844,"client":"1","":"downstairs","session_id":"4e1c4e43-8175-4cfc-b169-dd414323a583"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:27.022133981Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1844,"client":"1","":"downstairs","session_id":"4e1c4e43-8175-4cfc-b169-dd414323a583"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:27.022163724Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1844,"client":"2","":"downstairs","session_id":"4e1c4e43-8175-4cfc-b169-dd414323a583"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:27.022191148Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1844,"client":"2","":"downstairs","session_id":"4e1c4e43-8175-4cfc-b169-dd414323a583"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:27.022728787Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1844,"client":"0","":"downstairs","session_id":"4e1c4e43-8175-4cfc-b169-dd414323a583"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:27.022820299Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1844,"client":"1","":"downstairs","session_id":"4e1c4e43-8175-4cfc-b169-dd414323a583"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:27.022990211Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1844,"client":"2","":"downstairs","session_id":"4e1c4e43-8175-4cfc-b169-dd414323a583"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:27.023411305Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1844,"client":"0","":"downstairs","session_id":"4e1c4e43-8175-4cfc-b169-dd414323a583"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:27.023431968Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1844,"client":"0","":"downstairs","session_id":"4e1c4e43-8175-4cfc-b169-dd414323a583"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:27.02344598Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1844,"session_id":"4e1c4e43-8175-4cfc-b169-dd414323a583"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:27.02352362Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1844,"client":"1","":"downstairs","session_id":"4e1c4e43-8175-4cfc-b169-dd414323a583"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:27.023545703Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1844,"client":"1","":"downstairs","session_id":"4e1c4e43-8175-4cfc-b169-dd414323a583"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:27.023559564Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1844,"session_id":"4e1c4e43-8175-4cfc-b169-dd414323a583"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:27.023575116Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1844,"client":"2","":"downstairs","session_id":"4e1c4e43-8175-4cfc-b169-dd414323a583"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:27.023588928Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1844,"client":"2","":"downstairs","session_id":"4e1c4e43-8175-4cfc-b169-dd414323a583"} {"msg":"[0]R flush_numbers[0..12]: [280, 275, 255, 280, 274, 280, 267, 276, 276, 279, 279, 280]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:27.02360267Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1844,"":"downstairs","":"downstairs","session_id":"4e1c4e43-8175-4cfc-b169-dd414323a583"} {"msg":"[0]R generation[0..12]: [165, 162, 152, 165, 161, 165, 158, 162, 162, 164, 164, 165]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:27.023616752Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1844,"":"downstairs","":"downstairs","session_id":"4e1c4e43-8175-4cfc-b169-dd414323a583"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:27.023631023Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1844,"":"downstairs","":"downstairs","session_id":"4e1c4e43-8175-4cfc-b169-dd414323a583"} {"msg":"[1]R flush_numbers[0..12]: [280, 275, 255, 280, 274, 280, 267, 276, 276, 279, 279, 280]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:27.023645105Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1844,"":"downstairs","":"downstairs","session_id":"4e1c4e43-8175-4cfc-b169-dd414323a583"} {"msg":"[1]R generation[0..12]: [165, 162, 152, 165, 161, 165, 158, 162, 162, 164, 164, 165]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:27.023659047Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1844,"":"downstairs","":"downstairs","session_id":"4e1c4e43-8175-4cfc-b169-dd414323a583"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:27.023672999Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1844,"":"downstairs","":"downstairs","session_id":"4e1c4e43-8175-4cfc-b169-dd414323a583"} {"msg":"[2]R flush_numbers[0..12]: [280, 275, 255, 280, 274, 280, 267, 276, 276, 279, 279, 280]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:27.023686861Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1844,"":"downstairs","":"downstairs","session_id":"4e1c4e43-8175-4cfc-b169-dd414323a583"} {"msg":"[2]R generation[0..12]: [165, 162, 152, 165, 161, 165, 158, 162, 162, 164, 164, 165]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:27.023700712Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1844,"":"downstairs","":"downstairs","session_id":"4e1c4e43-8175-4cfc-b169-dd414323a583"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:27.023714764Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1844,"":"downstairs","":"downstairs","session_id":"4e1c4e43-8175-4cfc-b169-dd414323a583"} {"msg":"Max found gen is 166","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:27.023727816Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1844,"":"downstairs","":"downstairs","session_id":"4e1c4e43-8175-4cfc-b169-dd414323a583"} {"msg":"Generation requested: 166 >= found:166","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:27.023741158Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1844,"":"downstairs","":"downstairs","session_id":"4e1c4e43-8175-4cfc-b169-dd414323a583"} {"msg":"Next flush: 281","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:27.023754079Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1844,"":"downstairs","":"downstairs","session_id":"4e1c4e43-8175-4cfc-b169-dd414323a583"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:27.023769161Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1844,"":"downstairs","":"downstairs","session_id":"4e1c4e43-8175-4cfc-b169-dd414323a583"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","Wait for a query_work_queue command to finish before sending IO level":30,"time":"2024-11-01T16:56:27.023794084Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1844,"session_id":"4e1c4e43-8175-4cfc-b169-dd414323a583"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:27.023806696Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1844,"client":"0","":"downstairs","session_id":"4e1c4e43-8175-4cfc-b169-dd414323a583"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:27.023820068Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1844,"client":"1","":"downstairs","session_id":"4e1c4e43-8175-4cfc-b169-dd414323a583"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:27.02383306Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1844,"client":"2","":"downstairs","session_id":"4e1c4e43-8175-4cfc-b169-dd414323a583"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:27.023846541Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1844,"session_id":"4e1c4e43-8175-4cfc-b169-dd414323a583"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:27.023859523Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1844,"session_id":"4e1c4e43-8175-4cfc-b169-dd414323a583"} {"msg":"1389e638-7f61-4644-9c29-4e90f71de7e1 is now active with session: 4e1c4e43-8175-4cfc-b169-dd414323a583","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:27.023872294Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1844,"session_id":"4e1c4e43-8175-4cfc-b169-dd414323a583"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:27.023884386Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1844,"session_id":"4e1c4e43-8175-4cfc-b169-dd414323a583"} {"msg":"The guest has finished waiting for activation with:166","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:27.023896708Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1844} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 38 len 12288 data: 16 15 13 02/30 Flush 03/30 Write block 120 len 28672 data: 11 15 17 14 12 13 15 04/30 Write block 74 len 36864 data: 14 13 8 6 7 11 11 11 15 05/30 Read block 45 len 20480 06/30 Read block 336 len 4096 07/30 Write block 569 len 20480 data: 11 8 10 9 8 08/30 Write block 591 len 36864 data: 11 9 10 9 11 10 9 7 4 09/30 Read block 422 len 12288 10/30 Write block 194 len 8192 data: 21 20 11/30 Read block 554 len 28672 12/30 Read block 334 len 40960 13/30 Write block 74 len 8192 data: 15 14 14/30 Write block 344 len 16384 data: 10 11 11 12 15/30 Read block 261 len 40960 16/30 Read block 12 len 40960 17/30 Read block 242 len 24576 18/30 Write block 77 len 32768 data: 7 8 12 12 12 16 14 13 19/30 Flush 20/30 Write block 117 len 24576 data: 9 10 12 12 16 18 21/30 Read block 518 len 40960 22/30 Write block 162 len 36864 data: 10 12 11 12 10 9 10 10 11 23/30 Read block 435 len 12288 24/30 Flush 25/30 Write block 132 len 40960 data: 17 17 13 11 8 9 14 11 11 12 26/30 Write block 282 len 40960 data: 10 6 4 4 9 9 9 10 8 11 27/30 Write block 69 len 4096 data: 18 28/30 Read block 424 len 24576 29/30 Write block 8 len 28672 data: 15 15 14 15 17 15 16 30/30 Read block 486 len 4096 ---------------------------------------------------------------- Crucible gen:166 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 10 Done Done Done false 1025 Acked Write 10 Done Done Done false 1026 Acked Write 1 Done Done Done false 1027 Acked Read 6 Done Done Done false 1028 Acked Write 7 Done Done Done false 1029 Acked Read 1 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 6 6 6 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:56:27.051704141Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1844} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 165 165 165 280 280 280 T T T 1 020-039 166 166 166 281 281 281 F F F 2 040-059 166 166 166 281 281 281 F F F 3 060-079 166 166 166 282 282 282 T T T 4 080-099 166 166 166 282 282 282 F F F 5 100-119 166 166 166 283 283 283 F F F 6 120-139 166 166 166 283 283 283 T T T 7 140-159 162 162 162 276 276 276 T T T 8 160-179 166 166 166 283 283 283 F F F 9 180-199 166 166 166 282 282 282 F F F 10 200-219 164 164 164 279 279 279 F F F 11 220-239 165 165 165 280 280 280 F F F 12 240-259 165 165 165 280 280 280 F F F 13 260-279 164 164 164 279 279 279 F F F 14 280-299 160 160 160 273 273 273 T T T 15 300-319 162 162 162 277 277 277 F F F 16 320-339 161 161 161 274 274 274 F F F 17 340-359 166 166 166 282 282 282 F F F 18 360-379 162 162 162 276 276 276 F F F 19 380-399 163 163 163 278 278 278 F F F 20 400-419 161 161 161 274 274 274 F F F 21 420-439 161 161 161 274 274 274 F F F 22 440-459 160 160 160 271 271 271 F F F 23 460-479 160 160 160 272 272 272 F F F 24 480-499 156 156 156 264 264 264 F F F 25 500-519 165 165 165 280 280 280 F F F 26 520-539 155 155 155 263 263 263 F F F 27 540-559 155 155 155 263 263 263 F F F 28 560-579 166 166 166 282 282 282 F F F 29 580-599 166 166 166 282 282 282 F F F Max gen: 166, Max flush: 283 A Difference in extent metadata is expected here On loop 83 Verifying data now No change after verify Loop: 83 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 167 167 167 284 284 284 F F F 1 020-039 166 166 166 281 281 281 F F F 2 040-059 166 166 166 281 281 281 F F F 3 060-079 167 167 167 284 284 284 F F F 4 080-099 166 166 166 282 282 282 F F F 5 100-119 166 166 166 283 283 283 F F F 6 120-139 167 167 167 284 284 284 F F F 7 140-159 167 167 167 284 284 284 F F F 8 160-179 166 166 166 283 283 283 F F F 9 180-199 166 166 166 282 282 282 F F F 10 200-219 164 164 164 279 279 279 F F F 11 220-239 165 165 165 280 280 280 F F F 12 240-259 165 165 165 280 280 280 F F F 13 260-279 164 164 164 279 279 279 F F F 14 280-299 167 167 167 284 284 284 F F F 15 300-319 162 162 162 277 277 277 F F F 16 320-339 161 161 161 274 274 274 F F F 17 340-359 166 166 166 282 282 282 F F F 18 360-379 162 162 162 276 276 276 F F F 19 380-399 163 163 163 278 278 278 F F F 20 400-419 161 161 161 274 274 274 F F F 21 420-439 161 161 161 274 274 274 F F F 22 440-459 160 160 160 271 271 271 F F F 23 460-479 160 160 160 272 272 272 F F F 24 480-499 156 156 156 264 264 264 F F F 25 500-519 165 165 165 280 280 280 F F F 26 520-539 155 155 155 263 263 263 F F F 27 540-559 155 155 155 263 263 263 F F F 28 560-579 166 166 166 282 282 282 F F F 29 580-599 166 166 166 282 282 282 F F F Max gen: 167, Max flush: 284 Begin loop 84 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:31.361331245Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1852,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:31.406715303Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1852,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:31.407318861Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1852} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"3818f0fc76fe787c6cc412af30fd59b930203885\",\n git_commit_timestamp: \"2024-11-01T16:08:30.000000000Z\",\n git_branch: \"renovate/hyper-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:31.407366847Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1852} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:31.407384679Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1852} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:31.407402211Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1852} {"msg":"Crucible 71644bfc-e031-486c-ae04-74bf119a7147 has session id: 5750a26f-abb5-4c8e-b449-7cf7a0b029a1","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:31.407794162Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1852,"session_id":"5750a26f-abb5-4c8e-b449-7cf7a0b029a1"} {"msg":"Upstairs opts: Upstairs UUID: 71644bfc-e031-486c-ae04-74bf119a7147, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:31.407836547Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1852,"session_id":"5750a26f-abb5-4c8e-b449-7cf7a0b029a1"} {"msg":"Crucible stats registered with UUID: 71644bfc-e031-486c-ae04-74bf119a7147","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:31.40785691Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1852,"session_id":"5750a26f-abb5-4c8e-b449-7cf7a0b029a1"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:31.438844383Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1852,"task":"crutest"} {"msg":"The guest has requested activation with gen:168","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:31.438875256Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1852} {"msg":"71644bfc-e031-486c-ae04-74bf119a7147 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:31.438949676Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1852,"session_id":"5750a26f-abb5-4c8e-b449-7cf7a0b029a1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:31.43897732Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1852,"client":"0","":"downstairs","session_id":"5750a26f-abb5-4c8e-b449-7cf7a0b029a1"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:31.439006243Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1852,"client":"0","":"downstairs","session_id":"5750a26f-abb5-4c8e-b449-7cf7a0b029a1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:31.439024436Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1852,"client":"1","":"downstairs","session_id":"5750a26f-abb5-4c8e-b449-7cf7a0b029a1"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:31.439041688Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1852,"client":"1","":"downstairs","session_id":"5750a26f-abb5-4c8e-b449-7cf7a0b029a1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:31.43905841Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1852,"client":"2","":"downstairs","session_id":"5750a26f-abb5-4c8e-b449-7cf7a0b029a1"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:31.439080243Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1852,"client":"2","":"downstairs","session_id":"5750a26f-abb5-4c8e-b449-7cf7a0b029a1"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:31.439110377Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1852,"":"io task","client":"2","":"downstairs","session_id":"5750a26f-abb5-4c8e-b449-7cf7a0b029a1"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:31.439145581Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1852,"":"io task","client":"0","":"downstairs","session_id":"5750a26f-abb5-4c8e-b449-7cf7a0b029a1"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:31.439166694Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1852,"":"io task","client":"1","":"downstairs","session_id":"5750a26f-abb5-4c8e-b449-7cf7a0b029a1"} {"msg":"ds_connection connected from Ok(127.0.0.1:59087)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:31.439244194Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1852,"":"io task","client":"2","":"downstairs","session_id":"5750a26f-abb5-4c8e-b449-7cf7a0b029a1"} {"msg":"ds_connection connected from Ok(127.0.0.1:39823)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:31.43929283Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1852,"":"io task","client":"1","":"downstairs","session_id":"5750a26f-abb5-4c8e-b449-7cf7a0b029a1"} {"msg":"ds_connection connected from Ok(127.0.0.1:55269)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:31.439325454Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1852,"":"io task","client":"0","":"downstairs","session_id":"5750a26f-abb5-4c8e-b449-7cf7a0b029a1"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:31.440053818Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1852,"client":"0","":"downstairs","session_id":"5750a26f-abb5-4c8e-b449-7cf7a0b029a1"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:31.440093653Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1852,"client":"0","":"downstairs","session_id":"5750a26f-abb5-4c8e-b449-7cf7a0b029a1"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:31.440121267Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1852,"client":"1","":"downstairs","session_id":"5750a26f-abb5-4c8e-b449-7cf7a0b029a1"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:31.440157102Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1852,"client":"1","":"downstairs","session_id":"5750a26f-abb5-4c8e-b449-7cf7a0b029a1"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:31.440189836Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1852,"client":"2","":"downstairs","session_id":"5750a26f-abb5-4c8e-b449-7cf7a0b029a1"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:31.440206808Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1852,"client":"2","":"downstairs","session_id":"5750a26f-abb5-4c8e-b449-7cf7a0b029a1"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:31.440648965Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1852,"client":"0","":"downstairs","session_id":"5750a26f-abb5-4c8e-b449-7cf7a0b029a1"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:31.44084299Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1852,"client":"1","":"downstairs","session_id":"5750a26f-abb5-4c8e-b449-7cf7a0b029a1"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:31.440877925Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1852,"client":"2","":"downstairs","session_id":"5750a26f-abb5-4c8e-b449-7cf7a0b029a1"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:31.441251493Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1852,"client":"0","":"downstairs","session_id":"5750a26f-abb5-4c8e-b449-7cf7a0b029a1"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:31.441280436Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1852,"client":"0","":"downstairs","session_id":"5750a26f-abb5-4c8e-b449-7cf7a0b029a1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:31.441298919Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1852,"session_id":"5750a26f-abb5-4c8e-b449-7cf7a0b029a1"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:31.441363107Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1852,"client":"1","":"downstairs","session_id":"5750a26f-abb5-4c8e-b449-7cf7a0b029a1"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:31.44138994Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1852,"client":"1","":"downstairs","session_id":"5750a26f-abb5-4c8e-b449-7cf7a0b029a1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:31.441404342Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1852,"session_id":"5750a26f-abb5-4c8e-b449-7cf7a0b029a1"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:31.441475931Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1852,"client":"2","":"downstairs","session_id":"5750a26f-abb5-4c8e-b449-7cf7a0b029a1"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:31.441496784Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1852,"client":"2","":"downstairs","session_id":"5750a26f-abb5-4c8e-b449-7cf7a0b029a1"} {"msg":"[0]R flush_numbers[0..12]: [284, 281, 281, 284, 282, 283, 284, 284, 283, 282, 279, 280]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:31.441511316Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1852,"":"downstairs","":"downstairs","session_id":"5750a26f-abb5-4c8e-b449-7cf7a0b029a1"} {"msg":"[0]R generation[0..12]: [167, 166, 166, 167, 166, 166, 167, 167, 166, 166, 164, 165]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:31.441525578Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1852,"":"downstairs","":"downstairs","session_id":"5750a26f-abb5-4c8e-b449-7cf7a0b029a1"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:31.44154374Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1852,"":"downstairs","":"downstairs","session_id":"5750a26f-abb5-4c8e-b449-7cf7a0b029a1"} {"msg":"[1]R flush_numbers[0..12]: [284, 281, 281, 284, 282, 283, 284, 284, 283, 282, 279, 280]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:31.441558442Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1852,"":"downstairs","":"downstairs","session_id":"5750a26f-abb5-4c8e-b449-7cf7a0b029a1"} {"msg":"[1]R generation[0..12]: [167, 166, 166, 167, 166, 166, 167, 167, 166, 166, 164, 165]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:31.441572264Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1852,"":"downstairs","":"downstairs","session_id":"5750a26f-abb5-4c8e-b449-7cf7a0b029a1"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:31.441586116Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1852,"":"downstairs","":"downstairs","session_id":"5750a26f-abb5-4c8e-b449-7cf7a0b029a1"} {"msg":"[2]R flush_numbers[0..12]: [284, 281, 281, 284, 282, 283, 284, 284, 283, 282, 279, 280]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:31.441599967Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1852,"":"downstairs","":"downstairs","session_id":"5750a26f-abb5-4c8e-b449-7cf7a0b029a1"} {"msg":"[2]R generation[0..12]: [167, 166, 166, 167, 166, 166, 167, 167, 166, 166, 164, 165]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:31.441613849Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1852,"":"downstairs","":"downstairs","session_id":"5750a26f-abb5-4c8e-b449-7cf7a0b029a1"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:31.441627801Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1852,"":"downstairs","":"downstairs","session_id":"5750a26f-abb5-4c8e-b449-7cf7a0b029a1"} {"msg":"Max found gen is 168","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:31.441640683Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1852,"":"downstairs","":"downstairs","session_id":"5750a26f-abb5-4c8e-b449-7cf7a0b029a1"} {"msg":"Generation requested: 168 >= found:168","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:31.441653665Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1852,"":"downstairs","":"downstairs","session_id":"5750a26f-abb5-4c8e-b449-7cf7a0b029a1"} {"msg":"Next flush: 285","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:31.441666646Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1852,"":"downstairs","":"downstairs","session_id":"5750a26f-abb5-4c8e-b449-7cf7a0b029a1"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:31.441679528Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1852,"":"downstairs","":"downstairs","session_id":"5750a26f-abb5-4c8e-b449-7cf7a0b029a1"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:31.44169261Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1852,"session_id":"5750a26f-abb5-4c8e-b449-7cf7a0b029a1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-01T16:56:31.441704731Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1852,"client":"0","":"downstairs","session_id":"5750a26f-abb5-4c8e-b449-7cf7a0b029a1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:31.441724744Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1852,"client":"1","":"downstairs","session_id":"5750a26f-abb5-4c8e-b449-7cf7a0b029a1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:31.441739965Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1852,"client":"2","":"downstairs","session_id":"5750a26f-abb5-4c8e-b449-7cf7a0b029a1"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:31.441753437Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1852,"session_id":"5750a26f-abb5-4c8e-b449-7cf7a0b029a1"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:31.441765629Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1852,"session_id":"5750a26f-abb5-4c8e-b449-7cf7a0b029a1"} {"msg":"71644bfc-e031-486c-ae04-74bf119a7147 is now active with session: 5750a26f-abb5-4c8e-b449-7cf7a0b029a1","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:31.44177843Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1852,"session_id":"5750a26f-abb5-4c8e-b449-7cf7a0b029a1"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:31.441790502Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1852,"session_id":"5750a26f-abb5-4c8e-b449-7cf7a0b029a1"} {"msg":"The guest has finished waiting for activation with:168","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:31.441814175Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1852} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 110 len 32768 data: 14 13 12 12 12 11 9 10 02/30 Read block 212 len 20480 03/30 Write block 42 len 8192 data: 15 18 04/30 Read block 318 len 8192 05/30 Write block 447 len 36864 data: 12 12 13 17 16 14 14 14 10 06/30 Flush 07/30 Write block 249 len 40960 data: 21 23 21 18 16 15 12 11 12 13 08/30 Write block 327 len 16384 data: 7 5 6 7 09/30 Read block 82 len 24576 10/30 Write block 514 len 24576 data: 9 7 7 9 9 12 11/30 Read block 266 len 36864 12/30 Read block 323 len 40960 13/30 Write block 566 len 4096 data: 13 14/30 Read block 502 len 20480 15/30 Read block 32 len 20480 16/30 Write block 226 len 16384 data: 12 11 8 12 17/30 Write block 269 len 36864 data: 14 16 21 18 19 22 20 19 14 18/30 Write block 264 len 28672 data: 12 12 13 12 12 15 17 19/30 Read block 79 len 12288 20/30 Read block 433 len 20480 21/30 Read block 69 len 36864 22/30 Read block 438 len 8192 23/30 Read block 255 len 16384 24/30 Write block 274 len 16384 data: 23 21 20 15 25/30 Write block 53 len 24576 data: 12 11 10 11 11 10 26/30 Read block 121 len 8192 27/30 Write block 201 len 28672 data: 16 14 14 13 13 14 17 28/30 Read block 229 len 16384 29/30 Read block 512 len 16384 30/30 Write block 311 len 12288 data: 16 16 16 ---------------------------------------------------------------- Crucible gen:168 GIO:true work queues: Upstairs:0 downstairs:24 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1006 Acked Write 10 Done Done Done false 1007 Acked Write 4 Done Done Done false 1008 Acked Read 6 Done Done Done false 1009 Acked Write 6 Done Done Done false 1010 Acked Read 9 Done Done Done false 1011 Acked Read 10 Done Done Done false 1012 Acked Write 1 Done Done Done false 1013 Acked Read 5 Done Done Done false 1014 Acked Read 5 Done Done Done false 1015 Acked Write 4 Done Done Done false 1016 Acked Write 9 Done Done Done false 1017 Acked Write 7 Done Done Done false 1018 Acked Read 3 Done Done Done false 1019 Acked Read 5 Done Done Done false 1020 Acked Read 9 Done Done Done false 1021 Acked Read 2 Done Done Done false 1022 Acked Read 4 Done Done Done false 1023 Acked Write 4 Done Done Done false 1024 Acked Write 6 Done Done Done false 1025 Acked Read 2 Done Done Done false 1026 Acked Write 7 Done Done Done false 1027 Acked Read 4 Done Done Done false 1028 Acked Read 4 Done Done Done false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 23 23 23 69 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1005 1005 1005 Downstairs last five completed: 1005 1004 1003 1002 1001 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:56:31.463433651Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1852} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 167 167 167 284 284 284 F F F 1 020-039 166 166 166 281 281 281 F F F 2 040-059 168 168 168 285 285 285 T T T 3 060-079 167 167 167 284 284 284 F F F 4 080-099 166 166 166 282 282 282 F F F 5 100-119 168 168 168 285 285 285 F F F 6 120-139 167 167 167 284 284 284 F F F 7 140-159 167 167 167 284 284 284 F F F 8 160-179 166 166 166 283 283 283 F F F 9 180-199 166 166 166 282 282 282 F F F 10 200-219 164 164 164 279 279 279 T T T 11 220-239 165 165 165 280 280 280 T T T 12 240-259 165 165 165 280 280 280 T T T 13 260-279 164 164 164 279 279 279 T T T 14 280-299 167 167 167 284 284 284 F F F 15 300-319 162 162 162 277 277 277 T T T 16 320-339 161 161 161 274 274 274 T T T 17 340-359 166 166 166 282 282 282 F F F 18 360-379 162 162 162 276 276 276 F F F 19 380-399 163 163 163 278 278 278 F F F 20 400-419 161 161 161 274 274 274 F F F 21 420-439 161 161 161 274 274 274 F F F 22 440-459 168 168 168 285 285 285 F F F 23 460-479 160 160 160 272 272 272 F F F 24 480-499 156 156 156 264 264 264 F F F 25 500-519 165 165 165 280 280 280 T T T 26 520-539 155 155 155 263 263 263 F F F 27 540-559 155 155 155 263 263 263 F F F 28 560-579 166 166 166 282 282 282 T T T 29 580-599 166 166 166 282 282 282 F F F Max gen: 168, Max flush: 285 A Difference in extent metadata is expected here On loop 84 Verifying data now No change after verify Loop: 84 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 167 167 167 284 284 284 F F F 1 020-039 166 166 166 281 281 281 F F F 2 040-059 169 169 169 286 286 286 F F F 3 060-079 167 167 167 284 284 284 F F F 4 080-099 166 166 166 282 282 282 F F F 5 100-119 168 168 168 285 285 285 F F F 6 120-139 167 167 167 284 284 284 F F F 7 140-159 167 167 167 284 284 284 F F F 8 160-179 166 166 166 283 283 283 F F F 9 180-199 166 166 166 282 282 282 F F F 10 200-219 169 169 169 286 286 286 F F F 11 220-239 169 169 169 286 286 286 F F F 12 240-259 169 169 169 286 286 286 F F F 13 260-279 169 169 169 286 286 286 F F F 14 280-299 167 167 167 284 284 284 F F F 15 300-319 169 169 169 286 286 286 F F F 16 320-339 169 169 169 286 286 286 F F F 17 340-359 166 166 166 282 282 282 F F F 18 360-379 162 162 162 276 276 276 F F F 19 380-399 163 163 163 278 278 278 F F F 20 400-419 161 161 161 274 274 274 F F F 21 420-439 161 161 161 274 274 274 F F F 22 440-459 168 168 168 285 285 285 F F F 23 460-479 160 160 160 272 272 272 F F F 24 480-499 156 156 156 264 264 264 F F F 25 500-519 169 169 169 286 286 286 F F F 26 520-539 155 155 155 263 263 263 F F F 27 540-559 155 155 155 263 263 263 F F F 28 560-579 169 169 169 286 286 286 F F F 29 580-599 166 166 166 282 282 282 F F F Max gen: 169, Max flush: 286 Begin loop 85 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:35.817053222Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1860,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:56:35.859195082Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1860,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:35.859555778Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1860,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:35.8912352Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1860,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:35.893957391Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1860} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"3818f0fc76fe787c6cc412af30fd59b930203885\",\n git_commit_timestamp: \"2024-11-01T16:08:30.000000000Z\",\n git_branch: \"renovate/hyper-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:35.893993586Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1860} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:35.894020649Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1860} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:35.894036541Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1860} {"msg":"Crucible d6521745-d0d8-4483-ac5c-651bc35628b9 has session id: 5e6e9e29-497f-42e9-a760-19369ac0ddef","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:35.894449534Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1860,"session_id":"5e6e9e29-497f-42e9-a760-19369ac0ddef"} {"msg":"Upstairs opts: Upstairs UUID: d6521745-d0d8-4483-ac5c-651bc35628b9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:35.894542406Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1860,"session_id":"5e6e9e29-497f-42e9-a760-19369ac0ddef"} {"msg":"Crucible stats registered with UUID: d6521745-d0d8-4483-ac5c-651bc35628b9","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:35.894567379Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1860,"session_id":"5e6e9e29-497f-42e9-a760-19369ac0ddef"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:35.925604509Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1860,"task":"crutest"} {"msg":"The guest has requested activation with gen:170","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:35.925638613Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1860} {"msg":"d6521745-d0d8-4483-ac5c-651bc35628b9 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:35.925674648Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1860,"session_id":"5e6e9e29-497f-42e9-a760-19369ac0ddef"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:35.92569306Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1860,"client":"0","":"downstairs","session_id":"5e6e9e29-497f-42e9-a760-19369ac0ddef"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:35.925721684Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1860,"client":"0","":"downstairs","session_id":"5e6e9e29-497f-42e9-a760-19369ac0ddef"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:35.925739256Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1860,"client":"1","":"downstairs","session_id":"5e6e9e29-497f-42e9-a760-19369ac0ddef"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:35.925756578Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1860,"client":"1","":"downstairs","session_id":"5e6e9e29-497f-42e9-a760-19369ac0ddef"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:35.92577329Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1860,"client":"2","":"downstairs","session_id":"5e6e9e29-497f-42e9-a760-19369ac0ddef"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:35.925790082Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1860,"client":"2","":"downstairs","session_id":"5e6e9e29-497f-42e9-a760-19369ac0ddef"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:35.925817646Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1860,"":"io task","client":"2","":"downstairs","session_id":"5e6e9e29-497f-42e9-a760-19369ac0ddef"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:35.92584906Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1860,"":"io task","client":"0","":"downstairs","session_id":"5e6e9e29-497f-42e9-a760-19369ac0ddef"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:35.925879304Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1860,"":"io task","client":"1","":"downstairs","session_id":"5e6e9e29-497f-42e9-a760-19369ac0ddef"} {"msg":"ds_connection connected from Ok(127.0.0.1:34723)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:35.925912178Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1860,"":"io task","client":"1","":"downstairs","session_id":"5e6e9e29-497f-42e9-a760-19369ac0ddef"} {"msg":"ds_connection connected from Ok(127.0.0.1:52226)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:35.925941952Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1860,"":"io task","client":"2","":"downstairs","session_id":"5e6e9e29-497f-42e9-a760-19369ac0ddef"} {"msg":"ds_connection connected from Ok(127.0.0.1:49947)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:35.926151659Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1860,"":"io task","client":"0","":"downstairs","session_id":"5e6e9e29-497f-42e9-a760-19369ac0ddef"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:35.926592886Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1860,"client":"1","":"downstairs","session_id":"5e6e9e29-497f-42e9-a760-19369ac0ddef"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:35.926628301Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1860,"client":"1","":"downstairs","session_id":"5e6e9e29-497f-42e9-a760-19369ac0ddef"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:35.926655554Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1860,"client":"2","":"downstairs","session_id":"5e6e9e29-497f-42e9-a760-19369ac0ddef"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:35.926684898Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1860,"client":"2","":"downstairs","session_id":"5e6e9e29-497f-42e9-a760-19369ac0ddef"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:35.926895165Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1860,"client":"0","":"downstairs","session_id":"5e6e9e29-497f-42e9-a760-19369ac0ddef"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:35.92693622Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1860,"client":"0","":"downstairs","session_id":"5e6e9e29-497f-42e9-a760-19369ac0ddef"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:35.927337522Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1860,"client":"2","":"downstairs","session_id":"5e6e9e29-497f-42e9-a760-19369ac0ddef"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:35.927467809Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1860,"client":"1","":"downstairs","session_id":"5e6e9e29-497f-42e9-a760-19369ac0ddef"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:35.927502903Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1860,"client":"0","":"downstairs","session_id":"5e6e9e29-497f-42e9-a760-19369ac0ddef"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:35.927930978Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1860,"client":"2","":"downstairs","session_id":"5e6e9e29-497f-42e9-a760-19369ac0ddef"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:35.927956702Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1860,"client":"2","":"downstairs","session_id":"5e6e9e29-497f-42e9-a760-19369ac0ddef"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:35.927971103Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1860,"session_id":"5e6e9e29-497f-42e9-a760-19369ac0ddef"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:35.928052074Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1860,"client":"1","":"downstairs","session_id":"5e6e9e29-497f-42e9-a760-19369ac0ddef"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:35.928088059Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1860,"client":"1","":"downstairs","session_id":"5e6e9e29-497f-42e9-a760-19369ac0ddef"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:35.928110592Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1860,"session_id":"5e6e9e29-497f-42e9-a760-19369ac0ddef"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:35.928125174Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1860,"client":"0","":"downstairs","session_id":"5e6e9e29-497f-42e9-a760-19369ac0ddef"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:35.928143056Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1860,"client":"0","":"downstairs","session_id":"5e6e9e29-497f-42e9-a760-19369ac0ddef"} {"msg":"[0]R flush_numbers[0..12]: [284, 281, 286, 284, 282, 285, 284, 284, 283, 282, 286, 286]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:35.928158148Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1860,"":"downstairs","":"downstairs","session_id":"5e6e9e29-497f-42e9-a760-19369ac0ddef"} {"msg":"[0]R generation[0..12]: [167, 166, 169, 167, 166, 168, 167, 167, 166, 166, 169, 169]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:35.92817298Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1860,"":"downstairs","":"downstairs","session_id":"5e6e9e29-497f-42e9-a760-19369ac0ddef"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:35.928187151Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1860,"":"downstairs","":"downstairs","session_id":"5e6e9e29-497f-42e9-a760-19369ac0ddef"} {"msg":"[1]R flush_numbers[0..12]: [284, 281, 286, 284, 282, 285, 284, 284, 283, 282, 286, 286]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:35.928200983Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1860,"":"downstairs","":"downstairs","session_id":"5e6e9e29-497f-42e9-a760-19369ac0ddef"} {"msg":"[1]R generation[0..12]: [167, 166, 169, 167, 166, 168, 167, 167, 166, 166, 169, 169]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:35.928214585Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1860,"":"downstairs","":"downstairs","session_id":"5e6e9e29-497f-42e9-a760-19369ac0ddef"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:35.928229217Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1860,"":"downstairs","":"downstairs","session_id":"5e6e9e29-497f-42e9-a760-19369ac0ddef"} {"msg":"[2]R flush_numbers[0..12]: [284, 281, 286, 284, 282, 285, 284, 284, 283, 282, 286, 286]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:35.928243829Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1860,"":"downstairs","":"downstairs","session_id":"5e6e9e29-497f-42e9-a760-19369ac0ddef"} {"msg":"[2]R generation[0..12]: [167, 166, 169, 167, 166, 168, 167, 167, 166, 166, 169, 169]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:35.928258461Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1860,"":"downstairs","":"downstairs","session_id":"5e6e9e29-497f-42e9-a760-19369ac0ddef"} Wait for a query_work_queue command to finish before sending IO {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:35.928282924Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1860,"":"downstairs","":"downstairs","session_id":"5e6e9e29-497f-42e9-a760-19369ac0ddef"} {"msg":"Max found gen is 170","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:35.928308117Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1860,"":"downstairs","":"downstairs","session_id":"5e6e9e29-497f-42e9-a760-19369ac0ddef"} {"msg":"Generation requested: 170 >= found:170","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:35.92833092Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1860,"":"downstairs","":"downstairs","session_id":"5e6e9e29-497f-42e9-a760-19369ac0ddef"} {"msg":"Next flush: 287","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:35.928354813Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1860,"":"downstairs","":"downstairs","session_id":"5e6e9e29-497f-42e9-a760-19369ac0ddef"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:35.928379796Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1860,"":"downstairs","":"downstairs","session_id":"5e6e9e29-497f-42e9-a760-19369ac0ddef"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:35.928397338Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1860,"session_id":"5e6e9e29-497f-42e9-a760-19369ac0ddef"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:35.92841052Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1860,"client":"0","":"downstairs","session_id":"5e6e9e29-497f-42e9-a760-19369ac0ddef"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:35.928424702Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1860,"client":"1","":"downstairs","session_id":"5e6e9e29-497f-42e9-a760-19369ac0ddef"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:35.928438944Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1860,"client":"2","":"downstairs","session_id":"5e6e9e29-497f-42e9-a760-19369ac0ddef"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:35.928453856Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1860,"session_id":"5e6e9e29-497f-42e9-a760-19369ac0ddef"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:35.928466827Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1860,"session_id":"5e6e9e29-497f-42e9-a760-19369ac0ddef"} {"msg":"d6521745-d0d8-4483-ac5c-651bc35628b9 is now active with session: 5e6e9e29-497f-42e9-a760-19369ac0ddef","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:35.928480319Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1860,"session_id":"5e6e9e29-497f-42e9-a760-19369ac0ddef"} {"msg":"Set Active after no reconciliation","v":0,"Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 name":"crucible","level":30,"time":"2024-11-01T16:56:35.928496941Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1860,"session_id":"5e6e9e29-497f-42e9-a760-19369ac0ddef"} {"msg":"The guest has finished waiting for activation with:170","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:35.928509603Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1860} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 15 len 4096 data: 15 02/30 Flush 03/30 Write block 360 len 16384 data: 12 12 9 8 04/30 Read block 21 len 20480 05/30 Write block 74 len 12288 data: 16 15 9 06/30 Read block 377 len 12288 07/30 Write block 362 len 36864 data: 10 9 9 10 8 7 6 8 7 08/30 Write block 388 len 36864 data: 11 12 10 10 10 11 10 12 13 09/30 Read block 159 len 32768 10/30 Read block 38 len 20480 11/30 Read block 557 len 4096 12/30 Read block 250 len 32768 13/30 Read block 48 len 28672 14/30 Write block 148 len 28672 data: 11 16 19 16 18 15 16 15/30 Flush 16/30 Write block 458 len 24576 data: 11 10 8 10 12 11 17/30 Read block 225 len 32768 18/30 Write block 108 len 40960 data: 16 16 15 14 13 13 13 12 10 11 19/30 Write block 557 len 16384 data: 16 19 24 20 20/30 Read block 130 len 4096 21/30 Write block 15 len 36864 data: 16 16 16 14 14 13 8 10 14 22/30 Flush 23/30 Write block 164 len 20480 data: 12 13 11 10 11 24/30 Read block 576 len 8192 25/30 Flush 26/30 Write block 140 len 8192 data: 12 13 27/30 Read block 248 len 8192 28/30 Read block 161 len 12288 29/30 Read block 37 len 24576 30/30 Read block 325 len 36864 ---------------------------------------------------------------- Crucible gen:170 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 2 Done Done Done false 1026 Acked Read 2 Done Done Done false 1027 Acked Read 3 Done Done Done false 1028 Acked Read 6 Done Done Done false 1029 Acked Read 9 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 0 1 Done 4 5 5 14 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:56:35.953997187Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1860} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 170 170 170 289 289 289 F F F 1 020-039 170 170 170 289 289 289 F F F 2 040-059 169 169 169 286 286 286 F F F 3 060-079 170 170 170 288 288 288 F F F 4 080-099 166 166 166 282 282 282 F F F 5 100-119 170 170 170 289 289 289 F F F 6 120-139 167 167 167 284 284 284 F F F 7 140-159 170 170 170 288 288 288 T T T 8 160-179 170 170 170 290 290 290 F F F 9 180-199 166 166 166 282 282 282 F F F 10 200-219 169 169 169 286 286 286 F F F 11 220-239 169 169 169 286 286 286 F F F 12 240-259 169 169 169 286 286 286 F F F 13 260-279 169 169 169 286 286 286 F F F 14 280-299 167 167 167 284 284 284 F F F 15 300-319 169 169 169 286 286 286 F F F 16 320-339 169 169 169 286 286 286 F F F 17 340-359 166 166 166 282 282 282 F F F 18 360-379 170 170 170 288 288 288 F F F 19 380-399 170 170 170 288 288 288 F F F 20 400-419 161 161 161 274 274 274 F F F 21 420-439 161 161 161 274 274 274 F F F 22 440-459 170 170 170 289 289 289 F F F 23 460-479 170 170 170 289 289 289 F F F 24 480-499 156 156 156 264 264 264 F F F 25 500-519 169 169 169 286 286 286 F F F 26 520-539 155 155 155 263 263 263 F F F 27 540-559 170 170 170 289 289 289 F F F 28 560-579 170 170 170 289 289 289 F F F 29 580-599 166 166 166 282 282 282 F F F Max gen: 170, Max flush: 290 A Difference in extent metadata is expected here On loop 85 Verifying data now No change after verify Loop: 85 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 170 170 170 289 289 289 F F F 1 020-039 170 170 170 289 289 289 F F F 2 040-059 169 169 169 286 286 286 F F F 3 060-079 170 170 170 288 288 288 F F F 4 080-099 166 166 166 282 282 282 F F F 5 100-119 170 170 170 289 289 289 F F F 6 120-139 167 167 167 284 284 284 F F F 7 140-159 171 171 171 291 291 291 F F F 8 160-179 170 170 170 290 290 290 F F F 9 180-199 166 166 166 282 282 282 F F F 10 200-219 169 169 169 286 286 286 F F F 11 220-239 169 169 169 286 286 286 F F F 12 240-259 169 169 169 286 286 286 F F F 13 260-279 169 169 169 286 286 286 F F F 14 280-299 167 167 167 284 284 284 F F F 15 300-319 169 169 169 286 286 286 F F F 16 320-339 169 169 169 286 286 286 F F F 17 340-359 166 166 166 282 282 282 F F F 18 360-379 170 170 170 288 288 288 F F F 19 380-399 170 170 170 288 288 288 F F F 20 400-419 161 161 161 274 274 274 F F F 21 420-439 161 161 161 274 274 274 F F F 22 440-459 170 170 170 289 289 289 F F F 23 460-479 170 170 170 289 289 289 F F F 24 480-499 156 156 156 264 264 264 F F F 25 500-519 169 169 169 286 286 286 F F F 26 520-539 155 155 155 263 263 263 F F F 27 540-559 170 170 170 289 289 289 F F F 28 560-579 170 170 170 289 289 289 F F F 29 580-599 166 166 166 282 282 282 F F F Max gen: 171, Max flush: 291 Begin loop 86 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:40.236568365Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1868,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:56:40.279426957Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1868,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:40.279808786Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1868,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:40.311468576Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1868,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:40.311975621Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1868} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"3818f0fc76fe787c6cc412af30fd59b930203885\",\n git_commit_timestamp: \"2024-11-01T16:08:30.000000000Z\",\n git_branch: \"renovate/hyper-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:40.312007805Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1868} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:40.312022297Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1868} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:40.312036479Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1868} {"msg":"Crucible 81d2439b-c080-45f5-a703-02d5a5eea4d3 has session id: 5877b839-3583-4a06-9003-8b298bcf0019","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:40.312497748Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1868,"session_id":"5877b839-3583-4a06-9003-8b298bcf0019"} {"msg":"Upstairs opts: Upstairs UUID: 81d2439b-c080-45f5-a703-02d5a5eea4d3, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:40.312538663Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1868,"session_id":"5877b839-3583-4a06-9003-8b298bcf0019"} {"msg":"Crucible stats registered with UUID: 81d2439b-c080-45f5-a703-02d5a5eea4d3","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:40.312564127Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1868,"session_id":"5877b839-3583-4a06-9003-8b298bcf0019"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:40.343518895Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1868,"task":"crutest"} {"msg":"The guest has requested activation with gen:172","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:40.343550339Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1868} {"msg":"81d2439b-c080-45f5-a703-02d5a5eea4d3 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:40.343585994Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1868,"session_id":"5877b839-3583-4a06-9003-8b298bcf0019"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:40.343606906Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1868,"client":"0","":"downstairs","session_id":"5877b839-3583-4a06-9003-8b298bcf0019"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:40.34363664Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1868,"client":"0","":"downstairs","session_id":"5877b839-3583-4a06-9003-8b298bcf0019"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:40.343661433Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1868,"client":"1","":"downstairs","session_id":"5877b839-3583-4a06-9003-8b298bcf0019"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:40.343692897Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1868,"client":"1","":"downstairs","session_id":"5877b839-3583-4a06-9003-8b298bcf0019"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:40.343727122Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1868,"client":"2","":"downstairs","session_id":"5877b839-3583-4a06-9003-8b298bcf0019"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:40.343758126Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1868,"client":"2","":"downstairs","session_id":"5877b839-3583-4a06-9003-8b298bcf0019"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:40.34378688Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1868,"":"io task","client":"2","":"downstairs","session_id":"5877b839-3583-4a06-9003-8b298bcf0019"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:40.343814003Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1868,"":"io task","client":"0","":"downstairs","session_id":"5877b839-3583-4a06-9003-8b298bcf0019"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:40.343838316Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1868,"":"io task","client":"1","":"downstairs","session_id":"5877b839-3583-4a06-9003-8b298bcf0019"} {"msg":"ds_connection connected from Ok(127.0.0.1:42792)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:40.343879141Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1868,"":"io task","client":"1","":"downstairs","session_id":"5877b839-3583-4a06-9003-8b298bcf0019"} {"msg":"ds_connection connected from Ok(127.0.0.1:38013)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:40.343908705Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1868,"":"io task","client":"2","":"downstairs","session_id":"5877b839-3583-4a06-9003-8b298bcf0019"} {"msg":"ds_connection connected from Ok(127.0.0.1:59670)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:40.344047813Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1868,"":"io task","client":"0","":"downstairs","session_id":"5877b839-3583-4a06-9003-8b298bcf0019"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:40.344663342Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1868,"client":"1","":"downstairs","session_id":"5877b839-3583-4a06-9003-8b298bcf0019"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:40.344696917Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1868,"client":"1","":"downstairs","session_id":"5877b839-3583-4a06-9003-8b298bcf0019"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:40.344717569Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1868,"client":"2","":"downstairs","session_id":"5877b839-3583-4a06-9003-8b298bcf0019"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:40.344737492Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1868,"client":"2","":"downstairs","session_id":"5877b839-3583-4a06-9003-8b298bcf0019"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:40.344826664Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1868,"client":"0","":"downstairs","session_id":"5877b839-3583-4a06-9003-8b298bcf0019"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:40.344846856Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1868,"client":"0","":"downstairs","session_id":"5877b839-3583-4a06-9003-8b298bcf0019"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:40.345323798Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1868,"client":"1","":"downstairs","session_id":"5877b839-3583-4a06-9003-8b298bcf0019"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:40.345446203Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1868,"client":"0","":"downstairs","session_id":"5877b839-3583-4a06-9003-8b298bcf0019"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:40.345475557Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1868,"client":"2","":"downstairs","session_id":"5877b839-3583-4a06-9003-8b298bcf0019"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:40.345926805Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1868,"client":"1","":"downstairs","session_id":"5877b839-3583-4a06-9003-8b298bcf0019"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:40.345954899Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1868,"client":"1","":"downstairs","session_id":"5877b839-3583-4a06-9003-8b298bcf0019"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:40.345969541Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1868,"session_id":"5877b839-3583-4a06-9003-8b298bcf0019"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:40.346035269Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1868,"client":"0","":"downstairs","session_id":"5877b839-3583-4a06-9003-8b298bcf0019"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:40.346054342Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1868,"client":"0","":"downstairs","session_id":"5877b839-3583-4a06-9003-8b298bcf0019"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:40.346067683Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1868,"session_id":"5877b839-3583-4a06-9003-8b298bcf0019"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:40.346079785Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1868,"client":"2","":"downstairs","session_id":"5877b839-3583-4a06-9003-8b298bcf0019"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:40.346093197Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1868,"client":"2","":"downstairs","session_id":"5877b839-3583-4a06-9003-8b298bcf0019"} {"msg":"[0]R flush_numbers[0..12]: [289, 289, 286, 288, 282, 289, 284, 291, 290, 282, 286, 286]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:40.346107149Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1868,"":"downstairs","":"downstairs","session_id":"5877b839-3583-4a06-9003-8b298bcf0019"} {"msg":"[0]R generation[0..12]: [170, 170, 169, 170, 166, 170, 167, 171, 170, 166, 169, 169]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:40.346125231Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1868,"":"downstairs","":"downstairs","session_id":"5877b839-3583-4a06-9003-8b298bcf0019"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:40.346139963Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1868,"":"downstairs","":"downstairs","session_id":"5877b839-3583-4a06-9003-8b298bcf0019"} {"msg":"[1]R flush_numbers[0..12]: [289, 289, 286, 288, 282, 289, 284, 291, 290, 282, 286, 286]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:40.346154025Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1868,"":"downstairs","":"downstairs","session_id":"5877b839-3583-4a06-9003-8b298bcf0019"} {"msg":"[1]R generation[0..12]: [170, 170, 169, 170, 166, 170, 167, 171, 170, 166, 169, 169]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:40.346167867Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1868,"":"downstairs","":"downstairs","session_id":"5877b839-3583-4a06-9003-8b298bcf0019"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:40.346181908Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1868,"":"downstairs","":"downstairs","session_id":"5877b839-3583-4a06-9003-8b298bcf0019"} {"msg":"[2]R flush_numbers[0..12]: [289, 289, 286, 288, 282, 289, 284, 291, 290, 282, 286, 286]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:40.34619569Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1868,"":"downstairs","":"downstairs","session_id":"5877b839-3583-4a06-9003-8b298bcf0019"} {"msg":"[2]R generation[0..12]: [170, 170, 169, 170, 166, 170, 167, 171, 170, 166, 169, 169]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:40.346209682Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1868,"":"downstairs","":"downstairs","session_id":"5877b839-3583-4a06-9003-8b298bcf0019"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:40.346223844Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1868,"":"downstairs","":"downstairs","session_id":"5877b839-3583-4a06-9003-8b298bcf0019"} {"msg":"Max found gen is 172","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:40.346236945Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1868,"":"downstairs","":"downstairs","session_id":"5877b839-3583-4a06-9003-8b298bcf0019"} {"msg":"Generation requested: 172 >= found:172","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:40.346250127Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1868,"":"downstairs","":"downstairs","session_id":"5877b839-3583-4a06-9003-8b298bcf0019"} {"msg":"Next flush: 292","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:40.346263189Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1868,"":"downstairs","":"downstairs","session_id":"5877b839-3583-4a06-9003-8b298bcf0019"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:40.34627617Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1868,"":"downstairs","":"downstairs","session_id":"5877b839-3583-4a06-9003-8b298bcf0019"} {"msg":"No downstairs reconciliation required","v":0Wait for a query_work_queue command to finish before sending IO ,"name":"crucible","level":30,"time":"2024-11-01T16:56:40.346300744Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1868,"session_id":"5877b839-3583-4a06-9003-8b298bcf0019"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:40.346313025Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1868,"client":"0","":"downstairs","session_id":"5877b839-3583-4a06-9003-8b298bcf0019"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:40.346326357Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1868,"client":"1","":"downstairs","session_id":"5877b839-3583-4a06-9003-8b298bcf0019"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:40.346342209Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1868,"client":"2","":"downstairs","session_id":"5877b839-3583-4a06-9003-8b298bcf0019"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:40.346355721Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1868,"session_id":"5877b839-3583-4a06-9003-8b298bcf0019"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:40.346368102Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1868,"session_id":"5877b839-3583-4a06-9003-8b298bcf0019"} {"msg":"81d2439b-c080-45f5-a703-02d5a5eea4d3 is now active with session: 5877b839-3583-4a06-9003-8b298bcf0019","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:40.346380974Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1868,"session_id":"5877b839-3583-4a06-9003-8b298bcf0019"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:40.346392825Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1868,"session_id":"5877b839-3583-4a06-9003-8b298bcf0019"} {"msg":"The guest has finished waiting for activation with:172","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:40.346405107Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1868} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 164 len 28672 data: 13 14 12 11 12 11 12 02/30 Read block 49 len 36864 03/30 Read block 129 len 20480 04/30 Write block 287 len 32768 data: 10 10 11 9 12 14 14 16 05/30 Read block 235 len 12288 06/30 Read block 468 len 40960 07/30 Write block 45 len 36864 data: 15 16 12 10 13 13 11 13 13 08/30 Read block 278 len 36864 09/30 Read block 290 len 40960 10/30 Read block 581 len 36864 11/30 Read block 544 len 20480 12/30 Read block 211 len 12288 13/30 Write block 73 len 32768 data: 16 17 16 10 8 9 13 13 14/30 Read block 113 len 4096 15/30 Read block 17 len 4096 16/30 Write block 440 len 4096 data: 12 17/30 Write block 285 len 32768 data: 5 10 11 11 12 10 13 15 18/30 Write block 424 len 4096 data: 14 19/30 Write block 282 len 28672 data: 11 7 5 6 11 12 12 20/30 Read block 320 len 24576 21/30 Write block 285 len 36864 data: 7 12 13 13 13 11 14 16 15 22/30 Write block 93 len 12288 data: 17 14 17 23/30 Write block 79 len 12288 data: 14 14 13 24/30 Read block 61 len 20480 25/30 Flush 26/30 Write block 446 len 28672 data: 11 13 13 14 18 17 15 27/30 Read block 8 len 36864 28/30 Write block 65 len 24576 data: 11 16 17 17 19 22 29/30 Read block 110 len 16384 30/30 Read block 585 len 4096 ---------------------------------------------------------------- Crucible gen:172 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 7 Done Done Done false 1026 Acked Read 9 Done Done Done false 1027 Acked Write 6 Done Done Done false 1028 Acked Read 4 Done Done Done false 1029 Acked Read 1 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:56:40.370914935Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1868} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 170 170 170 289 289 289 F F F 1 020-039 170 170 170 289 289 289 F F F 2 040-059 172 172 172 292 292 292 F F F 3 060-079 172 172 172 292 292 292 T T T 4 080-099 172 172 172 292 292 292 F F F 5 100-119 170 170 170 289 289 289 F F F 6 120-139 167 167 167 284 284 284 F F F 7 140-159 171 171 171 291 291 291 F F F 8 160-179 172 172 172 292 292 292 F F F 9 180-199 166 166 166 282 282 282 F F F 10 200-219 169 169 169 286 286 286 F F F 11 220-239 169 169 169 286 286 286 F F F 12 240-259 169 169 169 286 286 286 F F F 13 260-279 169 169 169 286 286 286 F F F 14 280-299 172 172 172 292 292 292 F F F 15 300-319 169 169 169 286 286 286 F F F 16 320-339 169 169 169 286 286 286 F F F 17 340-359 166 166 166 282 282 282 F F F 18 360-379 170 170 170 288 288 288 F F F 19 380-399 170 170 170 288 288 288 F F F 20 400-419 161 161 161 274 274 274 F F F 21 420-439 172 172 172 292 292 292 F F F 22 440-459 172 172 172 292 292 292 T T T 23 460-479 170 170 170 289 289 289 F F F 24 480-499 156 156 156 264 264 264 F F F 25 500-519 169 169 169 286 286 286 F F F 26 520-539 155 155 155 263 263 263 F F F 27 540-559 170 170 170 289 289 289 F F F 28 560-579 170 170 170 289 289 289 F F F 29 580-599 166 166 166 282 282 282 F F F Max gen: 172, Max flush: 292 A Difference in extent metadata is expected here On loop 86 Verifying data now No change after verify Loop: 86 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 170 170 170 289 289 289 F F F 1 020-039 170 170 170 289 289 289 F F F 2 040-059 172 172 172 292 292 292 F F F 3 060-079 173 173 173 293 293 293 F F F 4 080-099 172 172 172 292 292 292 F F F 5 100-119 170 170 170 289 289 289 F F F 6 120-139 167 167 167 284 284 284 F F F 7 140-159 171 171 171 291 291 291 F F F 8 160-179 172 172 172 292 292 292 F F F 9 180-199 166 166 166 282 282 282 F F F 10 200-219 169 169 169 286 286 286 F F F 11 220-239 169 169 169 286 286 286 F F F 12 240-259 169 169 169 286 286 286 F F F 13 260-279 169 169 169 286 286 286 F F F 14 280-299 172 172 172 292 292 292 F F F 15 300-319 169 169 169 286 286 286 F F F 16 320-339 169 169 169 286 286 286 F F F 17 340-359 166 166 166 282 282 282 F F F 18 360-379 170 170 170 288 288 288 F F F 19 380-399 170 170 170 288 288 288 F F F 20 400-419 161 161 161 274 274 274 F F F 21 420-439 172 172 172 292 292 292 F F F 22 440-459 173 173 173 293 293 293 F F F 23 460-479 170 170 170 289 289 289 F F F 24 480-499 156 156 156 264 264 264 F F F 25 500-519 169 169 169 286 286 286 F F F 26 520-539 155 155 155 263 263 263 F F F 27 540-559 170 170 170 289 289 289 F F F 28 560-579 170 170 170 289 289 289 F F F 29 580-599 166 166 166 282 282 282 F F F Max gen: 173, Max flush: 293 Begin loop 87 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:44.670813386Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1876,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:44.71523826Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1876,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:44.715775339Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1876} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"3818f0fc76fe787c6cc412af30fd59b930203885\",\n git_commit_timestamp: \"2024-11-01T16:08:30.000000000Z\",\n git_branch: \"renovate/hyper-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:44.715813834Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1876} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:44.715828096Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1876} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:44.715842728Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1876} {"msg":"Crucible 21b36320-3b54-46f5-898d-128105c06130 has session id: d20f739f-80ef-4834-9edc-093d5cae91ee","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:44.716297506Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1876,"session_id":"d20f739f-80ef-4834-9edc-093d5cae91ee"} {"msg":"Upstairs opts: Upstairs UUID: 21b36320-3b54-46f5-898d-128105c06130, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:44.716336821Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1876,"session_id":"d20f739f-80ef-4834-9edc-093d5cae91ee"} {"msg":"Crucible stats registered with UUID: 21b36320-3b54-46f5-898d-128105c06130","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:44.716355294Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1876,"session_id":"d20f739f-80ef-4834-9edc-093d5cae91ee"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:44.747808726Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1876,"task":"crutest"} {"msg":"The guest has requested activation with gen:174","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:44.74783926Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1876} {"msg":"21b36320-3b54-46f5-898d-128105c06130 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:44.747888117Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1876,"session_id":"d20f739f-80ef-4834-9edc-093d5cae91ee"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:44.747909489Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1876,"client":"0","":"downstairs","session_id":"d20f739f-80ef-4834-9edc-093d5cae91ee"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:44.747938993Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1876,"client":"0","":"downstairs","session_id":"d20f739f-80ef-4834-9edc-093d5cae91ee"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:44.747956395Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1876,"client":"1","":"downstairs","session_id":"d20f739f-80ef-4834-9edc-093d5cae91ee"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:44.747983799Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1876,"client":"1","":"downstairs","session_id":"d20f739f-80ef-4834-9edc-093d5cae91ee"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:44.748015223Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1876,"client":"2","":"downstairs","session_id":"d20f739f-80ef-4834-9edc-093d5cae91ee"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:44.748047737Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1876,"client":"2","":"downstairs","session_id":"d20f739f-80ef-4834-9edc-093d5cae91ee"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:44.748080791Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1876,"":"io task","client":"2","":"downstairs","session_id":"d20f739f-80ef-4834-9edc-093d5cae91ee"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:44.748113936Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1876,"":"io task","client":"0","":"downstairs","session_id":"d20f739f-80ef-4834-9edc-093d5cae91ee"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:44.74814516Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1876,"":"io task","client":"1","":"downstairs","session_id":"d20f739f-80ef-4834-9edc-093d5cae91ee"} {"msg":"ds_connection connected from Ok(127.0.0.1:51470)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:44.748175954Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1876,"":"io task","client":"2","":"downstairs","session_id":"d20f739f-80ef-4834-9edc-093d5cae91ee"} {"msg":"ds_connection connected from Ok(127.0.0.1:52961)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:44.748208718Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1876,"":"io task","client":"1","":"downstairs","session_id":"d20f739f-80ef-4834-9edc-093d5cae91ee"} {"msg":"ds_connection connected from Ok(127.0.0.1:42259)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:44.748372189Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1876,"":"io task","client":"0","":"downstairs","session_id":"d20f739f-80ef-4834-9edc-093d5cae91ee"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:44.749034444Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1876,"client":"0","":"downstairs","session_id":"d20f739f-80ef-4834-9edc-093d5cae91ee"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:44.749061218Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1876,"client":"0","":"downstairs","session_id":"d20f739f-80ef-4834-9edc-093d5cae91ee"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:44.749085431Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1876,"client":"1","":"downstairs","session_id":"d20f739f-80ef-4834-9edc-093d5cae91ee"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:44.749100713Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1876,"client":"1","":"downstairs","session_id":"d20f739f-80ef-4834-9edc-093d5cae91ee"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:44.749114435Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1876,"client":"2","":"downstairs","session_id":"d20f739f-80ef-4834-9edc-093d5cae91ee"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:44.749127546Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1876,"client":"2","":"downstairs","session_id":"d20f739f-80ef-4834-9edc-093d5cae91ee"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:44.749737945Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1876,"client":"0","":"downstairs","session_id":"d20f739f-80ef-4834-9edc-093d5cae91ee"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:44.749771529Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1876,"client":"2","":"downstairs","session_id":"d20f739f-80ef-4834-9edc-093d5cae91ee"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:44.749788591Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1876,"client":"1","":"downstairs","session_id":"d20f739f-80ef-4834-9edc-093d5cae91ee"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:44.750334512Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1876,"client":"1","":"downstairs","session_id":"d20f739f-80ef-4834-9edc-093d5cae91ee"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:44.750360175Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1876,"client":"1","":"downstairs","session_id":"d20f739f-80ef-4834-9edc-093d5cae91ee"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:44.750374057Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1876,"session_id":"d20f739f-80ef-4834-9edc-093d5cae91ee"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:44.750386628Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1876,"client":"2","":"downstairs","session_id":"d20f739f-80ef-4834-9edc-093d5cae91ee"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:44.75040015Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1876,"client":"2","":"downstairs","session_id":"d20f739f-80ef-4834-9edc-093d5cae91ee"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:44.750413672Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1876,"session_id":"d20f739f-80ef-4834-9edc-093d5cae91ee"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:44.750425944Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1876,"client":"0","":"downstairs","session_id":"d20f739f-80ef-4834-9edc-093d5cae91ee"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:44.750439285Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1876,"client":"0","":"downstairs","session_id":"d20f739f-80ef-4834-9edc-093d5cae91ee"} {"msg":"[0]R flush_numbers[0..12]: [289, 289, 292, 293, 292, 289, 284, 291, 292, 282, 286, 286]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:44.750453027Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1876,"":"downstairs","":"downstairs","session_id":"d20f739f-80ef-4834-9edc-093d5cae91ee"} {"msg":"[0]R generation[0..12]: [170, 170, 172, 173, 172, 170, 167, 171, 172, 166, 169, 169]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:44.750466959Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1876,"":"downstairs","":"downstairs","session_id":"d20f739f-80ef-4834-9edc-093d5cae91ee"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:44.750481111Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1876,"":"downstairs","":"downstairs","session_id":"d20f739f-80ef-4834-9edc-093d5cae91ee"} {"msg":"[1]R flush_numbers[0..12]: [289, 289, 292, 293, 292, 289, 284, 291, 292, 282, 286, 286]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:44.750494852Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1876,"":"downstairs","":"downstairs","session_id":"d20f739f-80ef-4834-9edc-093d5cae91ee"} {"msg":"[1]R generation[0..12]: [170, 170, 172, 173, 172, 170, 167, 171, 172, 166, 169, 169]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:44.750511664Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1876,"":"downstairs","":"downstairs","session_id":"d20f739f-80ef-4834-9edc-093d5cae91ee"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:44.750526066Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1876,"":"downstairs","":"downstairs","session_id":"d20f739f-80ef-4834-9edc-093d5cae91ee"} {"msg":"[2]R flush_numbers[0..12]: [289, 289, 292, 293, 292, 289, 284, 291, 292, 282, 286, 286]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:44.750539848Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1876,"":"downstairs","":"downstairs","session_id":"d20f739f-80ef-4834-9edc-093d5cae91ee"} {"msg":"[2]R generation[0..12]: [170, 170, 172, 173, 172, 170, 167, 171, 172, 166, 169, 169]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:44.75055362Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1876,"":"downstairs","":"downstairs","session_id":"d20f739f-80ef-4834-9edc-093d5cae91ee"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:44.750567472Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1876,"":"downstairs","":"downstairs","session_id":"d20f739f-80ef-4834-9edc-093d5cae91ee"} {"msg":"Max found gen is 174","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:44.750580453Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1876,"":"downstairs","":"downstairs","session_id":"d20f739f-80ef-4834-9edc-093d5cae91ee"} {"msg":"Generation requested: 174 >= found:174","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:44.750593765Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1876,"":"downstairs","":"downstairs","session_id":"d20f739f-80ef-4834-9edc-093d5cae91ee"} {"msg":"Next flush: 294","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:44.750606867Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1876,"":"downstairs","":"downstairs","session_id":"d20f739f-80ef-4834-9edc-093d5cae91ee"} {Wait for a query_work_queue command to finish before sending IO "msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:44.75063115Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1876,"":"downstairs","":"downstairs","session_id":"d20f739f-80ef-4834-9edc-093d5cae91ee"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:44.750644812Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1876,"session_id":"d20f739f-80ef-4834-9edc-093d5cae91ee"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:44.750656873Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1876,"client":"0","":"downstairs","session_id":"d20f739f-80ef-4834-9edc-093d5cae91ee"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:44.750670165Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1876,"client":"1","":"downstairs","session_id":"d20f739f-80ef-4834-9edc-093d5cae91ee"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:44.750683247Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1876,"client":"2","":"downstairs","session_id":"d20f739f-80ef-4834-9edc-093d5cae91ee"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:44.750696448Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1876,"session_id":"d20f739f-80ef-4834-9edc-093d5cae91ee"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:44.75070866Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1876,"session_id":"d20f739f-80ef-4834-9edc-093d5cae91ee"} {"msg":"21b36320-3b54-46f5-898d-128105c06130 is now active with session: d20f739f-80ef-4834-9edc-093d5cae91ee","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:44.750723362Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1876,"session_id":"d20f739f-80ef-4834-9edc-093d5cae91ee"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:44.750735433Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1876,"session_id":"d20f739f-80ef-4834-9edc-093d5cae91ee"} {"msg":"The guest has finished waiting for activation with:174","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:44.750747695Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1876} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 418 len 40960 data: 13 11 11 13 14 14 15 13 15 14 02/30 Read block 359 len 20480 03/30 Write block 154 len 40960 data: 17 15 16 14 13 12 14 13 11 13 04/30 Write block 304 len 4096 data: 9 05/30 Read block 227 len 12288 06/30 Read block 510 len 12288 07/30 Read block 290 len 20480 08/30 Write block 326 len 4096 data: 9 09/30 Read block 479 len 4096 10/30 Read block 123 len 12288 11/30 Write block 516 len 20480 data: 8 10 10 13 12 12/30 Read block 72 len 28672 13/30 Flush 14/30 Write block 473 len 40960 data: 18 15 15 12 17 21 18 16 14 13 15/30 Write block 426 len 32768 data: 16 15 14 12 11 12 7 7 16/30 Flush 17/30 Write block 495 len 28672 data: 12 12 14 16 16 15 13 18/30 Read block 71 len 36864 19/30 Write block 326 len 28672 data: 10 8 6 7 8 8 8 20/30 Read block 337 len 16384 21/30 Read block 433 len 28672 22/30 Flush 23/30 Flush 24/30 Write block 418 len 16384 data: 14 12 12 14 25/30 Read block 376 len 12288 26/30 Read block 31 len 20480 27/30 Read block 435 len 8192 28/30 Write block 469 len 24576 data: 14 16 18 18 19 16 29/30 Write block 589 len 40960 data: 15 13 12 10 11 10 12 11 10 8 30/30 Write block 508 len 28672 data: 14 12 12 11 11 8 10 ---------------------------------------------------------------- Crucible gen:174 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 4 Done Done Done false 1024 Acked Read 3 Done Done Done false 1025 Acked Read 5 Done Done Done false 1026 Acked Read 2 Done Done Done false 1027 Acked Write 6 Done Done Sent false 1028 Acked Write 10 Sent Sent Sent false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 3 7 Done 5 5 4 14 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:56:44.777520474Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1876} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 170 170 170 289 289 289 F F F 1 020-039 170 170 170 289 289 289 F F F 2 040-059 172 172 172 292 292 292 F F F 3 060-079 173 173 173 293 293 293 F F F 4 080-099 172 172 172 292 292 292 F F F 5 100-119 170 170 170 289 289 289 F F F 6 120-139 167 167 167 284 284 284 F F F 7 140-159 174 174 174 294 294 294 F F F 8 160-179 174 174 174 294 294 294 F F F 9 180-199 166 166 166 282 282 282 F F F 10 200-219 169 169 169 286 286 286 F F F 11 220-239 169 169 169 286 286 286 F F F 12 240-259 169 169 169 286 286 286 F F F 13 260-279 169 169 169 286 286 286 F F F 14 280-299 172 172 172 292 292 292 F F F 15 300-319 174 174 174 294 294 294 F F F 16 320-339 174 174 174 296 296 296 F F F 17 340-359 166 166 166 282 282 282 F F F 18 360-379 170 170 170 288 288 288 F F F 19 380-399 170 170 170 288 288 288 F F F 20 400-419 174 174 174 294 294 294 T T T 21 420-439 174 174 174 295 295 295 T T T 22 440-459 173 173 173 293 293 293 F F F 23 460-479 174 174 174 295 295 295 T T T 24 480-499 174 174 174 296 296 296 F F F 25 500-519 174 174 174 296 296 296 T T T 26 520-539 174 174 174 294 294 294 F F F 27 540-559 170 170 170 289 289 289 F F F 28 560-579 170 170 170 289 289 289 F F F 29 580-599 166 166 166 282 282 282 T T T Max gen: 174, Max flush: 296 A Difference in extent metadata is expected here On loop 87 Verifying data now No change after verify Loop: 87 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 170 170 170 289 289 289 F F F 1 020-039 170 170 170 289 289 289 F F F 2 040-059 172 172 172 292 292 292 F F F 3 060-079 173 173 173 293 293 293 F F F 4 080-099 172 172 172 292 292 292 F F F 5 100-119 170 170 170 289 289 289 F F F 6 120-139 167 167 167 284 284 284 F F F 7 140-159 174 174 174 294 294 294 F F F 8 160-179 174 174 174 294 294 294 F F F 9 180-199 166 166 166 282 282 282 F F F 10 200-219 169 169 169 286 286 286 F F F 11 220-239 169 169 169 286 286 286 F F F 12 240-259 169 169 169 286 286 286 F F F 13 260-279 169 169 169 286 286 286 F F F 14 280-299 172 172 172 292 292 292 F F F 15 300-319 174 174 174 294 294 294 F F F 16 320-339 174 174 174 296 296 296 F F F 17 340-359 166 166 166 282 282 282 F F F 18 360-379 170 170 170 288 288 288 F F F 19 380-399 170 170 170 288 288 288 F F F 20 400-419 175 175 175 297 297 297 F F F 21 420-439 175 175 175 297 297 297 F F F 22 440-459 173 173 173 293 293 293 F F F 23 460-479 175 175 175 297 297 297 F F F 24 480-499 174 174 174 296 296 296 F F F 25 500-519 175 175 175 297 297 297 F F F 26 520-539 174 174 174 294 294 294 F F F 27 540-559 170 170 170 289 289 289 F F F 28 560-579 170 170 170 289 289 289 F F F 29 580-599 175 175 175 297 297 297 F F F Max gen: 175, Max flush: 297 Begin loop 88 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:49.082069503Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1884,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:49.125475776Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1884,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:49.126027277Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1884} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"3818f0fc76fe787c6cc412af30fd59b930203885\",\n git_commit_timestamp: \"2024-11-01T16:08:30.000000000Z\",\n git_branch: \"renovate/hyper-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:49.126068842Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1884} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:49.126087005Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1884} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:49.126101826Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1884} {"msg":"Crucible b2dad896-c22f-4d52-802a-6ced6ac3a6ee has session id: 3f167219-efd2-4fce-ac73-a6a2a9dd9838","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:49.12899991Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1884,"session_id":"3f167219-efd2-4fce-ac73-a6a2a9dd9838"} {"msg":"Upstairs opts: Upstairs UUID: b2dad896-c22f-4d52-802a-6ced6ac3a6ee, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:49.129028984Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1884,"session_id":"3f167219-efd2-4fce-ac73-a6a2a9dd9838"} {"msg":"Crucible stats registered with UUID: b2dad896-c22f-4d52-802a-6ced6ac3a6ee","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:49.129043986Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1884,"session_id":"3f167219-efd2-4fce-ac73-a6a2a9dd9838"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:49.159751142Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1884,"task":"crutest"} {"msg":"The guest has requested activation with gen:176","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:49.159786897Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1884} {"msg":"b2dad896-c22f-4d52-802a-6ced6ac3a6ee active request set","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:49.159822471Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1884,"session_id":"3f167219-efd2-4fce-ac73-a6a2a9dd9838"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:49.159842144Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1884,"client":"0","":"downstairs","session_id":"3f167219-efd2-4fce-ac73-a6a2a9dd9838"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:49.159871378Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1884,"client":"0","":"downstairs","session_id":"3f167219-efd2-4fce-ac73-a6a2a9dd9838"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:49.15988878Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1884,"client":"1","":"downstairs","session_id":"3f167219-efd2-4fce-ac73-a6a2a9dd9838"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:49.159905802Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1884,"client":"1","":"downstairs","session_id":"3f167219-efd2-4fce-ac73-a6a2a9dd9838"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:49.159922324Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1884,"client":"2","":"downstairs","session_id":"3f167219-efd2-4fce-ac73-a6a2a9dd9838"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:49.159938806Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1884,"client":"2","":"downstairs","session_id":"3f167219-efd2-4fce-ac73-a6a2a9dd9838"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:49.159956989Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1884,"":"io task","client":"2","":"downstairs","session_id":"3f167219-efd2-4fce-ac73-a6a2a9dd9838"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:49.159984672Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1884,"":"io task","client":"0","":"downstairs","session_id":"3f167219-efd2-4fce-ac73-a6a2a9dd9838"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:49.160028758Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1884,"":"io task","client":"1","":"downstairs","session_id":"3f167219-efd2-4fce-ac73-a6a2a9dd9838"} {"msg":"ds_connection connected from Ok(127.0.0.1:58628)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:49.160058672Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1884,"":"io task","client":"2","":"downstairs","session_id":"3f167219-efd2-4fce-ac73-a6a2a9dd9838"} {"msg":"ds_connection connected from Ok(127.0.0.1:42040)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:49.160104218Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1884,"":"io task","client":"1","":"downstairs","session_id":"3f167219-efd2-4fce-ac73-a6a2a9dd9838"} {"msg":"ds_connection connected from Ok(127.0.0.1:65358)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:49.160316075Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1884,"":"io task","client":"0","":"downstairs","session_id":"3f167219-efd2-4fce-ac73-a6a2a9dd9838"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:49.160768713Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1884,"client":"0","":"downstairs","session_id":"3f167219-efd2-4fce-ac73-a6a2a9dd9838"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:49.160796137Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1884,"client":"0","":"downstairs","session_id":"3f167219-efd2-4fce-ac73-a6a2a9dd9838"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:49.160811799Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1884,"client":"1","":"downstairs","session_id":"3f167219-efd2-4fce-ac73-a6a2a9dd9838"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:49.160825621Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1884,"client":"1","":"downstairs","session_id":"3f167219-efd2-4fce-ac73-a6a2a9dd9838"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:49.16089689Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1884,"client":"2","":"downstairs","session_id":"3f167219-efd2-4fce-ac73-a6a2a9dd9838"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:49.160937975Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1884,"client":"2","":"downstairs","session_id":"3f167219-efd2-4fce-ac73-a6a2a9dd9838"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:49.161399205Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1884,"client":"1","":"downstairs","session_id":"3f167219-efd2-4fce-ac73-a6a2a9dd9838"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:49.161427538Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1884,"client":"0","":"downstairs","session_id":"3f167219-efd2-4fce-ac73-a6a2a9dd9838"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:49.16159261Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1884,"client":"2","":"downstairs","session_id":"3f167219-efd2-4fce-ac73-a6a2a9dd9838"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:49.161969448Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1884,"client":"1","":"downstairs","session_id":"3f167219-efd2-4fce-ac73-a6a2a9dd9838"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:49.162001772Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1884,"client":"1","":"downstairs","session_id":"3f167219-efd2-4fce-ac73-a6a2a9dd9838"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:49.162018164Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1884,"session_id":"3f167219-efd2-4fce-ac73-a6a2a9dd9838"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:49.162112006Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1884,"client":"0","":"downstairs","session_id":"3f167219-efd2-4fce-ac73-a6a2a9dd9838"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:49.162144661Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1884,"client":"0","":"downstairs","session_id":"3f167219-efd2-4fce-ac73-a6a2a9dd9838"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:49.162169614Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1884,"session_id":"3f167219-efd2-4fce-ac73-a6a2a9dd9838"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:49.162183606Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1884,"client":"2","":"downstairs","session_id":"3f167219-efd2-4fce-ac73-a6a2a9dd9838"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:49.162197047Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1884,"client":"2","":"downstairs","session_id":"3f167219-efd2-4fce-ac73-a6a2a9dd9838"} {"msg":"[0]R flush_numbers[0..12]: [289, 289, 292, 293, 292, 289, 284, 294, 294, 282, 286, 286]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:49.162210929Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1884,"":"downstairs","":"downstairs","session_id":"3f167219-efd2-4fce-ac73-a6a2a9dd9838"} {"msg":"[0]R generation[0..12]: [170, 170, 172, 173, 172, 170, 167, 174, 174, 166, 169, 169]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:49.162225121Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1884,"":"downstairs","":"downstairs","session_id":"3f167219-efd2-4fce-ac73-a6a2a9dd9838"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:49.162239283Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1884,"":"downstairs","":"downstairs","session_id":"3f167219-efd2-4fce-ac73-a6a2a9dd9838"} {"msg":"[1]R flush_numbers[0..12]: [289, 289, 292, 293, 292, 289, 284, 294, 294, 282, 286, 286]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:49.162253155Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1884,"":"downstairs","":"downstairs","session_id":"3f167219-efd2-4fce-ac73-a6a2a9dd9838"} {"msg":"[1]R generation[0..12]: [170, 170, 172, 173, 172, 170, 167, 174, 174, 166, 169, 169]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:49.162266956Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1884,"":"downstairs","":"downstairs","session_id":"3f167219-efd2-4fce-ac73-a6a2a9dd9838"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:49.162280778Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1884,"":"downstairs","":"downstairs","session_id":"3f167219-efd2-4fce-ac73-a6a2a9dd9838"} {"msg":"[2]R flush_numbers[0..12]: [289, 289, 292, 293, 292, 289, 284, 294, 294, 282, 286, 286]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:49.1622945Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1884,"":"downstairs","":"downstairs","session_id":"3f167219-efd2-4fce-ac73-a6a2a9dd9838"} {"msg":"[2]R generation[0..12]: [170, 170, 172, 173, 172, 170, 167, 174, 174, 166, 169, 169]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:49.162311072Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1884,"":"downstairs","":"downstairs","session_id":"3f167219-efd2-4fce-ac73-a6a2a9dd9838"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:49.162325434Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1884,"":"downstairs","":"downstairs","session_id":"3f167219-efd2-4fce-ac73-a6a2a9dd9838"} {"msg":"Max found gen is 176","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:49.162341436Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1884,"":"downstairs","":"downstairs","session_id":"3f167219-efd2-4fce-ac73-a6a2a9dd9838"} {"msg":"Generation requested: 176 >= found:176","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:49.162365849Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1884,"":"downstairs","":"downstairs","session_idWait for a query_work_queue command to finish before sending IO ":"3f167219-efd2-4fce-ac73-a6a2a9dd9838"} {"msg":"Next flush: 298","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:49.162402454Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1884,"":"downstairs","":"downstairs","session_id":"3f167219-efd2-4fce-ac73-a6a2a9dd9838"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:49.162427777Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1884,"":"downstairs","":"downstairs","session_id":"3f167219-efd2-4fce-ac73-a6a2a9dd9838"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:49.16244853Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1884,"session_id":"3f167219-efd2-4fce-ac73-a6a2a9dd9838"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:49.162463242Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1884,"client":"0","":"downstairs","session_id":"3f167219-efd2-4fce-ac73-a6a2a9dd9838"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:49.162487335Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1884,"client":"1","":"downstairs","session_id":"3f167219-efd2-4fce-ac73-a6a2a9dd9838"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:49.162512658Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1884,"client":"2","":"downstairs","session_id":"3f167219-efd2-4fce-ac73-a6a2a9dd9838"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:49.16252902Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1884,"session_id":"3f167219-efd2-4fce-ac73-a6a2a9dd9838"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:49.162541712Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1884,"session_id":"3f167219-efd2-4fce-ac73-a6a2a9dd9838"} {"msg":"b2dad896-c22f-4d52-802a-6ced6ac3a6ee is now active with session: 3f167219-efd2-4fce-ac73-a6a2a9dd9838","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:49.162554523Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1884,"session_id":"3f167219-efd2-4fce-ac73-a6a2a9dd9838"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:49.162566585Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1884,"session_id":"3f167219-efd2-4fce-ac73-a6a2a9dd9838"} {"msg":"The guest has finished waiting for activation with:176","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:49.162578997Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1884} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 596 len 16384 data: 12 11 9 5 02/30 Write block 169 len 4096 data: 12 03/30 Write block 37 len 24576 data: 17 17 16 14 14 16 04/30 Write block 508 len 4096 data: 15 05/30 Write block 293 len 16384 data: 16 17 14 14 06/30 Write block 423 len 8192 data: 15 16 07/30 Write block 437 len 20480 data: 7 10 9 13 10 08/30 Write block 158 len 8192 data: 14 13 09/30 Write block 535 len 12288 data: 16 15 13 10/30 Read block 439 len 40960 11/30 Read block 429 len 20480 12/30 Flush 13/30 Write block 99 len 28672 data: 20 21 21 17 16 16 19 14/30 Flush 15/30 Write block 461 len 32768 data: 11 13 12 10 10 13 14 13 16/30 Read block 360 len 8192 17/30 Read block 305 len 40960 18/30 Write block 171 len 40960 data: 9 8 8 9 11 11 12 13 12 14 19/30 Read block 244 len 4096 20/30 Read block 331 len 32768 21/30 Read block 308 len 24576 22/30 Read block 506 len 28672 23/30 Read block 61 len 4096 24/30 Write block 20 len 20480 data: 14 9 11 15 18 25/30 Read block 429 len 40960 26/30 Write block 525 len 16384 data: 13 12 14 15 27/30 Write block 329 len 32768 data: 8 9 9 9 10 11 13 11 28/30 Read block 14 len 8192 29/30 Read block 588 len 12288 30/30 Read block 374 len 32768 ---------------------------------------------------------------- Crucible gen:176 GIO:true work queues: Upstairs:0 downstairs:16 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1014 Acked Write 8 Done Done Done false 1015 Acked Read 2 Done Done Done false 1016 Acked Read 10 Done Done Done false 1017 Acked Write 10 Done Done Done false 1018 Acked Read 1 Done Done Done false 1019 Acked Read 8 Done Done Done false 1020 Acked Read 6 Done Done Done false 1021 Acked Read 7 Done Done Done false 1022 Acked Read 1 Done Done Done false 1023 Acked Write 5 Done Done Done false 1024 Acked Read 10 Done Done Done false 1025 Acked Write 4 Done Done Done false 1026 Acked Write 8 Done Done Done false 1027 Acked Read 2 Done Done Done false 1028 Acked Read 3 Done Done Done false 1029 Acked Read 8 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 16 16 16 48 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1013 1013 1013 Downstairs last five completed: 1013 1012 1011 1010 1009 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:56:49.185573359Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1884} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 170 170 170 289 289 289 F F F 1 020-039 176 176 176 298 298 298 T T T 2 040-059 176 176 176 298 298 298 F F F 3 060-079 173 173 173 293 293 293 F F F 4 080-099 176 176 176 299 299 299 F F F 5 100-119 176 176 176 299 299 299 F F F 6 120-139 167 167 167 284 284 284 F F F 7 140-159 176 176 176 298 298 298 F F F 8 160-179 176 176 176 298 298 298 T T T 9 180-199 166 166 166 282 282 282 T T T 10 200-219 169 169 169 286 286 286 F F F 11 220-239 169 169 169 286 286 286 F F F 12 240-259 169 169 169 286 286 286 F F F 13 260-279 169 169 169 286 286 286 F F F 14 280-299 176 176 176 298 298 298 F F F 15 300-319 174 174 174 294 294 294 F F F 16 320-339 174 174 174 296 296 296 T T T 17 340-359 166 166 166 282 282 282 F F F 18 360-379 170 170 170 288 288 288 F F F 19 380-399 170 170 170 288 288 288 F F F 20 400-419 175 175 175 297 297 297 F F F 21 420-439 176 176 176 298 298 298 F F F 22 440-459 176 176 176 298 298 298 F F F 23 460-479 175 175 175 297 297 297 T T T 24 480-499 174 174 174 296 296 296 F F F 25 500-519 176 176 176 298 298 298 F F F 26 520-539 176 176 176 298 298 298 T T T 27 540-559 170 170 170 289 289 289 F F F 28 560-579 170 170 170 289 289 289 F F F 29 580-599 176 176 176 298 298 298 F F F Max gen: 176, Max flush: 299 A Difference in extent metadata is expected here On loop 88 Verifying data now No change after verify Loop: 88 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 170 170 170 289 289 289 F F F 1 020-039 177 177 177 300 300 300 F F F 2 040-059 176 176 176 298 298 298 F F F 3 060-079 173 173 173 293 293 293 F F F 4 080-099 176 176 176 299 299 299 F F F 5 100-119 176 176 176 299 299 299 F F F 6 120-139 167 167 167 284 284 284 F F F 7 140-159 176 176 176 298 298 298 F F F 8 160-179 177 177 177 300 300 300 F F F 9 180-199 177 177 177 300 300 300 F F F 10 200-219 169 169 169 286 286 286 F F F 11 220-239 169 169 169 286 286 286 F F F 12 240-259 169 169 169 286 286 286 F F F 13 260-279 169 169 169 286 286 286 F F F 14 280-299 176 176 176 298 298 298 F F F 15 300-319 174 174 174 294 294 294 F F F 16 320-339 177 177 177 300 300 300 F F F 17 340-359 166 166 166 282 282 282 F F F 18 360-379 170 170 170 288 288 288 F F F 19 380-399 170 170 170 288 288 288 F F F 20 400-419 175 175 175 297 297 297 F F F 21 420-439 176 176 176 298 298 298 F F F 22 440-459 176 176 176 298 298 298 F F F 23 460-479 177 177 177 300 300 300 F F F 24 480-499 174 174 174 296 296 296 F F F 25 500-519 176 176 176 298 298 298 F F F 26 520-539 177 177 177 300 300 300 F F F 27 540-559 170 170 170 289 289 289 F F F 28 560-579 170 170 170 289 289 289 F F F 29 580-599 176 176 176 298 298 298 F F F Max gen: 177, Max flush: 300 Begin loop 89 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:53.502673046Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1892,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:56:53.546178052Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1892,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:53.546535947Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1892,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:53.579105134Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1892,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:53.579736275Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1892} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"3818f0fc76fe787c6cc412af30fd59b930203885\",\n git_commit_timestamp: \"2024-11-01T16:08:30.000000000Z\",\n git_branch: \"renovate/hyper-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:53.57977202Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1892} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:53.579790792Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1892} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:53.579805504Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1892} {"msg":"Crucible 6c8b1539-0fb5-4912-b0df-f4a9f6901cc8 has session id: a0c92089-1719-4a7e-a4ab-fa584809b422","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:53.580221328Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1892,"session_id":"a0c92089-1719-4a7e-a4ab-fa584809b422"} {"msg":"Upstairs opts: Upstairs UUID: 6c8b1539-0fb5-4912-b0df-f4a9f6901cc8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:53.580249371Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1892,"session_id":"a0c92089-1719-4a7e-a4ab-fa584809b422"} {"msg":"Crucible stats registered with UUID: 6c8b1539-0fb5-4912-b0df-f4a9f6901cc8","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:53.580263953Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1892,"session_id":"a0c92089-1719-4a7e-a4ab-fa584809b422"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:53.61136271Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1892,"task":"crutest"} {"msg":"The guest has requested activation with gen:178","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:53.611393414Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1892} {"msg":"6c8b1539-0fb5-4912-b0df-f4a9f6901cc8 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:53.611442721Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1892,"session_id":"a0c92089-1719-4a7e-a4ab-fa584809b422"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:53.611462293Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1892,"client":"0","":"downstairs","session_id":"a0c92089-1719-4a7e-a4ab-fa584809b422"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:53.611491027Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1892,"client":"0","":"downstairs","session_id":"a0c92089-1719-4a7e-a4ab-fa584809b422"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:53.611508799Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1892,"client":"1","":"downstairs","session_id":"a0c92089-1719-4a7e-a4ab-fa584809b422"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:53.611529542Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1892,"client":"1","":"downstairs","session_id":"a0c92089-1719-4a7e-a4ab-fa584809b422"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:53.611561696Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1892,"client":"2","":"downstairs","session_id":"a0c92089-1719-4a7e-a4ab-fa584809b422"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:53.61159384Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1892,"client":"2","":"downstairs","session_id":"a0c92089-1719-4a7e-a4ab-fa584809b422"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:53.611626234Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1892,"":"io task","client":"2","":"downstairs","session_id":"a0c92089-1719-4a7e-a4ab-fa584809b422"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:53.611660219Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1892,"":"io task","client":"0","":"downstairs","session_id":"a0c92089-1719-4a7e-a4ab-fa584809b422"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:53.611694133Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1892,"":"io task","client":"1","":"downstairs","session_id":"a0c92089-1719-4a7e-a4ab-fa584809b422"} {"msg":"ds_connection connected from Ok(127.0.0.1:52826)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:53.611726457Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1892,"":"io task","client":"2","":"downstairs","session_id":"a0c92089-1719-4a7e-a4ab-fa584809b422"} {"msg":"ds_connection connected from Ok(127.0.0.1:34750)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:53.611758071Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1892,"":"io task","client":"1","":"downstairs","session_id":"a0c92089-1719-4a7e-a4ab-fa584809b422"} {"msg":"ds_connection connected from Ok(127.0.0.1:36169)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:53.611809788Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1892,"":"io task","client":"0","":"downstairs","session_id":"a0c92089-1719-4a7e-a4ab-fa584809b422"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:53.612584768Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1892,"client":"1","":"downstairs","session_id":"a0c92089-1719-4a7e-a4ab-fa584809b422"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:53.612612411Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1892,"client":"1","":"downstairs","session_id":"a0c92089-1719-4a7e-a4ab-fa584809b422"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:53.612626923Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1892,"client":"2","":"downstairs","session_id":"a0c92089-1719-4a7e-a4ab-fa584809b422"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:53.612640415Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1892,"client":"2","":"downstairs","session_id":"a0c92089-1719-4a7e-a4ab-fa584809b422"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:53.612653797Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1892,"client":"0","":"downstairs","session_id":"a0c92089-1719-4a7e-a4ab-fa584809b422"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:53.612674749Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1892,"client":"0","":"downstairs","session_id":"a0c92089-1719-4a7e-a4ab-fa584809b422"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:53.613234481Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1892,"client":"1","":"downstairs","session_id":"a0c92089-1719-4a7e-a4ab-fa584809b422"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:53.613359488Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1892,"client":"0","":"downstairs","session_id":"a0c92089-1719-4a7e-a4ab-fa584809b422"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:53.613399163Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1892,"client":"2","":"downstairs","session_id":"a0c92089-1719-4a7e-a4ab-fa584809b422"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:53.613817897Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1892,"client":"1","":"downstairs","session_id":"a0c92089-1719-4a7e-a4ab-fa584809b422"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:53.61384639Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1892,"client":"1","":"downstairs","session_id":"a0c92089-1719-4a7e-a4ab-fa584809b422"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:53.613861962Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1892,"session_id":"a0c92089-1719-4a7e-a4ab-fa584809b422"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:53.613950744Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1892,"client":"0","":"downstairs","session_id":"a0c92089-1719-4a7e-a4ab-fa584809b422"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:53.613977747Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1892,"client":"0","":"downstairs","session_id":"a0c92089-1719-4a7e-a4ab-fa584809b422"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:53.613993959Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1892,"session_id":"a0c92089-1719-4a7e-a4ab-fa584809b422"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:53.614006801Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1892,"client":"2","":"downstairs","session_id":"a0c92089-1719-4a7e-a4ab-fa584809b422"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:53.614020603Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1892,"client":"2","":"downstairs","session_id":"a0c92089-1719-4a7e-a4ab-fa584809b422"} {"msg":"[0]R flush_numbers[0..12]: [289, 300, 298, 293, 299, 299, 284, 298, 300, 300, 286, 286]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:53.614034404Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1892,"":"downstairs","":"downstairs","session_id":"a0c92089-1719-4a7e-a4ab-fa584809b422"} {"msg":"[0]R generation[0..12]: [170, 177, 176, 173, 176, 176, 167, 176, 177, 177, 169, 169]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:53.614048446Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1892,"":"downstairs","":"downstairs","session_id":"a0c92089-1719-4a7e-a4ab-fa584809b422"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:53.614062658Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1892,"":"downstairs","":"downstairs","session_id":"a0c92089-1719-4a7e-a4ab-fa584809b422"} {"msg":"[1]R flush_numbers[0..12]: [289, 300, 298, 293, 299, 299, 284, 298, 300, 300, 286, 286]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:53.61407651Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1892,"":"downstairs","":"downstairs","session_id":"a0c92089-1719-4a7e-a4ab-fa584809b422"} {"msg":"[1]R generation[0..12]: [170, 177, 176, 173, 176, 176, 167, 176, 177, 177, 169, 169]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:53.614090572Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1892,"":"downstairs","":"downstairs","session_id":"a0c92089-1719-4a7e-a4ab-fa584809b422"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:53.614104684Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1892,"":"downstairs","":"downstairs","session_id":"a0c92089-1719-4a7e-a4ab-fa584809b422"} {"msg":"[2]R flush_numbers[0..12]: [289, 300, 298, 293, 299, 299, 284, 298, 300, 300, 286, 286]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:53.614118535Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1892,"":"downstairs","":"downstairs","session_id":"a0c92089-1719-4a7e-a4ab-fa584809b422"} {"msg":"[2]R generation[0..12]: [170, 177, 176, 173, 176, 176, 167, 176, 177, 177, 169, 169]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:53.614135588Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1892,"":"downstairs","":"downstairs","session_id":"a0c92089-1719-4a7e-a4ab-fa584809b422"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:53.614150099Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1892,"":"downstairs","":"downstairs","session_id":"a0c92089-1719-4a7e-a4ab-fa584809b422"} {"msg":"Max found gen is 178","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:53.614163511Z","Wait for a query_work_queue command to finish before sending IO hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1892,"":"downstairs","":"downstairs","session_id":"a0c92089-1719-4a7e-a4ab-fa584809b422"} {"msg":"Generation requested: 178 >= found:178","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:53.614186814Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1892,"":"downstairs","":"downstairs","session_id":"a0c92089-1719-4a7e-a4ab-fa584809b422"} {"msg":"Next flush: 301","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:53.614200026Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1892,"":"downstairs","":"downstairs","session_id":"a0c92089-1719-4a7e-a4ab-fa584809b422"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:53.614213118Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1892,"":"downstairs","":"downstairs","session_id":"a0c92089-1719-4a7e-a4ab-fa584809b422"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:53.614226369Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1892,"session_id":"a0c92089-1719-4a7e-a4ab-fa584809b422"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:53.614238471Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1892,"client":"0","":"downstairs","session_id":"a0c92089-1719-4a7e-a4ab-fa584809b422"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:53.614251692Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1892,"client":"1","":"downstairs","session_id":"a0c92089-1719-4a7e-a4ab-fa584809b422"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:53.614264704Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1892,"client":"2","":"downstairs","session_id":"a0c92089-1719-4a7e-a4ab-fa584809b422"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:53.614277886Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1892,"session_id":"a0c92089-1719-4a7e-a4ab-fa584809b422"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:53.614290168Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1892,"session_id":"a0c92089-1719-4a7e-a4ab-fa584809b422"} {"msg":"6c8b1539-0fb5-4912-b0df-f4a9f6901cc8 is now active with session: a0c92089-1719-4a7e-a4ab-fa584809b422","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:53.614302819Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1892,"session_id":"a0c92089-1719-4a7e-a4ab-fa584809b422"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:53.614314971Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1892,"session_id":"a0c92089-1719-4a7e-a4ab-fa584809b422"} {"msg":"The guest has finished waiting for activation with:178","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:53.614327122Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1892} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 290 len 32768 data: 12 15 17 17 18 15 15 14 02/30 Write block 90 len 24576 data: 22 20 18 18 15 18 03/30 Write block 171 len 36864 data: 10 9 9 10 12 12 13 14 13 04/30 Write block 515 len 28672 data: 8 9 11 11 14 13 12 05/30 Read block 278 len 32768 06/30 Write block 423 len 8192 data: 16 17 07/30 Write block 221 len 8192 data: 15 15 08/30 Write block 274 len 36864 data: 24 22 21 16 12 12 12 11 12 09/30 Read block 594 len 16384 10/30 Write block 172 len 12288 data: 10 10 11 11/30 Read block 77 len 24576 12/30 Read block 294 len 40960 13/30 Write block 543 len 16384 data: 9 8 9 9 14/30 Write block 519 len 36864 data: 15 14 13 13 12 12 14 13 15 15/30 Read block 92 len 40960 16/30 Write block 44 len 28672 data: 18 16 17 13 11 14 14 17/30 Read block 424 len 12288 18/30 Flush 19/30 Write block 41 len 36864 data: 15 17 19 19 17 18 14 12 15 20/30 Write block 472 len 36864 data: 19 20 17 16 13 18 22 19 17 21/30 Read block 245 len 40960 22/30 Read block 416 len 28672 23/30 Write block 189 len 32768 data: 14 16 16 18 20 22 21 18 24/30 Read block 463 len 12288 25/30 Flush 26/30 Write block 571 len 4096 data: 11 27/30 Write block 18 len 20480 data: 15 15 15 10 12 28/30 Read block 428 len 16384 29/30 Read block 122 len 28672 30/30 Read block 216 len 8192 ---------------------------------------------------------------- Crucible gen:178 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 1 Done Done Done false 1026 Acked Write 5 Done Done Done false 1027 Acked Read 4 Done Done Done false 1028 Acked Read 7 Done Done Done false 1029 Acked Read 2 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:56:53.639867903Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1892} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 170 170 170 289 289 289 T T T 1 020-039 177 177 177 300 300 300 T T T 2 040-059 178 178 178 302 302 302 F F F 3 060-079 173 173 173 293 293 293 F F F 4 080-099 178 178 178 301 301 301 F F F 5 100-119 176 176 176 299 299 299 F F F 6 120-139 167 167 167 284 284 284 F F F 7 140-159 176 176 176 298 298 298 F F F 8 160-179 178 178 178 301 301 301 F F F 9 180-199 178 178 178 302 302 302 F F F 10 200-219 169 169 169 286 286 286 F F F 11 220-239 178 178 178 301 301 301 F F F 12 240-259 169 169 169 286 286 286 F F F 13 260-279 178 178 178 301 301 301 F F F 14 280-299 178 178 178 301 301 301 F F F 15 300-319 174 174 174 294 294 294 F F F 16 320-339 177 177 177 300 300 300 F F F 17 340-359 166 166 166 282 282 282 F F F 18 360-379 170 170 170 288 288 288 F F F 19 380-399 170 170 170 288 288 288 F F F 20 400-419 175 175 175 297 297 297 F F F 21 420-439 178 178 178 301 301 301 F F F 22 440-459 176 176 176 298 298 298 F F F 23 460-479 178 178 178 302 302 302 F F F 24 480-499 178 178 178 302 302 302 F F F 25 500-519 178 178 178 301 301 301 F F F 26 520-539 178 178 178 301 301 301 F F F 27 540-559 178 178 178 301 301 301 F F F 28 560-579 170 170 170 289 289 289 T T T 29 580-599 176 176 176 298 298 298 F F F Max gen: 178, Max flush: 302 A Difference in extent metadata is expected here On loop 89 Verifying data now No change after verify Loop: 89 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 179 179 179 303 303 303 F F F 1 020-039 179 179 179 303 303 303 F F F 2 040-059 178 178 178 302 302 302 F F F 3 060-079 173 173 173 293 293 293 F F F 4 080-099 178 178 178 301 301 301 F F F 5 100-119 176 176 176 299 299 299 F F F 6 120-139 167 167 167 284 284 284 F F F 7 140-159 176 176 176 298 298 298 F F F 8 160-179 178 178 178 301 301 301 F F F 9 180-199 178 178 178 302 302 302 F F F 10 200-219 169 169 169 286 286 286 F F F 11 220-239 178 178 178 301 301 301 F F F 12 240-259 169 169 169 286 286 286 F F F 13 260-279 178 178 178 301 301 301 F F F 14 280-299 178 178 178 301 301 301 F F F 15 300-319 174 174 174 294 294 294 F F F 16 320-339 177 177 177 300 300 300 F F F 17 340-359 166 166 166 282 282 282 F F F 18 360-379 170 170 170 288 288 288 F F F 19 380-399 170 170 170 288 288 288 F F F 20 400-419 175 175 175 297 297 297 F F F 21 420-439 178 178 178 301 301 301 F F F 22 440-459 176 176 176 298 298 298 F F F 23 460-479 178 178 178 302 302 302 F F F 24 480-499 178 178 178 302 302 302 F F F 25 500-519 178 178 178 301 301 301 F F F 26 520-539 178 178 178 301 301 301 F F F 27 540-559 178 178 178 301 301 301 F F F 28 560-579 179 179 179 303 303 303 F F F 29 580-599 176 176 176 298 298 298 F F F Max gen: 179, Max flush: 303 Begin loop 90 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:57.953966183Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1900,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:56:57.99631471Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1900,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:57.996687038Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1900,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:58.028300581Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1900,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:58.028885846Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1900} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"3818f0fc76fe787c6cc412af30fd59b930203885\",\n git_commit_timestamp: \"2024-11-01T16:08:30.000000000Z\",\n git_branch: \"renovate/hyper-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:58.02891691Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1900} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:58.028931052Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1900} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:58.028950005Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1900} {"msg":"Crucible 2d3f136d-be2c-4860-a7fa-4f3d047a7e78 has session id: 58de8d8b-23c2-4579-9d9b-72204232d63a","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:58.029384681Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1900,"session_id":"58de8d8b-23c2-4579-9d9b-72204232d63a"} {"msg":"Upstairs opts: Upstairs UUID: 2d3f136d-be2c-4860-a7fa-4f3d047a7e78, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:58.029426666Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1900,"session_id":"58de8d8b-23c2-4579-9d9b-72204232d63a"} {"msg":"Crucible stats registered with UUID: 2d3f136d-be2c-4860-a7fa-4f3d047a7e78","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:58.029452039Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1900,"session_id":"58de8d8b-23c2-4579-9d9b-72204232d63a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:58.0601939Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1900,"task":"crutest"} {"msg":"The guest has requested activation with gen:180","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:58.060228455Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1900} {"msg":"2d3f136d-be2c-4860-a7fa-4f3d047a7e78 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:58.060253568Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1900,"session_id":"58de8d8b-23c2-4579-9d9b-72204232d63a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:58.06027147Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1900,"client":"0","":"downstairs","session_id":"58de8d8b-23c2-4579-9d9b-72204232d63a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:58.060299604Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1900,"client":"0","":"downstairs","session_id":"58de8d8b-23c2-4579-9d9b-72204232d63a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:58.060317367Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1900,"client":"1","":"downstairs","session_id":"58de8d8b-23c2-4579-9d9b-72204232d63a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:58.060334619Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1900,"client":"1","":"downstairs","session_id":"58de8d8b-23c2-4579-9d9b-72204232d63a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:58.060351861Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1900,"client":"2","":"downstairs","session_id":"58de8d8b-23c2-4579-9d9b-72204232d63a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:58.060368763Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1900,"client":"2","":"downstairs","session_id":"58de8d8b-23c2-4579-9d9b-72204232d63a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:58.060385375Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1900,"":"io task","client":"2","":"downstairs","session_id":"58de8d8b-23c2-4579-9d9b-72204232d63a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:58.060402537Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1900,"":"io task","client":"0","":"downstairs","session_id":"58de8d8b-23c2-4579-9d9b-72204232d63a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:58.0604242Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1900,"":"io task","client":"1","":"downstairs","session_id":"58de8d8b-23c2-4579-9d9b-72204232d63a"} {"msg":"ds_connection connected from Ok(127.0.0.1:53529)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:58.060639448Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1900,"":"io task","client":"2","":"downstairs","session_id":"58de8d8b-23c2-4579-9d9b-72204232d63a"} {"msg":"ds_connection connected from Ok(127.0.0.1:39791)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:58.060684284Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1900,"":"io task","client":"1","":"downstairs","session_id":"58de8d8b-23c2-4579-9d9b-72204232d63a"} {"msg":"ds_connection connected from Ok(127.0.0.1:46947)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:58.060713298Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1900,"":"io task","client":"0","":"downstairs","session_id":"58de8d8b-23c2-4579-9d9b-72204232d63a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:58.061247736Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1900,"client":"1","":"downstairs","session_id":"58de8d8b-23c2-4579-9d9b-72204232d63a"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:58.06127567Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1900,"client":"1","":"downstairs","session_id":"58de8d8b-23c2-4579-9d9b-72204232d63a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:58.061291552Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1900,"client":"2","":"downstairs","session_id":"58de8d8b-23c2-4579-9d9b-72204232d63a"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:58.061312885Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1900,"client":"2","":"downstairs","session_id":"58de8d8b-23c2-4579-9d9b-72204232d63a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:58.061339668Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1900,"client":"0","":"downstairs","session_id":"58de8d8b-23c2-4579-9d9b-72204232d63a"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:58.061367492Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1900,"client":"0","":"downstairs","session_id":"58de8d8b-23c2-4579-9d9b-72204232d63a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:58.061866526Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1900,"client":"2","":"downstairs","session_id":"58de8d8b-23c2-4579-9d9b-72204232d63a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:58.062001133Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1900,"client":"1","":"downstairs","session_id":"58de8d8b-23c2-4579-9d9b-72204232d63a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:58.062025967Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1900,"client":"0","":"downstairs","session_id":"58de8d8b-23c2-4579-9d9b-72204232d63a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:58.062407076Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1900,"client":"2","":"downstairs","session_id":"58de8d8b-23c2-4579-9d9b-72204232d63a"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:58.06243608Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1900,"client":"2","":"downstairs","session_id":"58de8d8b-23c2-4579-9d9b-72204232d63a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:58.062456132Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1900,"session_id":"58de8d8b-23c2-4579-9d9b-72204232d63a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:58.062537173Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1900,"client":"1","":"downstairs","session_id":"58de8d8b-23c2-4579-9d9b-72204232d63a"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:58.062557475Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1900,"client":"1","":"downstairs","session_id":"58de8d8b-23c2-4579-9d9b-72204232d63a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:58.062572057Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1900,"session_id":"58de8d8b-23c2-4579-9d9b-72204232d63a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:58.062650987Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1900,"client":"0","":"downstairs","session_id":"58de8d8b-23c2-4579-9d9b-72204232d63a"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:58.06267056Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1900,"client":"0","":"downstairs","session_id":"58de8d8b-23c2-4579-9d9b-72204232d63a"} {"msg":"[0]R flush_numbers[0..12]: [303, 303, 302, 293, 301, 299, 284, 298, 301, 302, 286, 301]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:58.062685701Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1900,"":"downstairs","":"downstairs","session_id":"58de8d8b-23c2-4579-9d9b-72204232d63a"} {"msg":"[0]R generation[0..12]: [179, 179, 178, 173, 178, 176, 167, 176, 178, 178, 169, 178]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:58.062699963Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1900,"":"downstairs","":"downstairs","session_id":"58de8d8b-23c2-4579-9d9b-72204232d63a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:58.062715125Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1900,"":"downstairs","":"downstairs","session_id":"58de8d8b-23c2-4579-9d9b-72204232d63a"} {"msg":"[1]R flush_numbers[0..12]: [303, 303, 302, 293, 301, 299, 284, 298, 301, 302, 286, 301]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:58.062729057Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1900,"":"downstairs","":"downstairs","session_id":"58de8d8b-23c2-4579-9d9b-72204232d63a"} {"msg":"[1]R generation[0..12]: [179, 179, 178, 173, 178, 176, 167, 176, 178, 178, 169, 178]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:58.062743659Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1900,"":"downstairs","":"downstairs","session_id":"58de8d8b-23c2-4579-9d9b-72204232d63a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:58.062757721Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1900,"":"downstairs","":"downstairs","session_id":"58de8d8b-23c2-4579-9d9b-72204232d63a"} {"msg":"[2]R flush_numbers[0..12]: [303, 303, 302, 293, 301, 299, 284, 298, 301, 302, 286, 301]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:58.062771593Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1900,"":"downstairs","":"downstairs","session_id":"58de8d8b-23c2-4579-9d9b-72204232d63a"} {"msg":"[2]R generation[0..12]: [179, 179, 178, 173, 178, 176, 167, 176, 178, 178, 169, 178]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:58.062785525Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1900,"":"downstairs","":"downstairs","session_id":"58de8d8b-23c2-4579-9d9b-72204232d63a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:58.062802067Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1900,"":"downstairs","":"downstairs","session_id":"58de8d8b-23c2-4579-9d9b-72204232d63a"} {"msg":"Max found gen is 180","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:58.062815548Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1900,"":"downstairs","":"downstairs","session_id":"58de8d8b-23c2-4579-9d9b-72204232d63a"} {"msg":"Generation requested: 180 >= found:180","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:58.06282896Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1900,"":"downstairs","":"downstairs","session_id":"58de8d8b-23c2-4579-9d9b-72204232d63a"} {"msg":"Next flush: 304","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:58.062842252Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1900,"":"downstairs","":"downstairs","session_id":"58de8d8b-23c2-4579-9d9b-72204232d63a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:58.062855563Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1900,"":"downstairs","":"downstairs","session_id":"58de8d8b-23c2-4579-9d9b-72204232d63a"} Wait for a query_work_queue command to finish before sending IO {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:58.062876046Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1900,"session_id":"58de8d8b-23c2-4579-9d9b-72204232d63a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:58.062888578Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1900,"client":"0","":"downstairs","session_id":"58de8d8b-23c2-4579-9d9b-72204232d63a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:58.062901849Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1900,"client":"1","":"downstairs","session_id":"58de8d8b-23c2-4579-9d9b-72204232d63a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:58.062915051Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1900,"client":"2","":"downstairs","session_id":"58de8d8b-23c2-4579-9d9b-72204232d63a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:58.062928233Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1900,"session_id":"58de8d8b-23c2-4579-9d9b-72204232d63a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:58.062940565Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1900,"session_id":"58de8d8b-23c2-4579-9d9b-72204232d63a"} {"msg":"2d3f136d-be2c-4860-a7fa-4f3d047a7e78 is now active with session: 58de8d8b-23c2-4579-9d9b-72204232d63a","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:58.062953356Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1900,"session_id":"58de8d8b-23c2-4579-9d9b-72204232d63a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:58.062965328Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1900,"session_id":"58de8d8b-23c2-4579-9d9b-72204232d63a"} {"msg":"The guest has finished waiting for activation with:180","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:56:58.062978529Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1900} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 140 len 40960 data: 13 14 12 9 11 9 8 11 12 17 02/30 Write block 277 len 4096 data: 17 03/30 Write block 13 len 20480 data: 16 17 17 17 17 04/30 Flush 05/30 Write block 543 len 4096 data: 10 06/30 Write block 577 len 32768 data: 12 14 14 10 11 10 12 15 07/30 Flush 08/30 Write block 435 len 4096 data: 7 09/30 Flush 10/30 Write block 169 len 24576 data: 13 13 11 11 11 12 11/30 Write block 143 len 32768 data: 10 12 10 9 12 13 18 20 12/30 Flush 13/30 Write block 353 len 24576 data: 15 11 11 12 17 16 14/30 Write block 498 len 4096 data: 17 15/30 Write block 202 len 16384 data: 15 15 14 14 16/30 Read block 458 len 32768 17/30 Write block 116 len 8192 data: 11 12 18/30 Write block 227 len 32768 data: 12 9 13 11 10 9 8 11 19/30 Read block 516 len 40960 20/30 Write block 157 len 16384 data: 15 15 14 15 21/30 Write block 116 len 36864 data: 12 13 11 13 13 17 19 15 13 22/30 Write block 239 len 28672 data: 10 10 12 11 11 12 17 23/30 Write block 293 len 8192 data: 18 19 24/30 Read block 74 len 16384 25/30 Read block 391 len 36864 26/30 Write block 560 len 20480 data: 21 20 20 22 20 27/30 Read block 532 len 40960 28/30 Read block 464 len 28672 29/30 Read block 268 len 32768 30/30 Write block 436 len 36864 data: 8 8 11 10 14 11 10 10 12 ---------------------------------------------------------------- Crucible gen:180 GIO:true work queues: Upstairs:0 downstairs:18 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1012 Acked Write 6 Done Done Done false 1013 Acked Write 1 Done Done Done false 1014 Acked Write 4 Done Done Done false 1015 Acked Read 8 Done Done Done false 1016 Acked Write 2 Done Done Done false 1017 Acked Write 8 Done Done Done false 1018 Acked Read 10 Done Done Done false 1019 Acked Write 4 Done Done Done false 1020 Acked Write 9 Done Done Done false 1021 Acked Write 7 Done Done Done false 1022 Acked Write 2 Done Done Done false 1023 Acked Read 4 Done Done Done false 1024 Acked Read 9 Done Done Done false 1025 Acked Write 5 Done Done Done false 1026 Acked Read 10 Done Done Done false 1027 Acked Read 7 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 17 17 17 51 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1011 1011 1011 Downstairs last five completed: 1011 1010 1009 1008 1007 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:56:58.089328925Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1900} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 180 180 180 304 304 304 F F F 1 020-039 179 179 179 303 303 303 F F F 2 040-059 178 178 178 302 302 302 F F F 3 060-079 173 173 173 293 293 293 F F F 4 080-099 178 178 178 301 301 301 F F F 5 100-119 176 176 176 299 299 299 T T T 6 120-139 167 167 167 284 284 284 T T T 7 140-159 180 180 180 307 307 307 T T T 8 160-179 180 180 180 307 307 307 T T T 9 180-199 178 178 178 302 302 302 F F F 10 200-219 169 169 169 286 286 286 T T T 11 220-239 178 178 178 301 301 301 T T T 12 240-259 169 169 169 286 286 286 T T T 13 260-279 180 180 180 304 304 304 F F F 14 280-299 178 178 178 301 301 301 T T T 15 300-319 174 174 174 294 294 294 F F F 16 320-339 177 177 177 300 300 300 F F F 17 340-359 166 166 166 282 282 282 T T T 18 360-379 170 170 170 288 288 288 F F F 19 380-399 170 170 170 288 288 288 F F F 20 400-419 175 175 175 297 297 297 F F F 21 420-439 180 180 180 306 306 306 T T T 22 440-459 176 176 176 298 298 298 T T T 23 460-479 178 178 178 302 302 302 F F F 24 480-499 178 178 178 302 302 302 T T T 25 500-519 178 178 178 301 301 301 F F F 26 520-539 178 178 178 301 301 301 F F F 27 540-559 180 180 180 305 305 305 F F F 28 560-579 180 180 180 305 305 305 T T T 29 580-599 180 180 180 305 305 305 F F F Max gen: 180, Max flush: 307 A Difference in extent metadata is expected here On loop 90 Verifying data now No change after verify Loop: 90 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 180 180 180 304 304 304 F F F 1 020-039 179 179 179 303 303 303 F F F 2 040-059 178 178 178 302 302 302 F F F 3 060-079 173 173 173 293 293 293 F F F 4 080-099 178 178 178 301 301 301 F F F 5 100-119 181 181 181 308 308 308 F F F 6 120-139 181 181 181 308 308 308 F F F 7 140-159 181 181 181 308 308 308 F F F 8 160-179 181 181 181 308 308 308 F F F 9 180-199 178 178 178 302 302 302 F F F 10 200-219 181 181 181 308 308 308 F F F 11 220-239 181 181 181 308 308 308 F F F 12 240-259 181 181 181 308 308 308 F F F 13 260-279 180 180 180 304 304 304 F F F 14 280-299 181 181 181 308 308 308 F F F 15 300-319 174 174 174 294 294 294 F F F 16 320-339 177 177 177 300 300 300 F F F 17 340-359 181 181 181 308 308 308 F F F 18 360-379 170 170 170 288 288 288 F F F 19 380-399 170 170 170 288 288 288 F F F 20 400-419 175 175 175 297 297 297 F F F 21 420-439 181 181 181 308 308 308 F F F 22 440-459 181 181 181 308 308 308 F F F 23 460-479 178 178 178 302 302 302 F F F 24 480-499 181 181 181 308 308 308 F F F 25 500-519 178 178 178 301 301 301 F F F 26 520-539 178 178 178 301 301 301 F F F 27 540-559 180 180 180 305 305 305 F F F 28 560-579 181 181 181 308 308 308 F F F 29 580-599 180 180 180 305 305 305 F F F Max gen: 181, Max flush: 308 Begin loop 91 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:02.533113816Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1908,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:57:02.577563573Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1908,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:02.577946542Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1908,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:02.610258196Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1908,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:02.610766581Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1908} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"3818f0fc76fe787c6cc412af30fd59b930203885\",\n git_commit_timestamp: \"2024-11-01T16:08:30.000000000Z\",\n git_branch: \"renovate/hyper-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:02.610801706Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1908} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:02.610819998Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1908} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:02.61083483Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1908} {"msg":"Crucible 20a95240-4be6-4db7-b505-5508780db235 has session id: 7310ec40-ddc4-498a-9402-fd831fc82fe2","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:02.611267786Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1908,"session_id":"7310ec40-ddc4-498a-9402-fd831fc82fe2"} {"msg":"Upstairs opts: Upstairs UUID: 20a95240-4be6-4db7-b505-5508780db235, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:02.611309311Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1908,"session_id":"7310ec40-ddc4-498a-9402-fd831fc82fe2"} {"msg":"Crucible stats registered with UUID: 20a95240-4be6-4db7-b505-5508780db235","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:02.611333454Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1908,"session_id":"7310ec40-ddc4-498a-9402-fd831fc82fe2"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:02.641919285Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1908,"task":"crutest"} {"msg":"The guest has requested activation with gen:182","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:02.641951989Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1908} {"msg":"20a95240-4be6-4db7-b505-5508780db235 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:02.642010327Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1908,"session_id":"7310ec40-ddc4-498a-9402-fd831fc82fe2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:02.64203829Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1908,"client":"0","":"downstairs","session_id":"7310ec40-ddc4-498a-9402-fd831fc82fe2"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:02.642063723Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1908,"client":"0","":"downstairs","session_id":"7310ec40-ddc4-498a-9402-fd831fc82fe2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:02.642078505Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1908,"client":"1","":"downstairs","session_id":"7310ec40-ddc4-498a-9402-fd831fc82fe2"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:02.642092397Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1908,"client":"1","":"downstairs","session_id":"7310ec40-ddc4-498a-9402-fd831fc82fe2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:02.642105889Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1908,"client":"2","":"downstairs","session_id":"7310ec40-ddc4-498a-9402-fd831fc82fe2"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:02.642126852Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1908,"client":"2","":"downstairs","session_id":"7310ec40-ddc4-498a-9402-fd831fc82fe2"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:02.642150035Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1908,"":"io task","client":"2","":"downstairs","session_id":"7310ec40-ddc4-498a-9402-fd831fc82fe2"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:02.642179449Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1908,"":"io task","client":"0","":"downstairs","session_id":"7310ec40-ddc4-498a-9402-fd831fc82fe2"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:02.642204882Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1908,"":"io task","client":"1","":"downstairs","session_id":"7310ec40-ddc4-498a-9402-fd831fc82fe2"} {"msg":"ds_connection connected from Ok(127.0.0.1:58482)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:02.642257708Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1908,"":"io task","client":"2","":"downstairs","session_id":"7310ec40-ddc4-498a-9402-fd831fc82fe2"} {"msg":"ds_connection connected from Ok(127.0.0.1:39602)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:02.642287732Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1908,"":"io task","client":"0","":"downstairs","session_id":"7310ec40-ddc4-498a-9402-fd831fc82fe2"} {"msg":"ds_connection connected from Ok(127.0.0.1:37437)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:02.642314036Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1908,"":"io task","client":"1","":"downstairs","session_id":"7310ec40-ddc4-498a-9402-fd831fc82fe2"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:02.643071904Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1908,"client":"2","":"downstairs","session_id":"7310ec40-ddc4-498a-9402-fd831fc82fe2"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:02.643100787Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1908,"client":"2","":"downstairs","session_id":"7310ec40-ddc4-498a-9402-fd831fc82fe2"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:02.643116459Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1908,"client":"1","":"downstairs","session_id":"7310ec40-ddc4-498a-9402-fd831fc82fe2"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:02.643130411Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1908,"client":"1","":"downstairs","session_id":"7310ec40-ddc4-498a-9402-fd831fc82fe2"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:02.643146763Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1908,"client":"0","":"downstairs","session_id":"7310ec40-ddc4-498a-9402-fd831fc82fe2"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:02.643160315Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1908,"client":"0","":"downstairs","session_id":"7310ec40-ddc4-498a-9402-fd831fc82fe2"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:02.643714036Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1908,"client":"1","":"downstairs","session_id":"7310ec40-ddc4-498a-9402-fd831fc82fe2"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:02.6437442Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1908,"client":"2","":"downstairs","session_id":"7310ec40-ddc4-498a-9402-fd831fc82fe2"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:02.643825681Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1908,"client":"0","":"downstairs","session_id":"7310ec40-ddc4-498a-9402-fd831fc82fe2"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:02.644313173Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1908,"client":"1","":"downstairs","session_id":"7310ec40-ddc4-498a-9402-fd831fc82fe2"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:02.644344287Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1908,"client":"1","":"downstairs","session_id":"7310ec40-ddc4-498a-9402-fd831fc82fe2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:02.64436023Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1908,"session_id":"7310ec40-ddc4-498a-9402-fd831fc82fe2"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:02.644373001Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1908,"client":"0","":"downstairs","session_id":"7310ec40-ddc4-498a-9402-fd831fc82fe2"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:02.644386863Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1908,"client":"0","":"downstairs","session_id":"7310ec40-ddc4-498a-9402-fd831fc82fe2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:02.644400535Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1908,"session_id":"7310ec40-ddc4-498a-9402-fd831fc82fe2"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:02.644412946Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1908,"client":"2","":"downstairs","session_id":"7310ec40-ddc4-498a-9402-fd831fc82fe2"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:02.644426168Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1908,"client":"2","":"downstairs","session_id":"7310ec40-ddc4-498a-9402-fd831fc82fe2"} {"msg":"[0]R flush_numbers[0..12]: [304, 303, 302, 293, 301, 308, 308, 308, 308, 302, 308, 308]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:02.64444008Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1908,"":"downstairs","":"downstairs","session_id":"7310ec40-ddc4-498a-9402-fd831fc82fe2"} {"msg":"[0]R generation[0..12]: [180, 179, 178, 173, 178, 181, 181, 181, 181, 178, 181, 181]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:02.644454211Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1908,"":"downstairs","":"downstairs","session_id":"7310ec40-ddc4-498a-9402-fd831fc82fe2"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:02.644468473Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1908,"":"downstairs","":"downstairs","session_id":"7310ec40-ddc4-498a-9402-fd831fc82fe2"} {"msg":"[1]R flush_numbers[0..12]: [304, 303, 302, 293, 301, 308, 308, 308, 308, 302, 308, 308]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:02.644482385Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1908,"":"downstairs","":"downstairs","session_id":"7310ec40-ddc4-498a-9402-fd831fc82fe2"} {"msg":"[1]R generation[0..12]: [180, 179, 178, 173, 178, 181, 181, 181, 181, 178, 181, 181]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:02.644496217Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1908,"":"downstairs","":"downstairs","session_id":"7310ec40-ddc4-498a-9402-fd831fc82fe2"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:02.644510119Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1908,"":"downstairs","":"downstairs","session_id":"7310ec40-ddc4-498a-9402-fd831fc82fe2"} {"msg":"[2]R flush_numbers[0..12]: [304, 303, 302, 293, 301, 308, 308, 308, 308, 302, 308, 308]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:02.6445238Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1908,"":"downstairs","":"downstairs","session_id":"7310ec40-ddc4-498a-9402-fd831fc82fe2"} {"msg":"[2]R generation[0..12]: [180, 179, 178, 173, 178, 181, 181, 181, 181, 178, 181, 181]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:02.644539673Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1908,"":"downstairs","":"downstairs","session_id":"7310ec40-ddc4-498a-9402-fd831fc82fe2"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:02.644554094Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1908,"":"downstairs","":"downstairs","session_id":"7310ec40-ddc4-498a-9402-fd831fc82fe2"} {"msg":"Max found gen is 182","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:02.644567446Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1908,"":"downstairs","":"downstairs","session_id":"7310ec40-ddc4-498a-9402-fd831fc82fe2"} {"msg":"Generation requested: 182 >= found:182","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:02.644580958Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1908,"":"downstairs","":"downstairs","session_id":"7310ec40-ddc4-498a-9402-fd831fc82fe2"} {Wait for a query_work_queue command to finish before sending IO "msg":"Next flush: 309","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:02.64460191Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1908,"":"downstairs","":"downstairs","session_id":"7310ec40-ddc4-498a-9402-fd831fc82fe2"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:02.644615382Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1908,"":"downstairs","":"downstairs","session_id":"7310ec40-ddc4-498a-9402-fd831fc82fe2"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:02.644628654Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1908,"session_id":"7310ec40-ddc4-498a-9402-fd831fc82fe2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:02.644640806Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1908,"client":"0","":"downstairs","session_id":"7310ec40-ddc4-498a-9402-fd831fc82fe2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:02.644653957Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1908,"client":"1","":"downstairs","session_id":"7310ec40-ddc4-498a-9402-fd831fc82fe2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:02.644667139Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1908,"client":"2","":"downstairs","session_id":"7310ec40-ddc4-498a-9402-fd831fc82fe2"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:02.644680371Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1908,"session_id":"7310ec40-ddc4-498a-9402-fd831fc82fe2"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:02.644692622Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1908,"session_id":"7310ec40-ddc4-498a-9402-fd831fc82fe2"} {"msg":"20a95240-4be6-4db7-b505-5508780db235 is now active with session: 7310ec40-ddc4-498a-9402-fd831fc82fe2","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:02.644705304Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1908,"session_id":"7310ec40-ddc4-498a-9402-fd831fc82fe2"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:02.644717436Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1908,"session_id":"7310ec40-ddc4-498a-9402-fd831fc82fe2"} {"msg":"The guest has finished waiting for activation with:182","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:02.644729757Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1908} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 349 len 40960 data: 13 12 14 15 16 12 12 13 18 17 02/30 Read block 223 len 4096 03/30 Flush 04/30 Write block 460 len 40960 data: 9 12 14 13 11 11 14 15 14 15 05/30 Read block 81 len 4096 06/30 Flush 07/30 Write block 456 len 16384 data: 11 10 12 11 08/30 Read block 6 len 8192 09/30 Read block 125 len 20480 10/30 Read block 252 len 20480 11/30 Read block 387 len 16384 12/30 Read block 414 len 12288 13/30 Write block 214 len 12288 data: 16 16 19 14/30 Write block 367 len 8192 data: 8 7 15/30 Read block 8 len 24576 16/30 Flush 17/30 Write block 510 len 16384 data: 13 12 12 9 18/30 Write block 346 len 40960 data: 12 13 12 14 13 15 16 17 13 13 19/30 Write block 38 len 24576 data: 18 17 15 16 18 20 20/30 Write block 501 len 36864 data: 14 11 8 9 9 11 11 16 13 21/30 Read block 275 len 24576 22/30 Write block 393 len 32768 data: 12 11 13 14 14 16 14 12 23/30 Write block 571 len 20480 data: 12 10 9 8 11 24/30 Flush 25/30 Write block 290 len 16384 data: 13 16 18 19 26/30 Write block 382 len 36864 data: 16 16 15 14 13 11 12 13 11 27/30 Read block 188 len 12288 28/30 Read block 431 len 36864 29/30 Write block 148 len 40960 data: 14 19 21 17 19 16 18 16 17 16 30/30 Read block 46 len 8192 ---------------------------------------------------------------- Crucible gen:182 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 4 Done Done Done false 1025 Acked Write 9 Done Done Done false 1026 Acked Read 3 Done Done Done false 1027 Acked Read 9 Done Done Done false 1028 Acked Write 10 Done Done Done false 1029 Acked Read 2 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 6 6 6 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:57:02.670927513Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1908} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 180 180 180 304 304 304 F F F 1 020-039 182 182 182 312 312 312 F F F 2 040-059 182 182 182 312 312 312 F F F 3 060-079 173 173 173 293 293 293 F F F 4 080-099 178 178 178 301 301 301 F F F 5 100-119 181 181 181 308 308 308 F F F 6 120-139 181 181 181 308 308 308 F F F 7 140-159 181 181 181 308 308 308 T T T 8 160-179 181 181 181 308 308 308 F F F 9 180-199 178 178 178 302 302 302 F F F 10 200-219 182 182 182 311 311 311 F F F 11 220-239 181 181 181 308 308 308 F F F 12 240-259 181 181 181 308 308 308 F F F 13 260-279 180 180 180 304 304 304 F F F 14 280-299 181 181 181 308 308 308 T T T 15 300-319 174 174 174 294 294 294 F F F 16 320-339 177 177 177 300 300 300 F F F 17 340-359 182 182 182 312 312 312 F F F 18 360-379 182 182 182 311 311 311 F F F 19 380-399 182 182 182 312 312 312 T T T 20 400-419 182 182 182 312 312 312 F F F 21 420-439 181 181 181 308 308 308 F F F 22 440-459 182 182 182 311 311 311 F F F 23 460-479 182 182 182 310 310 310 F F F 24 480-499 181 181 181 308 308 308 F F F 25 500-519 182 182 182 312 312 312 F F F 26 520-539 178 178 178 301 301 301 F F F 27 540-559 180 180 180 305 305 305 F F F 28 560-579 182 182 182 312 312 312 F F F 29 580-599 180 180 180 305 305 305 F F F Max gen: 182, Max flush: 312 A Difference in extent metadata is expected here On loop 91 Verifying data now No change after verify Loop: 91 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 180 180 180 304 304 304 F F F 1 020-039 182 182 182 312 312 312 F F F 2 040-059 182 182 182 312 312 312 F F F 3 060-079 173 173 173 293 293 293 F F F 4 080-099 178 178 178 301 301 301 F F F 5 100-119 181 181 181 308 308 308 F F F 6 120-139 181 181 181 308 308 308 F F F 7 140-159 183 183 183 313 313 313 F F F 8 160-179 181 181 181 308 308 308 F F F 9 180-199 178 178 178 302 302 302 F F F 10 200-219 182 182 182 311 311 311 F F F 11 220-239 181 181 181 308 308 308 F F F 12 240-259 181 181 181 308 308 308 F F F 13 260-279 180 180 180 304 304 304 F F F 14 280-299 183 183 183 313 313 313 F F F 15 300-319 174 174 174 294 294 294 F F F 16 320-339 177 177 177 300 300 300 F F F 17 340-359 182 182 182 312 312 312 F F F 18 360-379 182 182 182 311 311 311 F F F 19 380-399 183 183 183 313 313 313 F F F 20 400-419 182 182 182 312 312 312 F F F 21 420-439 181 181 181 308 308 308 F F F 22 440-459 182 182 182 311 311 311 F F F 23 460-479 182 182 182 310 310 310 F F F 24 480-499 181 181 181 308 308 308 F F F 25 500-519 182 182 182 312 312 312 F F F 26 520-539 178 178 178 301 301 301 F F F 27 540-559 180 180 180 305 305 305 F F F 28 560-579 182 182 182 312 312 312 F F F 29 580-599 180 180 180 305 305 305 F F F Max gen: 183, Max flush: 313 Begin loop 92 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:06.975845119Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1916,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:07.020375607Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1916,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:07.021054914Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1916} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"3818f0fc76fe787c6cc412af30fd59b930203885\",\n git_commit_timestamp: \"2024-11-01T16:08:30.000000000Z\",\n git_branch: \"renovate/hyper-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:07.021107741Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1916} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:07.021127474Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1916} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:07.021145646Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1916} {"msg":"Crucible 012b2e8d-2c2c-466d-99c2-a79308f717a9 has session id: 31ce5b39-a4e9-4c7d-9aa0-98b4325f77f9","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:07.021526205Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1916,"session_id":"31ce5b39-a4e9-4c7d-9aa0-98b4325f77f9"} {"msg":"Upstairs opts: Upstairs UUID: 012b2e8d-2c2c-466d-99c2-a79308f717a9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:07.021558789Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1916,"session_id":"31ce5b39-a4e9-4c7d-9aa0-98b4325f77f9"} {"msg":"Crucible stats registered with UUID: 012b2e8d-2c2c-466d-99c2-a79308f717a9","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:07.021581172Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1916,"session_id":"31ce5b39-a4e9-4c7d-9aa0-98b4325f77f9"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:07.052281678Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1916,"task":"crutest"} {"msg":"The guest has requested activation with gen:184","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:07.052313552Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1916} {"msg":"012b2e8d-2c2c-466d-99c2-a79308f717a9 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:07.052388582Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1916,"session_id":"31ce5b39-a4e9-4c7d-9aa0-98b4325f77f9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:07.052411675Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1916,"client":"0","":"downstairs","session_id":"31ce5b39-a4e9-4c7d-9aa0-98b4325f77f9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:07.052440558Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1916,"client":"0","":"downstairs","session_id":"31ce5b39-a4e9-4c7d-9aa0-98b4325f77f9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:07.05245482Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1916,"client":"1","":"downstairs","session_id":"31ce5b39-a4e9-4c7d-9aa0-98b4325f77f9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:07.052469012Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1916,"client":"1","":"downstairs","session_id":"31ce5b39-a4e9-4c7d-9aa0-98b4325f77f9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:07.052482354Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1916,"client":"2","":"downstairs","session_id":"31ce5b39-a4e9-4c7d-9aa0-98b4325f77f9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:07.052496015Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1916,"client":"2","":"downstairs","session_id":"31ce5b39-a4e9-4c7d-9aa0-98b4325f77f9"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:07.052509447Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1916,"":"io task","client":"2","":"downstairs","session_id":"31ce5b39-a4e9-4c7d-9aa0-98b4325f77f9"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:07.052523779Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1916,"":"io task","client":"0","":"downstairs","session_id":"31ce5b39-a4e9-4c7d-9aa0-98b4325f77f9"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:07.052547632Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1916,"":"io task","client":"1","":"downstairs","session_id":"31ce5b39-a4e9-4c7d-9aa0-98b4325f77f9"} {"msg":"ds_connection connected from Ok(127.0.0.1:38639)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:07.052822108Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1916,"":"io task","client":"2","":"downstairs","session_id":"31ce5b39-a4e9-4c7d-9aa0-98b4325f77f9"} {"msg":"ds_connection connected from Ok(127.0.0.1:51200)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:07.052879635Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1916,"":"io task","client":"0","":"downstairs","session_id":"31ce5b39-a4e9-4c7d-9aa0-98b4325f77f9"} {"msg":"ds_connection connected from Ok(127.0.0.1:47798)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:07.05291757Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1916,"":"io task","client":"1","":"downstairs","session_id":"31ce5b39-a4e9-4c7d-9aa0-98b4325f77f9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:07.053442608Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1916,"client":"2","":"downstairs","session_id":"31ce5b39-a4e9-4c7d-9aa0-98b4325f77f9"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:07.053474861Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1916,"client":"2","":"downstairs","session_id":"31ce5b39-a4e9-4c7d-9aa0-98b4325f77f9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:07.053512246Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1916,"client":"1","":"downstairs","session_id":"31ce5b39-a4e9-4c7d-9aa0-98b4325f77f9"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:07.053594567Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1916,"client":"1","":"downstairs","session_id":"31ce5b39-a4e9-4c7d-9aa0-98b4325f77f9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:07.053627731Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1916,"client":"0","":"downstairs","session_id":"31ce5b39-a4e9-4c7d-9aa0-98b4325f77f9"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:07.053644073Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1916,"client":"0","":"downstairs","session_id":"31ce5b39-a4e9-4c7d-9aa0-98b4325f77f9"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:07.054201145Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1916,"client":"1","":"downstairs","session_id":"31ce5b39-a4e9-4c7d-9aa0-98b4325f77f9"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:07.054232009Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1916,"client":"0","":"downstairs","session_id":"31ce5b39-a4e9-4c7d-9aa0-98b4325f77f9"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:07.054252022Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1916,"client":"2","":"downstairs","session_id":"31ce5b39-a4e9-4c7d-9aa0-98b4325f77f9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:07.054804353Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1916,"client":"1","":"downstairs","session_id":"31ce5b39-a4e9-4c7d-9aa0-98b4325f77f9"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:07.054833587Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1916,"client":"1","":"downstairs","session_id":"31ce5b39-a4e9-4c7d-9aa0-98b4325f77f9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:07.054849229Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1916,"session_id":"31ce5b39-a4e9-4c7d-9aa0-98b4325f77f9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:07.05486223Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1916,"client":"2","":"downstairs","session_id":"31ce5b39-a4e9-4c7d-9aa0-98b4325f77f9"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:07.054879093Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1916,"client":"2","":"downstairs","session_id":"31ce5b39-a4e9-4c7d-9aa0-98b4325f77f9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:07.054893024Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1916,"session_id":"31ce5b39-a4e9-4c7d-9aa0-98b4325f77f9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:07.054905396Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1916,"client":"0","":"downstairs","session_id":"31ce5b39-a4e9-4c7d-9aa0-98b4325f77f9"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:07.054919118Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1916,"client":"0","":"downstairs","session_id":"31ce5b39-a4e9-4c7d-9aa0-98b4325f77f9"} {"msg":"[0]R flush_numbers[0..12]: [304, 312, 312, 293, 301, 308, 308, 313, 308, 302, 311, 308]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:07.0549331Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1916,"":"downstairs","":"downstairs","session_id":"31ce5b39-a4e9-4c7d-9aa0-98b4325f77f9"} {"msg":"[0]R generation[0..12]: [180, 182, 182, 173, 178, 181, 181, 183, 181, 178, 182, 181]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:07.054947211Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1916,"":"downstairs","":"downstairs","session_id":"31ce5b39-a4e9-4c7d-9aa0-98b4325f77f9"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:07.054961323Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1916,"":"downstairs","":"downstairs","session_id":"31ce5b39-a4e9-4c7d-9aa0-98b4325f77f9"} {"msg":"[1]R flush_numbers[0..12]: [304, 312, 312, 293, 301, 308, 308, 313, 308, 302, 311, 308]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:07.054976445Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1916,"":"downstairs","":"downstairs","session_id":"31ce5b39-a4e9-4c7d-9aa0-98b4325f77f9"} {"msg":"[1]R generation[0..12]: [180, 182, 182, 173, 178, 181, 181, 183, 181, 178, 182, 181]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:07.054990667Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1916,"":"downstairs","":"downstairs","session_id":"31ce5b39-a4e9-4c7d-9aa0-98b4325f77f9"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:07.055004869Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1916,"":"downstairs","":"downstairs","session_id":"31ce5b39-a4e9-4c7d-9aa0-98b4325f77f9"} {"msg":"[2]R flush_numbers[0..12]: [304, 312, 312, 293, 301, 308, 308, 313, 308, 302, 311, 308]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:07.055018791Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1916,"":"downstairs","":"downstairs","session_id":"31ce5b39-a4e9-4c7d-9aa0-98b4325f77f9"} {"msg":"[2]R generation[0..12]: [180, 182, 182, 173, 178, 181, 181, 183, 181, 178, 182, 181]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:07.055032782Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1916,"":"downstairs","":"downstairs","session_id":"31ce5b39-a4e9-4c7d-9aa0-98b4325f77f9"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:07.055047014Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1916,"":"downstairs","":"downstairs","session_id":"31ce5b39-a4e9-4c7d-9aa0-98b4325f77f9"} {"msg":"Max found gen is 184","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:07.055060286Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1916,"":"downstairs","":"downstairs","session_id":"31ce5b39-a4e9-4c7d-9aa0-98b4325f77f9"} Wait for a query_work_queue command to finish before sending IO {"msg":"Generation requested: 184 >= found:184","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:07.055083439Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1916,"":"downstairs","":"downstairs","session_id":"31ce5b39-a4e9-4c7d-9aa0-98b4325f77f9"} {"msg":"Next flush: 314","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:07.055097201Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1916,"":"downstairs","":"downstairs","session_id":"31ce5b39-a4e9-4c7d-9aa0-98b4325f77f9"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:07.055110552Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1916,"":"downstairs","":"downstairs","session_id":"31ce5b39-a4e9-4c7d-9aa0-98b4325f77f9"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:07.055124384Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1916,"session_id":"31ce5b39-a4e9-4c7d-9aa0-98b4325f77f9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:07.055136736Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1916,"client":"0","":"downstairs","session_id":"31ce5b39-a4e9-4c7d-9aa0-98b4325f77f9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:07.055150428Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1916,"client":"1","":"downstairs","session_id":"31ce5b39-a4e9-4c7d-9aa0-98b4325f77f9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:07.055163949Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1916,"client":"2","":"downstairs","session_id":"31ce5b39-a4e9-4c7d-9aa0-98b4325f77f9"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:07.055177641Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1916,"session_id":"31ce5b39-a4e9-4c7d-9aa0-98b4325f77f9"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:07.055190213Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1916,"session_id":"31ce5b39-a4e9-4c7d-9aa0-98b4325f77f9"} {"msg":"012b2e8d-2c2c-466d-99c2-a79308f717a9 is now active with session: 31ce5b39-a4e9-4c7d-9aa0-98b4325f77f9","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:07.055203054Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1916,"session_id":"31ce5b39-a4e9-4c7d-9aa0-98b4325f77f9"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:07.055215206Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1916,"session_id":"31ce5b39-a4e9-4c7d-9aa0-98b4325f77f9"} {"msg":"The guest has finished waiting for activation with:184","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:07.055227627Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1916} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 430 len 16384 data: 12 13 8 8 02/30 Read block 190 len 12288 03/30 Read block 123 len 32768 04/30 Read block 460 len 16384 05/30 Write block 398 len 16384 data: 17 15 13 11 06/30 Read block 65 len 4096 07/30 Read block 555 len 20480 08/30 Read block 233 len 8192 09/30 Write block 34 len 4096 data: 11 10/30 Read block 325 len 20480 11/30 Read block 417 len 8192 12/30 Flush 13/30 Write block 23 len 24576 data: 16 19 19 18 18 20 14/30 Read block 564 len 4096 15/30 Read block 50 len 24576 16/30 Read block 239 len 24576 17/30 Read block 392 len 28672 18/30 Read block 579 len 28672 19/30 Read block 538 len 32768 20/30 Flush 21/30 Write block 545 len 36864 data: 10 10 9 8 9 7 13 14 14 22/30 Read block 497 len 4096 23/30 Read block 357 len 24576 24/30 Write block 252 len 36864 data: 19 17 16 13 12 13 14 10 11 25/30 Read block 449 len 8192 26/30 Write block 245 len 36864 data: 18 20 22 21 22 24 22 20 18 27/30 Read block 28 len 8192 28/30 Write block 133 len 20480 data: 18 14 12 9 10 29/30 Read block 118 len 36864 30/30 Read block 200 len 16384 ---------------------------------------------------------------- Crucible gen:184 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1020 Acked Write 9 Done Done Done false 1021 Acked Read 1 Done Done Done false 1022 Acked Read 6 Done Done Done false 1023 Acked Write 9 Done Done Done false 1024 Acked Read 2 Done Done Done false 1025 Acked Write 9 Done Done Done false 1026 Acked Read 2 Done Done Done false 1027 Acked Write 5 Done Done Done false 1028 Acked Read 9 Done Done Done false 1029 Acked Read 4 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 10 10 10 30 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1019 1019 1019 Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:57:07.07891772Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1916} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 180 180 180 304 304 304 F F F 1 020-039 184 184 184 315 315 315 F F F 2 040-059 182 182 182 312 312 312 F F F 3 060-079 173 173 173 293 293 293 F F F 4 080-099 178 178 178 301 301 301 F F F 5 100-119 181 181 181 308 308 308 F F F 6 120-139 181 181 181 308 308 308 T T T 7 140-159 183 183 183 313 313 313 F F F 8 160-179 181 181 181 308 308 308 F F F 9 180-199 178 178 178 302 302 302 F F F 10 200-219 182 182 182 311 311 311 F F F 11 220-239 181 181 181 308 308 308 F F F 12 240-259 181 181 181 308 308 308 T T T 13 260-279 180 180 180 304 304 304 T T T 14 280-299 183 183 183 313 313 313 F F F 15 300-319 174 174 174 294 294 294 F F F 16 320-339 177 177 177 300 300 300 F F F 17 340-359 182 182 182 312 312 312 F F F 18 360-379 182 182 182 311 311 311 F F F 19 380-399 184 184 184 314 314 314 F F F 20 400-419 184 184 184 314 314 314 F F F 21 420-439 184 184 184 314 314 314 F F F 22 440-459 182 182 182 311 311 311 F F F 23 460-479 182 182 182 310 310 310 F F F 24 480-499 181 181 181 308 308 308 F F F 25 500-519 182 182 182 312 312 312 F F F 26 520-539 178 178 178 301 301 301 F F F 27 540-559 180 180 180 305 305 305 T T T 28 560-579 182 182 182 312 312 312 F F F 29 580-599 180 180 180 305 305 305 F F F Max gen: 184, Max flush: 315 A Difference in extent metadata is expected here On loop 92 Verifying data now No change after verify Loop: 92 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 180 180 180 304 304 304 F F F 1 020-039 184 184 184 315 315 315 F F F 2 040-059 182 182 182 312 312 312 F F F 3 060-079 173 173 173 293 293 293 F F F 4 080-099 178 178 178 301 301 301 F F F 5 100-119 181 181 181 308 308 308 F F F 6 120-139 185 185 185 316 316 316 F F F 7 140-159 183 183 183 313 313 313 F F F 8 160-179 181 181 181 308 308 308 F F F 9 180-199 178 178 178 302 302 302 F F F 10 200-219 182 182 182 311 311 311 F F F 11 220-239 181 181 181 308 308 308 F F F 12 240-259 185 185 185 316 316 316 F F F 13 260-279 185 185 185 316 316 316 F F F 14 280-299 183 183 183 313 313 313 F F F 15 300-319 174 174 174 294 294 294 F F F 16 320-339 177 177 177 300 300 300 F F F 17 340-359 182 182 182 312 312 312 F F F 18 360-379 182 182 182 311 311 311 F F F 19 380-399 184 184 184 314 314 314 F F F 20 400-419 184 184 184 314 314 314 F F F 21 420-439 184 184 184 314 314 314 F F F 22 440-459 182 182 182 311 311 311 F F F 23 460-479 182 182 182 310 310 310 F F F 24 480-499 181 181 181 308 308 308 F F F 25 500-519 182 182 182 312 312 312 F F F 26 520-539 178 178 178 301 301 301 F F F 27 540-559 185 185 185 316 316 316 F F F 28 560-579 182 182 182 312 312 312 F F F 29 580-599 180 180 180 305 305 305 F F F Max gen: 185, Max flush: 316 Begin loop 93 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:11.382522647Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1924,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:57:11.424875755Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1924,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:11.425235881Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1924,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:11.456940766Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1924,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:11.457461333Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1924} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"3818f0fc76fe787c6cc412af30fd59b930203885\",\n git_commit_timestamp: \"2024-11-01T16:08:30.000000000Z\",\n git_branch: \"renovate/hyper-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:11.457507259Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1924} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:11.457524761Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1924} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:11.457542784Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1924} {"msg":"Crucible a81f017a-c1fe-47eb-a3fc-5fa3c3029611 has session id: 4c85976b-ecc6-4636-ae53-6b576108fabd","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:11.457924883Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1924,"session_id":"4c85976b-ecc6-4636-ae53-6b576108fabd"} {"msg":"Upstairs opts: Upstairs UUID: a81f017a-c1fe-47eb-a3fc-5fa3c3029611, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:11.457990031Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1924,"session_id":"4c85976b-ecc6-4636-ae53-6b576108fabd"} {"msg":"Crucible stats registered with UUID: a81f017a-c1fe-47eb-a3fc-5fa3c3029611","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:11.458038088Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1924,"session_id":"4c85976b-ecc6-4636-ae53-6b576108fabd"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:11.488609616Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1924,"task":"crutest"} {"msg":"The guest has requested activation with gen:186","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:11.48863741Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1924} {"msg":"a81f017a-c1fe-47eb-a3fc-5fa3c3029611 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:11.48871335Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1924,"session_id":"4c85976b-ecc6-4636-ae53-6b576108fabd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:11.488736333Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1924,"client":"0","":"downstairs","session_id":"4c85976b-ecc6-4636-ae53-6b576108fabd"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:11.488765757Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1924,"client":"0","":"downstairs","session_id":"4c85976b-ecc6-4636-ae53-6b576108fabd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:11.488783079Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1924,"client":"1","":"downstairs","session_id":"4c85976b-ecc6-4636-ae53-6b576108fabd"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:11.488800451Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1924,"client":"1","":"downstairs","session_id":"4c85976b-ecc6-4636-ae53-6b576108fabd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:11.488817173Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1924,"client":"2","":"downstairs","session_id":"4c85976b-ecc6-4636-ae53-6b576108fabd"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:11.488834036Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1924,"client":"2","":"downstairs","session_id":"4c85976b-ecc6-4636-ae53-6b576108fabd"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:11.488850618Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1924,"":"io task","client":"2","":"downstairs","session_id":"4c85976b-ecc6-4636-ae53-6b576108fabd"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:11.488880391Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1924,"":"io task","client":"0","":"downstairs","session_id":"4c85976b-ecc6-4636-ae53-6b576108fabd"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:11.488914666Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1924,"":"io task","client":"1","":"downstairs","session_id":"4c85976b-ecc6-4636-ae53-6b576108fabd"} {"msg":"ds_connection connected from Ok(127.0.0.1:53134)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:11.48895096Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1924,"":"io task","client":"2","":"downstairs","session_id":"4c85976b-ecc6-4636-ae53-6b576108fabd"} {"msg":"ds_connection connected from Ok(127.0.0.1:40903)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:11.488987075Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1924,"":"io task","client":"1","":"downstairs","session_id":"4c85976b-ecc6-4636-ae53-6b576108fabd"} {"msg":"ds_connection connected from Ok(127.0.0.1:39446)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:11.489186921Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1924,"":"io task","client":"0","":"downstairs","session_id":"4c85976b-ecc6-4636-ae53-6b576108fabd"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:11.489743233Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1924,"client":"2","":"downstairs","session_id":"4c85976b-ecc6-4636-ae53-6b576108fabd"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:11.489771436Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1924,"client":"2","":"downstairs","session_id":"4c85976b-ecc6-4636-ae53-6b576108fabd"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:11.489787758Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1924,"client":"1","":"downstairs","session_id":"4c85976b-ecc6-4636-ae53-6b576108fabd"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:11.48980203Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1924,"client":"1","":"downstairs","session_id":"4c85976b-ecc6-4636-ae53-6b576108fabd"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:11.489815792Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1924,"client":"0","":"downstairs","session_id":"4c85976b-ecc6-4636-ae53-6b576108fabd"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:11.489829124Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1924,"client":"0","":"downstairs","session_id":"4c85976b-ecc6-4636-ae53-6b576108fabd"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:11.490438272Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1924,"client":"2","":"downstairs","session_id":"4c85976b-ecc6-4636-ae53-6b576108fabd"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:11.490472907Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1924,"client":"1","":"downstairs","session_id":"4c85976b-ecc6-4636-ae53-6b576108fabd"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:11.490540876Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1924,"client":"0","":"downstairs","session_id":"4c85976b-ecc6-4636-ae53-6b576108fabd"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:11.491023018Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1924,"client":"2","":"downstairs","session_id":"4c85976b-ecc6-4636-ae53-6b576108fabd"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:11.491051851Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1924,"client":"2","":"downstairs","session_id":"4c85976b-ecc6-4636-ae53-6b576108fabd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:11.491067363Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1924,"session_id":"4c85976b-ecc6-4636-ae53-6b576108fabd"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:11.491084146Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1924,"client":"1","":"downstairs","session_id":"4c85976b-ecc6-4636-ae53-6b576108fabd"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:11.491097567Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1924,"client":"1","":"downstairs","session_id":"4c85976b-ecc6-4636-ae53-6b576108fabd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:11.491110629Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1924,"session_id":"4c85976b-ecc6-4636-ae53-6b576108fabd"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:11.491125481Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1924,"client":"0","":"downstairs","session_id":"4c85976b-ecc6-4636-ae53-6b576108fabd"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:11.491139062Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1924,"client":"0","":"downstairs","session_id":"4c85976b-ecc6-4636-ae53-6b576108fabd"} {"msg":"[0]R flush_numbers[0..12]: [304, 315, 312, 293, 301, 308, 316, 313, 308, 302, 311, 308]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:11.491152984Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1924,"":"downstairs","":"downstairs","session_id":"4c85976b-ecc6-4636-ae53-6b576108fabd"} {"msg":"[0]R generation[0..12]: [180, 184, 182, 173, 178, 181, 185, 183, 181, 178, 182, 181]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:11.491167156Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1924,"":"downstairs","":"downstairs","session_id":"4c85976b-ecc6-4636-ae53-6b576108fabd"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:11.491181478Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1924,"":"downstairs","":"downstairs","session_id":"4c85976b-ecc6-4636-ae53-6b576108fabd"} {"msg":"[1]R flush_numbers[0..12]: [304, 315, 312, 293, 301, 308, 316, 313, 308, 302, 311, 308]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:11.49119523Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1924,"":"downstairs","":"downstairs","session_id":"4c85976b-ecc6-4636-ae53-6b576108fabd"} {"msg":"[1]R generation[0..12]: [180, 184, 182, 173, 178, 181, 185, 183, 181, 178, 182, 181]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:11.491208962Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1924,"":"downstairs","":"downstairs","session_id":"4c85976b-ecc6-4636-ae53-6b576108fabd"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:11.491222983Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1924,"":"downstairs","":"downstairs","session_id":"4c85976b-ecc6-4636-ae53-6b576108fabd"} {"msg":"[2]R flush_numbers[0..12]: [304, 315, 312, 293, 301, 308, 316, 313, 308, 302, 311, 308]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:11.491236825Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1924,"":"downstairs","":"downstairs","session_id":"4c85976b-ecc6-4636-ae53-6b576108fabd"} {"msg":"[2]R generation[0..12]: [180, 184, 182, 173, 178, 181, 185, 183, 181, 178, 182, 181]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:11.491250637Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1924,"":"downstairs","":"downstairs","session_id":"4c85976b-ecc6-4636-ae53-6b576108fabd"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:11.491265709Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1924,"":"downstairs","":"downstairs","session_id":"4c85976b-ecc6-4636-ae53-6b576108fabd"} {"msg":"Max found gen is 186","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:11.49127884Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1924,"":"downstairs","":"downstairs","session_id":"4c85976b-ecc6-4636-ae53-6b576108fabd"} {"msg":"Generation requested: 186 >= found:186","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:11.491292132Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1924,"":"downstairs","":"downstairs","session_id":"4c85976b-ecc6-4636-ae53-6b576108fabd"} {"msg":"Next flush: 317","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:11.491305084Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1924,"":"downstairs","":"downstairs","session_id":"4c85976b-ecc6-4636-ae53-6b576108fabd"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:11.491320366Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1924,"":"downstairs","":"downstairs","session_id":"4c85976b-ecc6-4636-ae53-6b576108fabd"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:11.491333988Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1924,"session_id":"4c85976b-ecc6-4636-ae53-6b576108fabd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"Wait for a query_work_queue command to finish before sending IO crucible","level":30,"time":"2024-11-01T16:57:11.49135544Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1924,"client":"0","":"downstairs","session_id":"4c85976b-ecc6-4636-ae53-6b576108fabd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:11.491369142Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1924,"client":"1","":"downstairs","session_id":"4c85976b-ecc6-4636-ae53-6b576108fabd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:11.491383344Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1924,"client":"2","":"downstairs","session_id":"4c85976b-ecc6-4636-ae53-6b576108fabd"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:11.491396856Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1924,"session_id":"4c85976b-ecc6-4636-ae53-6b576108fabd"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:11.491409167Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1924,"session_id":"4c85976b-ecc6-4636-ae53-6b576108fabd"} {"msg":"a81f017a-c1fe-47eb-a3fc-5fa3c3029611 is now active with session: 4c85976b-ecc6-4636-ae53-6b576108fabd","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:11.491421919Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1924,"session_id":"4c85976b-ecc6-4636-ae53-6b576108fabd"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:11.491433811Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1924,"session_id":"4c85976b-ecc6-4636-ae53-6b576108fabd"} {"msg":"The guest has finished waiting for activation with:186","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:11.491446042Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1924} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 336 len 16384 data: 12 12 12 10 02/30 Write block 91 len 12288 data: 21 19 19 03/30 Read block 355 len 20480 04/30 Write block 452 len 36864 data: 16 15 15 11 12 11 13 12 10 05/30 Read block 507 len 32768 06/30 Read block 226 len 4096 07/30 Read block 88 len 24576 08/30 Read block 415 len 24576 09/30 Write block 420 len 8192 data: 13 15 10/30 Read block 264 len 16384 11/30 Read block 341 len 12288 12/30 Flush 13/30 Write block 294 len 4096 data: 20 14/30 Write block 315 len 40960 data: 16 15 12 13 14 16 16 15 11 11 15/30 Write block 462 len 16384 data: 15 14 12 12 16/30 Write block 589 len 16384 data: 16 14 13 11 17/30 Flush 18/30 Flush 19/30 Write block 280 len 28672 data: 13 12 13 8 6 8 13 20/30 Read block 216 len 36864 21/30 Write block 484 len 32768 data: 16 16 14 17 19 17 17 16 22/30 Read block 408 len 28672 23/30 Read block 270 len 16384 24/30 Read block 4 len 32768 25/30 Write block 331 len 4096 data: 10 26/30 Write block 342 len 24576 data: 8 10 11 12 13 14 27/30 Read block 328 len 28672 28/30 Write block 219 len 12288 data: 16 15 16 29/30 Write block 330 len 16384 data: 10 11 10 11 30/30 Read block 558 len 12288 ---------------------------------------------------------------- Crucible gen:186 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1018 Acked Write 7 Done Done Done false 1019 Acked Read 9 Done Done Done false 1020 Acked Write 8 Done Done Done false 1021 Acked Read 7 Done Done Done false 1022 Acked Read 4 Done Done Done false 1023 Acked Read 8 Done Done Done false 1024 Acked Write 1 Done Done Done false 1025 Acked Write 6 Done Done Done false 1026 Acked Read 7 Done Done Done false 1027 Acked Write 3 Done Done Done false 1028 Acked Write 4 Done Done Sent false 1029 Acked Read 3 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 2 2 Done 12 12 10 34 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1017 1017 1017 Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:57:11.51541377Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1924} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 180 180 180 304 304 304 F F F 1 020-039 184 184 184 315 315 315 F F F 2 040-059 182 182 182 312 312 312 F F F 3 060-079 173 173 173 293 293 293 F F F 4 080-099 186 186 186 317 317 317 F F F 5 100-119 181 181 181 308 308 308 F F F 6 120-139 185 185 185 316 316 316 F F F 7 140-159 183 183 183 313 313 313 F F F 8 160-179 181 181 181 308 308 308 F F F 9 180-199 178 178 178 302 302 302 F F F 10 200-219 182 182 182 311 311 311 T T T 11 220-239 181 181 181 308 308 308 T T T 12 240-259 185 185 185 316 316 316 F F F 13 260-279 185 185 185 316 316 316 F F F 14 280-299 186 186 186 318 318 318 T T T 15 300-319 186 186 186 318 318 318 F F F 16 320-339 186 186 186 318 318 318 T T T 17 340-359 182 182 182 312 312 312 T T T 18 360-379 182 182 182 311 311 311 F F F 19 380-399 184 184 184 314 314 314 F F F 20 400-419 184 184 184 314 314 314 F F F 21 420-439 186 186 186 317 317 317 F F F 22 440-459 186 186 186 317 317 317 F F F 23 460-479 186 186 186 318 318 318 F F F 24 480-499 181 181 181 308 308 308 T T T 25 500-519 182 182 182 312 312 312 F F F 26 520-539 178 178 178 301 301 301 F F F 27 540-559 185 185 185 316 316 316 F F F 28 560-579 182 182 182 312 312 312 F F F 29 580-599 186 186 186 318 318 318 F F F Max gen: 186, Max flush: 318 A Difference in extent metadata is expected here On loop 93 Verifying data now No change after verify Loop: 93 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 180 180 180 304 304 304 F F F 1 020-039 184 184 184 315 315 315 F F F 2 040-059 182 182 182 312 312 312 F F F 3 060-079 173 173 173 293 293 293 F F F 4 080-099 186 186 186 317 317 317 F F F 5 100-119 181 181 181 308 308 308 F F F 6 120-139 185 185 185 316 316 316 F F F 7 140-159 183 183 183 313 313 313 F F F 8 160-179 181 181 181 308 308 308 F F F 9 180-199 178 178 178 302 302 302 F F F 10 200-219 187 187 187 319 319 319 F F F 11 220-239 187 187 187 319 319 319 F F F 12 240-259 185 185 185 316 316 316 F F F 13 260-279 185 185 185 316 316 316 F F F 14 280-299 187 187 187 319 319 319 F F F 15 300-319 186 186 186 318 318 318 F F F 16 320-339 187 187 187 319 319 319 F F F 17 340-359 187 187 187 319 319 319 F F F 18 360-379 182 182 182 311 311 311 F F F 19 380-399 184 184 184 314 314 314 F F F 20 400-419 184 184 184 314 314 314 F F F 21 420-439 186 186 186 317 317 317 F F F 22 440-459 186 186 186 317 317 317 F F F 23 460-479 186 186 186 318 318 318 F F F 24 480-499 187 187 187 319 319 319 F F F 25 500-519 182 182 182 312 312 312 F F F 26 520-539 178 178 178 301 301 301 F F F 27 540-559 185 185 185 316 316 316 F F F 28 560-579 182 182 182 312 312 312 F F F 29 580-599 186 186 186 318 318 318 F F F Max gen: 187, Max flush: 319 Begin loop 94 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:15.861519254Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1932,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:15.905655931Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1932,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:15.906221524Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1932} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"3818f0fc76fe787c6cc412af30fd59b930203885\",\n git_commit_timestamp: \"2024-11-01T16:08:30.000000000Z\",\n git_branch: \"renovate/hyper-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:15.90626319Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1932} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:15.906278481Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1932} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:15.906293253Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1932} {"msg":"Crucible d2d3cd86-f7a3-4f92-b423-097b18851485 has session id: d30711c5-37d4-45d2-94b0-273103574400","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:15.90681285Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1932,"session_id":"d30711c5-37d4-45d2-94b0-273103574400"} {"msg":"Upstairs opts: Upstairs UUID: d2d3cd86-f7a3-4f92-b423-097b18851485, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:15.906852775Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1932,"session_id":"d30711c5-37d4-45d2-94b0-273103574400"} {"msg":"Crucible stats registered with UUID: d2d3cd86-f7a3-4f92-b423-097b18851485","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:15.906876349Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1932,"session_id":"d30711c5-37d4-45d2-94b0-273103574400"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:15.937349625Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1932,"task":"crutest"} {"msg":"The guest has requested activation with gen:188","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:15.93738672Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1932} {"msg":"d2d3cd86-f7a3-4f92-b423-097b18851485 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:15.937420924Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1932,"session_id":"d30711c5-37d4-45d2-94b0-273103574400"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:15.937438476Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1932,"client":"0","":"downstairs","session_id":"d30711c5-37d4-45d2-94b0-273103574400"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:15.93746731Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1932,"client":"0","":"downstairs","session_id":"d30711c5-37d4-45d2-94b0-273103574400"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:15.937484992Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1932,"client":"1","":"downstairs","session_id":"d30711c5-37d4-45d2-94b0-273103574400"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:15.937502035Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1932,"client":"1","":"downstairs","session_id":"d30711c5-37d4-45d2-94b0-273103574400"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:15.937524547Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1932,"client":"2","":"downstairs","session_id":"d30711c5-37d4-45d2-94b0-273103574400"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:15.937555651Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1932,"client":"2","":"downstairs","session_id":"d30711c5-37d4-45d2-94b0-273103574400"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:15.937588586Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1932,"":"io task","client":"0","":"downstairs","session_id":"d30711c5-37d4-45d2-94b0-273103574400"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:15.93762219Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1932,"":"io task","client":"1","":"downstairs","session_id":"d30711c5-37d4-45d2-94b0-273103574400"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:15.937655484Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1932,"":"io task","client":"2","":"downstairs","session_id":"d30711c5-37d4-45d2-94b0-273103574400"} {"msg":"ds_connection connected from Ok(127.0.0.1:59405)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:15.937687768Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1932,"":"io task","client":"0","":"downstairs","session_id":"d30711c5-37d4-45d2-94b0-273103574400"} {"msg":"ds_connection connected from Ok(127.0.0.1:34751)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:15.937716432Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1932,"":"io task","client":"1","":"downstairs","session_id":"d30711c5-37d4-45d2-94b0-273103574400"} {"msg":"ds_connection connected from Ok(127.0.0.1:50176)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:15.93785675Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1932,"":"io task","client":"2","":"downstairs","session_id":"d30711c5-37d4-45d2-94b0-273103574400"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:15.938522076Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1932,"client":"0","":"downstairs","session_id":"d30711c5-37d4-45d2-94b0-273103574400"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:15.93855079Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1932,"client":"0","":"downstairs","session_id":"d30711c5-37d4-45d2-94b0-273103574400"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:15.938569672Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1932,"client":"1","":"downstairs","session_id":"d30711c5-37d4-45d2-94b0-273103574400"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:15.938587554Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1932,"client":"1","":"downstairs","session_id":"d30711c5-37d4-45d2-94b0-273103574400"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:15.938612608Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1932,"client":"2","":"downstairs","session_id":"d30711c5-37d4-45d2-94b0-273103574400"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:15.938639611Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1932,"client":"2","":"downstairs","session_id":"d30711c5-37d4-45d2-94b0-273103574400"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:15.939182381Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1932,"client":"0","":"downstairs","session_id":"d30711c5-37d4-45d2-94b0-273103574400"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:15.939213015Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1932,"client":"1","":"downstairs","session_id":"d30711c5-37d4-45d2-94b0-273103574400"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:15.939370365Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1932,"client":"2","":"downstairs","session_id":"d30711c5-37d4-45d2-94b0-273103574400"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:15.939762716Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1932,"client":"1","":"downstairs","session_id":"d30711c5-37d4-45d2-94b0-273103574400"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:15.93979223Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1932,"client":"1","":"downstairs","session_id":"d30711c5-37d4-45d2-94b0-273103574400"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:15.939807432Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1932,"session_id":"d30711c5-37d4-45d2-94b0-273103574400"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:15.939888492Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1932,"client":"0","":"downstairs","session_id":"d30711c5-37d4-45d2-94b0-273103574400"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:15.939910115Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1932,"client":"0","":"downstairs","session_id":"d30711c5-37d4-45d2-94b0-273103574400"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:15.939923956Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1932,"session_id":"d30711c5-37d4-45d2-94b0-273103574400"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:15.939936318Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1932,"client":"2","":"downstairs","session_id":"d30711c5-37d4-45d2-94b0-273103574400"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:15.93994975Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1932,"client":"2","":"downstairs","session_id":"d30711c5-37d4-45d2-94b0-273103574400"} {"msg":"[0]R flush_numbers[0..12]: [304, 315, 312, 293, 317, 308, 316, 313, 308, 302, 319, 319]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:15.939967592Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1932,"":"downstairs","":"downstairs","session_id":"d30711c5-37d4-45d2-94b0-273103574400"} {"msg":"[0]R generation[0..12]: [180, 184, 182, 173, 186, 181, 185, 183, 181, 178, 187, 187]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:15.939982244Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1932,"":"downstairs","":"downstairs","session_id":"d30711c5-37d4-45d2-94b0-273103574400"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:15.939996456Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1932,"":"downstairs","":"downstairs","session_id":"d30711c5-37d4-45d2-94b0-273103574400"} {"msg":"[1]R flush_numbers[0..12]: [304, 315, 312, 293, 317, 308, 316, 313, 308, 302, 319, 319]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:15.940010388Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1932,"":"downstairs","":"downstairs","session_id":"d30711c5-37d4-45d2-94b0-273103574400"} {"msg":"[1]R generation[0..12]: [180, 184, 182, 173, 186, 181, 185, 183, 181, 178, 187, 187]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:15.94002416Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1932,"":"downstairs","":"downstairs","session_id":"d30711c5-37d4-45d2-94b0-273103574400"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:15.940038141Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1932,"":"downstairs","":"downstairs","session_id":"d30711c5-37d4-45d2-94b0-273103574400"} {"msg":"[2]R flush_numbers[0..12]: [304, 315, 312, 293, 317, 308, 316, 313, 308, 302, 319, 319]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:15.940052033Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1932,"":"downstairs","":"downstairs","session_id":"d30711c5-37d4-45d2-94b0-273103574400"} {"msg":"[2]R generation[0..12]: [180, 184, 182, 173, 186, 181, 185, 183, 181, 178, 187, 187]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:15.940065975Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1932,"":"downstairs","":"downstairs","session_id":"d30711c5-37d4-45d2-94b0-273103574400"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:15.940080217Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1932,"":"downstairs","":"downstairs","session_id":"d30711c5-37d4-45d2-94b0-273103574400"} {"msg":"Max found gen is 188","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:15.940093228Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1932,"":"downstairs","":"downstairs","session_id":"d30711c5-37d4-45d2-94b0-273103574400"} {"msg":"Generation requested: 188 >= found:188","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:15.94010669Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1932,"":"downstairs","":"downstairs","session_id":"d30711c5-37d4-45d2-94b0-273103574400"} {"msg":"Next flush: 320","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:15.940119842Z","hostname":Wait for a query_work_queue command to finish before sending IO "w-01JBMA2T0XD4W63AXFF321JBBP","pid":1932,"":"downstairs","":"downstairs","session_id":"d30711c5-37d4-45d2-94b0-273103574400"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:15.940141064Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1932,"":"downstairs","":"downstairs","session_id":"d30711c5-37d4-45d2-94b0-273103574400"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:15.940154636Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1932,"session_id":"d30711c5-37d4-45d2-94b0-273103574400"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:15.940168438Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1932,"client":"0","":"downstairs","session_id":"d30711c5-37d4-45d2-94b0-273103574400"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:15.94018211Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1932,"client":"1","":"downstairs","session_id":"d30711c5-37d4-45d2-94b0-273103574400"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:15.940195582Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1932,"client":"2","":"downstairs","session_id":"d30711c5-37d4-45d2-94b0-273103574400"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:15.940208893Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1932,"session_id":"d30711c5-37d4-45d2-94b0-273103574400"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:15.940221285Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1932,"session_id":"d30711c5-37d4-45d2-94b0-273103574400"} {"msg":"d2d3cd86-f7a3-4f92-b423-097b18851485 is now active with session: d30711c5-37d4-45d2-94b0-273103574400","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:15.940234096Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1932,"session_id":"d30711c5-37d4-45d2-94b0-273103574400"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:15.940245998Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1932,"session_id":"d30711c5-37d4-45d2-94b0-273103574400"} {"msg":"The guest has finished waiting for activation with:188","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:15.94025834Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1932} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 102 len 32768 data: 18 17 17 20 21 18 17 17 02/30 Read block 557 len 4096 03/30 Read block 557 len 32768 04/30 Write block 512 len 36864 data: 13 10 11 9 10 12 12 16 15 05/30 Write block 218 len 12288 data: 18 17 16 06/30 Write block 56 len 8192 data: 12 12 07/30 Write block 394 len 32768 data: 12 14 15 15 18 16 14 12 08/30 Write block 476 len 16384 data: 14 19 23 20 09/30 Read block 187 len 16384 10/30 Read block 142 len 28672 11/30 Read block 569 len 20480 12/30 Read block 273 len 16384 13/30 Read block 520 len 24576 14/30 Write block 589 len 4096 data: 17 15/30 Read block 308 len 24576 16/30 Write block 469 len 32768 data: 16 17 19 20 21 18 17 15 17/30 Write block 256 len 24576 data: 13 14 15 11 12 16 18/30 Read block 475 len 12288 19/30 Write block 85 len 8192 data: 16 18 20/30 Read block 492 len 28672 21/30 Read block 405 len 28672 22/30 Read block 147 len 36864 23/30 Read block 58 len 32768 24/30 Read block 392 len 20480 25/30 Write block 90 len 4096 data: 23 26/30 Flush 27/30 Write block 220 len 20480 data: 17 17 16 17 16 28/30 Write block 555 len 40960 data: 15 15 17 20 25 22 21 21 23 21 29/30 Read block 334 len 36864 30/30 Write block 200 len 12288 data: 15 17 16 ---------------------------------------------------------------- Crucible gen:188 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 5 Done Done Done false 1027 Acked Write 10 Done Done Done false 1028 Acked Read 9 Sent Sent Done false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 1 5 Done 2 2 3 7 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:57:15.962025764Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1932} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 180 180 180 304 304 304 F F F 1 020-039 184 184 184 315 315 315 F F F 2 040-059 188 188 188 320 320 320 F F F 3 060-079 173 173 173 293 293 293 F F F 4 080-099 188 188 188 320 320 320 F F F 5 100-119 188 188 188 320 320 320 F F F 6 120-139 185 185 185 316 316 316 F F F 7 140-159 183 183 183 313 313 313 F F F 8 160-179 181 181 181 308 308 308 F F F 9 180-199 178 178 178 302 302 302 F F F 10 200-219 188 188 188 320 320 320 T T T 11 220-239 188 188 188 320 320 320 T T T 12 240-259 188 188 188 320 320 320 F F F 13 260-279 188 188 188 320 320 320 F F F 14 280-299 187 187 187 319 319 319 F F F 15 300-319 186 186 186 318 318 318 F F F 16 320-339 187 187 187 319 319 319 F F F 17 340-359 187 187 187 319 319 319 F F F 18 360-379 182 182 182 311 311 311 F F F 19 380-399 188 188 188 320 320 320 F F F 20 400-419 188 188 188 320 320 320 F F F 21 420-439 186 186 186 317 317 317 F F F 22 440-459 186 186 186 317 317 317 F F F 23 460-479 188 188 188 320 320 320 F F F 24 480-499 187 187 187 319 319 319 F F F 25 500-519 188 188 188 320 320 320 F F F 26 520-539 188 188 188 320 320 320 F F F 27 540-559 185 185 185 316 316 316 T T T 28 560-579 182 182 182 312 312 312 T T T 29 580-599 188 188 188 320 320 320 F F F Max gen: 188, Max flush: 320 A Difference in extent metadata is expected here On loop 94 Verifying data now No change after verify Loop: 94 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 180 180 180 304 304 304 F F F 1 020-039 184 184 184 315 315 315 F F F 2 040-059 188 188 188 320 320 320 F F F 3 060-079 173 173 173 293 293 293 F F F 4 080-099 188 188 188 320 320 320 F F F 5 100-119 188 188 188 320 320 320 F F F 6 120-139 185 185 185 316 316 316 F F F 7 140-159 183 183 183 313 313 313 F F F 8 160-179 181 181 181 308 308 308 F F F 9 180-199 178 178 178 302 302 302 F F F 10 200-219 189 189 189 321 321 321 F F F 11 220-239 189 189 189 321 321 321 F F F 12 240-259 188 188 188 320 320 320 F F F 13 260-279 188 188 188 320 320 320 F F F 14 280-299 187 187 187 319 319 319 F F F 15 300-319 186 186 186 318 318 318 F F F 16 320-339 187 187 187 319 319 319 F F F 17 340-359 187 187 187 319 319 319 F F F 18 360-379 182 182 182 311 311 311 F F F 19 380-399 188 188 188 320 320 320 F F F 20 400-419 188 188 188 320 320 320 F F F 21 420-439 186 186 186 317 317 317 F F F 22 440-459 186 186 186 317 317 317 F F F 23 460-479 188 188 188 320 320 320 F F F 24 480-499 187 187 187 319 319 319 F F F 25 500-519 188 188 188 320 320 320 F F F 26 520-539 188 188 188 320 320 320 F F F 27 540-559 189 189 189 321 321 321 F F F 28 560-579 189 189 189 321 321 321 F F F 29 580-599 188 188 188 320 320 320 F F F Max gen: 189, Max flush: 321 Begin loop 95 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:20.258774878Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1940,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:20.30263684Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1940,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:20.30325792Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1940} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"3818f0fc76fe787c6cc412af30fd59b930203885\",\n git_commit_timestamp: \"2024-11-01T16:08:30.000000000Z\",\n git_branch: \"renovate/hyper-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:20.303304136Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1940} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:20.303329009Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1940} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:20.303356943Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1940} {"msg":"Crucible d5c04c88-7bcb-492a-8dbd-9ed14e5665ec has session id: 3c712bfe-ae11-4693-bf4b-31a3435acc86","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:20.303783778Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1940,"session_id":"3c712bfe-ae11-4693-bf4b-31a3435acc86"} {"msg":"Upstairs opts: Upstairs UUID: d5c04c88-7bcb-492a-8dbd-9ed14e5665ec, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:20.303827093Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1940,"session_id":"3c712bfe-ae11-4693-bf4b-31a3435acc86"} {"msg":"Crucible stats registered with UUID: d5c04c88-7bcb-492a-8dbd-9ed14e5665ec","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:20.303852547Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1940,"session_id":"3c712bfe-ae11-4693-bf4b-31a3435acc86"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:20.334638473Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1940,"task":"crutest"} {"msg":"The guest has requested activation with gen:190","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:20.334666577Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1940} {"msg":"d5c04c88-7bcb-492a-8dbd-9ed14e5665ec active request set","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:20.334740456Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1940,"session_id":"3c712bfe-ae11-4693-bf4b-31a3435acc86"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:20.334762039Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1940,"client":"0","":"downstairs","session_id":"3c712bfe-ae11-4693-bf4b-31a3435acc86"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:20.334792313Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1940,"client":"0","":"downstairs","session_id":"3c712bfe-ae11-4693-bf4b-31a3435acc86"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:20.334814746Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1940,"client":"1","":"downstairs","session_id":"3c712bfe-ae11-4693-bf4b-31a3435acc86"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:20.334832278Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1940,"client":"1","":"downstairs","session_id":"3c712bfe-ae11-4693-bf4b-31a3435acc86"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:20.33484913Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1940,"client":"2","":"downstairs","session_id":"3c712bfe-ae11-4693-bf4b-31a3435acc86"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:20.334866052Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1940,"client":"2","":"downstairs","session_id":"3c712bfe-ae11-4693-bf4b-31a3435acc86"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:20.334886625Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1940,"":"io task","client":"2","":"downstairs","session_id":"3c712bfe-ae11-4693-bf4b-31a3435acc86"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:20.334916959Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1940,"":"io task","client":"0","":"downstairs","session_id":"3c712bfe-ae11-4693-bf4b-31a3435acc86"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:20.334947323Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1940,"":"io task","client":"1","":"downstairs","session_id":"3c712bfe-ae11-4693-bf4b-31a3435acc86"} {"msg":"ds_connection connected from Ok(127.0.0.1:34909)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:20.334978417Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1940,"":"io task","client":"2","":"downstairs","session_id":"3c712bfe-ae11-4693-bf4b-31a3435acc86"} {"msg":"ds_connection connected from Ok(127.0.0.1:62030)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:20.335254022Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1940,"":"io task","client":"0","":"downstairs","session_id":"3c712bfe-ae11-4693-bf4b-31a3435acc86"} {"msg":"ds_connection connected from Ok(127.0.0.1:34415)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:20.335296218Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1940,"":"io task","client":"1","":"downstairs","session_id":"3c712bfe-ae11-4693-bf4b-31a3435acc86"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:20.335828006Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1940,"client":"1","":"downstairs","session_id":"3c712bfe-ae11-4693-bf4b-31a3435acc86"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:20.33585749Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1940,"client":"1","":"downstairs","session_id":"3c712bfe-ae11-4693-bf4b-31a3435acc86"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:20.33593101Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1940,"client":"2","":"downstairs","session_id":"3c712bfe-ae11-4693-bf4b-31a3435acc86"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:20.335967084Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1940,"client":"2","":"downstairs","session_id":"3c712bfe-ae11-4693-bf4b-31a3435acc86"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:20.335989057Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1940,"client":"0","":"downstairs","session_id":"3c712bfe-ae11-4693-bf4b-31a3435acc86"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:20.336032973Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1940,"client":"0","":"downstairs","session_id":"3c712bfe-ae11-4693-bf4b-31a3435acc86"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:20.33655189Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1940,"client":"0","":"downstairs","session_id":"3c712bfe-ae11-4693-bf4b-31a3435acc86"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:20.336581384Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1940,"client":"1","":"downstairs","session_id":"3c712bfe-ae11-4693-bf4b-31a3435acc86"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:20.336611467Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1940,"client":"2","":"downstairs","session_id":"3c712bfe-ae11-4693-bf4b-31a3435acc86"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:20.337144976Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1940,"client":"1","":"downstairs","session_id":"3c712bfe-ae11-4693-bf4b-31a3435acc86"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:20.33717361Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1940,"client":"1","":"downstairs","session_id":"3c712bfe-ae11-4693-bf4b-31a3435acc86"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:20.337188682Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1940,"session_id":"3c712bfe-ae11-4693-bf4b-31a3435acc86"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:20.337201313Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1940,"client":"2","":"downstairs","session_id":"3c712bfe-ae11-4693-bf4b-31a3435acc86"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:20.337214955Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1940,"client":"2","":"downstairs","session_id":"3c712bfe-ae11-4693-bf4b-31a3435acc86"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:20.337228267Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1940,"session_id":"3c712bfe-ae11-4693-bf4b-31a3435acc86"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:20.337240618Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1940,"client":"0","":"downstairs","session_id":"3c712bfe-ae11-4693-bf4b-31a3435acc86"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:20.337254Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1940,"client":"0","":"downstairs","session_id":"3c712bfe-ae11-4693-bf4b-31a3435acc86"} {"msg":"[0]R flush_numbers[0..12]: [304, 315, 320, 293, 320, 320, 316, 313, 308, 302, 321, 321]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:20.337268062Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1940,"":"downstairs","":"downstairs","session_id":"3c712bfe-ae11-4693-bf4b-31a3435acc86"} {"msg":"[0]R generation[0..12]: [180, 184, 188, 173, 188, 188, 185, 183, 181, 178, 189, 189]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:20.337282054Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1940,"":"downstairs","":"downstairs","session_id":"3c712bfe-ae11-4693-bf4b-31a3435acc86"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:20.337296245Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1940,"":"downstairs","":"downstairs","session_id":"3c712bfe-ae11-4693-bf4b-31a3435acc86"} {"msg":"[1]R flush_numbers[0..12]: [304, 315, 320, 293, 320, 320, 316, 313, 308, 302, 321, 321]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:20.337313368Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1940,"":"downstairs","":"downstairs","session_id":"3c712bfe-ae11-4693-bf4b-31a3435acc86"} {"msg":"[1]R generation[0..12]: [180, 184, 188, 173, 188, 188, 185, 183, 181, 178, 189, 189]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:20.33732764Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1940,"":"downstairs","":"downstairs","session_id":"3c712bfe-ae11-4693-bf4b-31a3435acc86"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:20.337341662Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1940,"":"downstairs","":"downstairs","session_id":"3c712bfe-ae11-4693-bf4b-31a3435acc86"} {"msg":"[2]R flush_numbers[0..12]: [304, 315, 320, 293, 320, 320, 316, 313, 308, 302, 321, 321]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:20.337355463Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1940,"":"downstairs","":"downstairs","session_id":"3c712bfe-ae11-4693-bf4b-31a3435acc86"} {"msg":"[2]R generation[0..12]: [180, 184, 188, 173, 188, 188, 185, 183, 181, 178, 189, 189]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:20.337369275Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1940,"":"downstairs","":"downstairs","session_id":"3c712bfe-ae11-4693-bf4b-31a3435acc86"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:20.337383177Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1940,"":"downstairs","":"downstairs","session_id":"3c712bfe-ae11-4693-bf4b-31a3435acc86"} {"msg":"Max found gen is 190","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-01T16:57:20.337396129Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1940,"":"downstairs","":"downstairs","session_id":"3c712bfe-ae11-4693-bf4b-31a3435acc86"} {"msg":"Generation requested: 190 >= found:190","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:20.337416891Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1940,"":"downstairs","":"downstairs","session_id":"3c712bfe-ae11-4693-bf4b-31a3435acc86"} {"msg":"Next flush: 322","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:20.337430113Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1940,"":"downstairs","":"downstairs","session_id":"3c712bfe-ae11-4693-bf4b-31a3435acc86"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:20.337443255Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1940,"":"downstairs","":"downstairs","session_id":"3c712bfe-ae11-4693-bf4b-31a3435acc86"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:20.337456466Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1940,"session_id":"3c712bfe-ae11-4693-bf4b-31a3435acc86"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:20.337468688Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1940,"client":"0","":"downstairs","session_id":"3c712bfe-ae11-4693-bf4b-31a3435acc86"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:20.33748186Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1940,"client":"1","":"downstairs","session_id":"3c712bfe-ae11-4693-bf4b-31a3435acc86"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:20.337495191Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1940,"client":"2","":"downstairs","session_id":"3c712bfe-ae11-4693-bf4b-31a3435acc86"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:20.337508323Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1940,"session_id":"3c712bfe-ae11-4693-bf4b-31a3435acc86"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:20.337522355Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1940,"session_id":"3c712bfe-ae11-4693-bf4b-31a3435acc86"} {"msg":"d5c04c88-7bcb-492a-8dbd-9ed14e5665ec is now active with session: 3c712bfe-ae11-4693-bf4b-31a3435acc86","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:20.337535466Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1940,"session_id":"3c712bfe-ae11-4693-bf4b-31a3435acc86"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:20.337547348Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1940,"session_id":"3c712bfe-ae11-4693-bf4b-31a3435acc86"} {"msg":"The guest has finished waiting for activation with:190","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:20.33755959Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1940} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 122 len 8192 data: 20 16 02/30 Read block 113 len 16384 03/30 Read block 494 len 8192 04/30 Read block 373 len 4096 05/30 Write block 218 len 40960 data: 19 18 18 18 17 18 17 15 13 13 06/30 Write block 27 len 4096 data: 19 07/30 Write block 159 len 16384 data: 15 16 14 12 08/30 Read block 260 len 4096 09/30 Read block 440 len 28672 10/30 Read block 340 len 40960 11/30 Write block 589 len 20480 data: 18 15 14 12 12 12/30 Write block 341 len 16384 data: 7 9 11 12 13/30 Flush 14/30 Write block 537 len 36864 data: 14 14 15 14 9 7 11 9 11 15/30 Read block 104 len 36864 16/30 Read block 589 len 4096 17/30 Write block 363 len 40960 data: 10 10 11 9 9 8 9 8 6 8 18/30 Write block 282 len 36864 data: 14 9 7 9 14 14 14 14 14 19/30 Write block 90 len 12288 data: 24 22 20 20/30 Write block 369 len 36864 data: 10 9 7 9 12 12 11 12 15 21/30 Read block 285 len 4096 22/30 Read block 549 len 28672 23/30 Read block 247 len 36864 24/30 Read block 98 len 32768 25/30 Read block 371 len 24576 26/30 Read block 331 len 40960 27/30 Write block 122 len 28672 data: 21 17 14 14 16 14 15 28/30 Read block 180 len 20480 29/30 Write block 110 len 16384 data: 16 15 14 14 30/30 Read block 486 len 20480 ---------------------------------------------------------------- Crucible gen:190 GIO:true work queues: Upstairs:0 downstairs:17 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1013 Acked Write 9 Done Done Done false 1014 Acked Read 9 Done Done Done false 1015 Acked Read 1 Done Done Done false 1016 Acked Write 10 Done Done Done false 1017 Acked Write 9 Done Done Done false 1018 Acked Write 3 Done Done Done false 1019 Acked Write 9 Done Done Done false 1020 Acked Read 1 Done Done Done false 1021 Acked Read 7 Done Done Done false 1022 Acked Read 9 Done Done Done false 1023 Acked Read 8 Done Done Done false 1024 Acked Read 6 Done Done Done false 1025 Acked Read 10 Done Done Done false 1026 Acked Write 7 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Write 4 Done Done Sent false 1029 Acked Read 5 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 2 3 Done 17 16 15 48 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1012 1012 1012 Downstairs last five completed: 1012 1011 1010 1009 1008 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:57:20.360241942Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1940} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 180 180 180 304 304 304 F F F 1 020-039 190 190 190 322 322 322 F F F 2 040-059 188 188 188 320 320 320 F F F 3 060-079 173 173 173 293 293 293 F F F 4 080-099 188 188 188 320 320 320 T T T 5 100-119 188 188 188 320 320 320 T T T 6 120-139 190 190 190 322 322 322 T T T 7 140-159 190 190 190 322 322 322 F F F 8 160-179 190 190 190 322 322 322 F F F 9 180-199 178 178 178 302 302 302 F F F 10 200-219 190 190 190 322 322 322 F F F 11 220-239 190 190 190 322 322 322 F F F 12 240-259 188 188 188 320 320 320 F F F 13 260-279 188 188 188 320 320 320 F F F 14 280-299 187 187 187 319 319 319 T T T 15 300-319 186 186 186 318 318 318 F F F 16 320-339 187 187 187 319 319 319 F F F 17 340-359 190 190 190 322 322 322 F F F 18 360-379 182 182 182 311 311 311 T T T 19 380-399 188 188 188 320 320 320 F F F 20 400-419 188 188 188 320 320 320 F F F 21 420-439 186 186 186 317 317 317 F F F 22 440-459 186 186 186 317 317 317 F F F 23 460-479 188 188 188 320 320 320 F F F 24 480-499 187 187 187 319 319 319 F F F 25 500-519 188 188 188 320 320 320 F F F 26 520-539 188 188 188 320 320 320 T T T 27 540-559 189 189 189 321 321 321 T T T 28 560-579 189 189 189 321 321 321 F F F 29 580-599 190 190 190 322 322 322 F F F Max gen: 190, Max flush: 322 A Difference in extent metadata is expected here On loop 95 Verifying data now No change after verify Loop: 95 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 180 180 180 304 304 304 F F F 1 020-039 190 190 190 322 322 322 F F F 2 040-059 188 188 188 320 320 320 F F F 3 060-079 173 173 173 293 293 293 F F F 4 080-099 191 191 191 323 323 323 F F F 5 100-119 191 191 191 323 323 323 F F F 6 120-139 191 191 191 323 323 323 F F F 7 140-159 190 190 190 322 322 322 F F F 8 160-179 190 190 190 322 322 322 F F F 9 180-199 178 178 178 302 302 302 F F F 10 200-219 190 190 190 322 322 322 F F F 11 220-239 190 190 190 322 322 322 F F F 12 240-259 188 188 188 320 320 320 F F F 13 260-279 188 188 188 320 320 320 F F F 14 280-299 191 191 191 323 323 323 F F F 15 300-319 186 186 186 318 318 318 F F F 16 320-339 187 187 187 319 319 319 F F F 17 340-359 190 190 190 322 322 322 F F F 18 360-379 191 191 191 323 323 323 F F F 19 380-399 188 188 188 320 320 320 F F F 20 400-419 188 188 188 320 320 320 F F F 21 420-439 186 186 186 317 317 317 F F F 22 440-459 186 186 186 317 317 317 F F F 23 460-479 188 188 188 320 320 320 F F F 24 480-499 187 187 187 319 319 319 F F F 25 500-519 188 188 188 320 320 320 F F F 26 520-539 191 191 191 323 323 323 F F F 27 540-559 191 191 191 323 323 323 F F F 28 560-579 189 189 189 321 321 321 F F F 29 580-599 190 190 190 322 322 322 F F F Max gen: 191, Max flush: 323 Begin loop 96 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:24.690345175Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1948,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:24.734527938Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1948,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:24.735237719Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1948} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"3818f0fc76fe787c6cc412af30fd59b930203885\",\n git_commit_timestamp: \"2024-11-01T16:08:30.000000000Z\",\n git_branch: \"renovate/hyper-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:24.735284475Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1948} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:24.735301937Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1948} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:24.73531974Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1948} {"msg":"Crucible 78d900ce-f1e1-44eb-9347-bfeb1ea7a6af has session id: ecf9b911-8cfb-40e8-b923-800e74a0a3a2","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:24.735717491Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1948,"session_id":"ecf9b911-8cfb-40e8-b923-800e74a0a3a2"} {"msg":"Upstairs opts: Upstairs UUID: 78d900ce-f1e1-44eb-9347-bfeb1ea7a6af, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:24.735766387Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1948,"session_id":"ecf9b911-8cfb-40e8-b923-800e74a0a3a2"} {"msg":"Crucible stats registered with UUID: 78d900ce-f1e1-44eb-9347-bfeb1ea7a6af","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:24.735795501Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1948,"session_id":"ecf9b911-8cfb-40e8-b923-800e74a0a3a2"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:24.767170014Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1948,"task":"crutest"} {"msg":"The guest has requested activation with gen:192","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:24.767201168Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1948} {"msg":"78d900ce-f1e1-44eb-9347-bfeb1ea7a6af active request set","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:24.767270437Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1948,"session_id":"ecf9b911-8cfb-40e8-b923-800e74a0a3a2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:24.76729339Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1948,"client":"0","":"downstairs","session_id":"ecf9b911-8cfb-40e8-b923-800e74a0a3a2"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:24.767322283Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1948,"client":"0","":"downstairs","session_id":"ecf9b911-8cfb-40e8-b923-800e74a0a3a2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:24.767339585Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1948,"client":"1","":"downstairs","session_id":"ecf9b911-8cfb-40e8-b923-800e74a0a3a2"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:24.767360708Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1948,"client":"1","":"downstairs","session_id":"ecf9b911-8cfb-40e8-b923-800e74a0a3a2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:24.767377681Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1948,"client":"2","":"downstairs","session_id":"ecf9b911-8cfb-40e8-b923-800e74a0a3a2"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:24.767394392Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1948,"client":"2","":"downstairs","session_id":"ecf9b911-8cfb-40e8-b923-800e74a0a3a2"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:24.767417066Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1948,"":"io task","client":"2","":"downstairs","session_id":"ecf9b911-8cfb-40e8-b923-800e74a0a3a2"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:24.76744956Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1948,"":"io task","client":"0","":"downstairs","session_id":"ecf9b911-8cfb-40e8-b923-800e74a0a3a2"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:24.767495136Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1948,"":"io task","client":"1","":"downstairs","session_id":"ecf9b911-8cfb-40e8-b923-800e74a0a3a2"} {"msg":"ds_connection connected from Ok(127.0.0.1:55441)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:24.76753047Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1948,"":"io task","client":"2","":"downstairs","session_id":"ecf9b911-8cfb-40e8-b923-800e74a0a3a2"} {"msg":"ds_connection connected from Ok(127.0.0.1:49699)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:24.767566505Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1948,"":"io task","client":"0","":"downstairs","session_id":"ecf9b911-8cfb-40e8-b923-800e74a0a3a2"} {"msg":"ds_connection connected from Ok(127.0.0.1:45704)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:24.767601639Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1948,"":"io task","client":"1","":"downstairs","session_id":"ecf9b911-8cfb-40e8-b923-800e74a0a3a2"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:24.76830644Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1948,"client":"2","":"downstairs","session_id":"ecf9b911-8cfb-40e8-b923-800e74a0a3a2"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:24.768333794Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1948,"client":"2","":"downstairs","session_id":"ecf9b911-8cfb-40e8-b923-800e74a0a3a2"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:24.768362377Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1948,"client":"1","":"downstairs","session_id":"ecf9b911-8cfb-40e8-b923-800e74a0a3a2"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:24.768377229Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1948,"client":"1","":"downstairs","session_id":"ecf9b911-8cfb-40e8-b923-800e74a0a3a2"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:24.768390661Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1948,"client":"0","":"downstairs","session_id":"ecf9b911-8cfb-40e8-b923-800e74a0a3a2"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:24.768404083Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1948,"client":"0","":"downstairs","session_id":"ecf9b911-8cfb-40e8-b923-800e74a0a3a2"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:24.769057887Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1948,"client":"0","":"downstairs","session_id":"ecf9b911-8cfb-40e8-b923-800e74a0a3a2"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:24.769086591Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1948,"client":"2","":"downstairs","session_id":"ecf9b911-8cfb-40e8-b923-800e74a0a3a2"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:24.769102073Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1948,"client":"1","":"downstairs","session_id":"ecf9b911-8cfb-40e8-b923-800e74a0a3a2"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:24.769610128Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1948,"client":"0","":"downstairs","session_id":"ecf9b911-8cfb-40e8-b923-800e74a0a3a2"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:24.769636212Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1948,"client":"0","":"downstairs","session_id":"ecf9b911-8cfb-40e8-b923-800e74a0a3a2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:24.769650043Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1948,"session_id":"ecf9b911-8cfb-40e8-b923-800e74a0a3a2"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:24.769662555Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1948,"client":"1","":"downstairs","session_id":"ecf9b911-8cfb-40e8-b923-800e74a0a3a2"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:24.769676057Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1948,"client":"1","":"downstairs","session_id":"ecf9b911-8cfb-40e8-b923-800e74a0a3a2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:24.769689438Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1948,"session_id":"ecf9b911-8cfb-40e8-b923-800e74a0a3a2"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:24.76970153Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1948,"client":"2","":"downstairs","session_id":"ecf9b911-8cfb-40e8-b923-800e74a0a3a2"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:24.769714902Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1948,"client":"2","":"downstairs","session_id":"ecf9b911-8cfb-40e8-b923-800e74a0a3a2"} {"msg":"[0]R flush_numbers[0..12]: [304, 322, 320, 293, 323, 323, 323, 322, 322, 302, 322, 322]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:24.769728553Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1948,"":"downstairs","":"downstairs","session_id":"ecf9b911-8cfb-40e8-b923-800e74a0a3a2"} {"msg":"[0]R generation[0..12]: [180, 190, 188, 173, 191, 191, 191, 190, 190, 178, 190, 190]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:24.769742445Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1948,"":"downstairs","":"downstairs","session_id":"ecf9b911-8cfb-40e8-b923-800e74a0a3a2"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:24.769756377Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1948,"":"downstairs","":"downstairs","session_id":"ecf9b911-8cfb-40e8-b923-800e74a0a3a2"} {"msg":"[1]R flush_numbers[0..12]: [304, 322, 320, 293, 323, 323, 323, 322, 322, 302, 322, 322]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:24.769770009Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1948,"":"downstairs","":"downstairs","session_id":"ecf9b911-8cfb-40e8-b923-800e74a0a3a2"} {"msg":"[1]R generation[0..12]: [180, 190, 188, 173, 191, 191, 191, 190, 190, 178, 190, 190]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:24.769786681Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1948,"":"downstairs","":"downstairs","session_id":"ecf9b911-8cfb-40e8-b923-800e74a0a3a2"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:24.769801043Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1948,"":"downstairs","":"downstairs","session_id":"ecf9b911-8cfb-40e8-b923-800e74a0a3a2"} {"msg":"[2]R flush_numbers[0..12]: [304, 322, 320, 293, 323, 323, 323, 322, 322, 302, 322, 322]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:24.769814955Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1948,"":"downstairs","":"downstairs","session_id":"ecf9b911-8cfb-40e8-b923-800e74a0a3a2"} {"msg":"[2]R generation[0..12]: [180, 190, 188, 173, 191, 191, 191, 190, 190, 178, 190, 190]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:24.769828756Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1948,"":"downstairs","":"downstairs","session_id":"ecf9b911-8cfb-40e8-b923-800e74a0a3a2"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:24.769842678Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1948,"":"downstairs","":"downstairs","session_id":"ecf9b911-8cfb-40e8-b923-800e74a0a3a2"} {"msg":"Max found gen is 192","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-01T16:57:24.7698558Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1948,"":"downstairs","":"downstairs","session_id":"ecf9b911-8cfb-40e8-b923-800e74a0a3a2"} {"msg":"Generation requested: 192 >= found:192","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:24.769873882Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1948,"":"downstairs","":"downstairs","session_id":"ecf9b911-8cfb-40e8-b923-800e74a0a3a2"} {"msg":"Next flush: 324","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:24.769886914Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1948,"":"downstairs","":"downstairs","session_id":"ecf9b911-8cfb-40e8-b923-800e74a0a3a2"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:24.769900686Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1948,"":"downstairs","":"downstairs","session_id":"ecf9b911-8cfb-40e8-b923-800e74a0a3a2"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:24.769913767Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1948,"session_id":"ecf9b911-8cfb-40e8-b923-800e74a0a3a2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:24.769925839Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1948,"client":"0","":"downstairs","session_id":"ecf9b911-8cfb-40e8-b923-800e74a0a3a2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:24.769938951Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1948,"client":"1","":"downstairs","session_id":"ecf9b911-8cfb-40e8-b923-800e74a0a3a2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:24.769952222Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1948,"client":"2","":"downstairs","session_id":"ecf9b911-8cfb-40e8-b923-800e74a0a3a2"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:24.769965354Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1948,"session_id":"ecf9b911-8cfb-40e8-b923-800e74a0a3a2"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:24.769977665Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1948,"session_id":"ecf9b911-8cfb-40e8-b923-800e74a0a3a2"} {"msg":"78d900ce-f1e1-44eb-9347-bfeb1ea7a6af is now active with session: ecf9b911-8cfb-40e8-b923-800e74a0a3a2","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:24.769992147Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1948,"session_id":"ecf9b911-8cfb-40e8-b923-800e74a0a3a2"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:24.770004289Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1948,"session_id":"ecf9b911-8cfb-40e8-b923-800e74a0a3a2"} {"msg":"The guest has finished waiting for activation with:192","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:24.770016561Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1948} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 41 len 32768 data: 17 19 21 20 18 19 15 13 02/30 Read block 13 len 16384 03/30 Flush 04/30 Write block 229 len 12288 data: 14 12 11 05/30 Write block 274 len 24576 data: 25 23 22 18 13 13 06/30 Read block 593 len 24576 07/30 Write block 533 len 20480 data: 15 17 17 16 15 08/30 Write block 305 len 16384 data: 11 12 16 18 09/30 Flush 10/30 Write block 511 len 4096 data: 13 11/30 Flush 12/30 Write block 379 len 8192 data: 14 13 13/30 Read block 394 len 40960 14/30 Read block 562 len 40960 15/30 Write block 99 len 24576 data: 21 22 22 19 18 18 16/30 Read block 405 len 28672 17/30 Write block 165 len 28672 data: 15 13 12 13 14 14 12 18/30 Read block 190 len 24576 19/30 Write block 183 len 12288 data: 10 9 11 20/30 Read block 103 len 8192 21/30 Read block 141 len 40960 22/30 Read block 214 len 36864 23/30 Read block 117 len 8192 24/30 Read block 62 len 32768 25/30 Read block 517 len 12288 26/30 Read block 346 len 4096 27/30 Read block 351 len 16384 28/30 Read block 300 len 12288 29/30 Read block 96 len 36864 30/30 Read block 268 len 36864 ---------------------------------------------------------------- Crucible gen:192 GIO:true work queues: Upstairs:0 downstairs:19 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1011 Acked Write 2 Done Done Done false 1012 Acked Read 10 Done Done Done false 1013 Acked Read 10 Done Done Done false 1014 Acked Write 6 Done Done Done false 1015 Acked Read 7 Done Done Done false 1016 Acked Write 7 Done Done Done false 1017 Acked Read 6 Done Done Done false 1018 Acked Write 3 Done Done Done false 1019 Acked Read 2 Done Done Done false 1020 Acked Read 10 Done Done Done false 1021 Acked Read 9 Done Done Done false 1022 Acked Read 2 Done Done Done false 1023 Acked Read 8 Done Done Done false 1024 Acked Read 3 Done Done Done false 1025 Acked Read 1 Done Done Done false 1026 Acked Read 4 Done Done Done false 1027 Acked Read 3 Done Done Done false 1028 Acked Read 9 Done Done Done false 1029 Acked Read 9 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 19 19 19 57 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1010 1010 1010 Downstairs last five completed: 1010 1009 1008 1007 1006 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:57:24.793326283Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1948} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 180 180 180 304 304 304 F F F 1 020-039 190 190 190 322 322 322 F F F 2 040-059 192 192 192 324 324 324 F F F 3 060-079 173 173 173 293 293 293 F F F 4 080-099 191 191 191 323 323 323 T T T 5 100-119 191 191 191 323 323 323 T T T 6 120-139 191 191 191 323 323 323 F F F 7 140-159 190 190 190 322 322 322 F F F 8 160-179 190 190 190 322 322 322 T T T 9 180-199 178 178 178 302 302 302 T T T 10 200-219 190 190 190 322 322 322 F F F 11 220-239 192 192 192 325 325 325 F F F 12 240-259 188 188 188 320 320 320 F F F 13 260-279 192 192 192 325 325 325 F F F 14 280-299 191 191 191 323 323 323 F F F 15 300-319 192 192 192 325 325 325 F F F 16 320-339 187 187 187 319 319 319 F F F 17 340-359 190 190 190 322 322 322 F F F 18 360-379 191 191 191 323 323 323 T T T 19 380-399 188 188 188 320 320 320 T T T 20 400-419 188 188 188 320 320 320 F F F 21 420-439 186 186 186 317 317 317 F F F 22 440-459 186 186 186 317 317 317 F F F 23 460-479 188 188 188 320 320 320 F F F 24 480-499 187 187 187 319 319 319 F F F 25 500-519 192 192 192 326 326 326 F F F 26 520-539 192 192 192 325 325 325 F F F 27 540-559 191 191 191 323 323 323 F F F 28 560-579 189 189 189 321 321 321 F F F 29 580-599 190 190 190 322 322 322 F F F Max gen: 192, Max flush: 326 A Difference in extent metadata is expected here On loop 96 Verifying data now No change after verify Loop: 96 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 180 180 180 304 304 304 F F F 1 020-039 190 190 190 322 322 322 F F F 2 040-059 192 192 192 324 324 324 F F F 3 060-079 173 173 173 293 293 293 F F F 4 080-099 193 193 193 327 327 327 F F F 5 100-119 193 193 193 327 327 327 F F F 6 120-139 191 191 191 323 323 323 F F F 7 140-159 190 190 190 322 322 322 F F F 8 160-179 193 193 193 327 327 327 F F F 9 180-199 193 193 193 327 327 327 F F F 10 200-219 190 190 190 322 322 322 F F F 11 220-239 192 192 192 325 325 325 F F F 12 240-259 188 188 188 320 320 320 F F F 13 260-279 192 192 192 325 325 325 F F F 14 280-299 191 191 191 323 323 323 F F F 15 300-319 192 192 192 325 325 325 F F F 16 320-339 187 187 187 319 319 319 F F F 17 340-359 190 190 190 322 322 322 F F F 18 360-379 193 193 193 327 327 327 F F F 19 380-399 193 193 193 327 327 327 F F F 20 400-419 188 188 188 320 320 320 F F F 21 420-439 186 186 186 317 317 317 F F F 22 440-459 186 186 186 317 317 317 F F F 23 460-479 188 188 188 320 320 320 F F F 24 480-499 187 187 187 319 319 319 F F F 25 500-519 192 192 192 326 326 326 F F F 26 520-539 192 192 192 325 325 325 F F F 27 540-559 191 191 191 323 323 323 F F F 28 560-579 189 189 189 321 321 321 F F F 29 580-599 190 190 190 322 322 322 F F F Max gen: 193, Max flush: 327 Begin loop 97 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:29.104523349Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1956,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:57:29.147124628Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1956,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:29.147505257Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1956,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:29.17927172Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1956,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:29.179799778Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1956} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"3818f0fc76fe787c6cc412af30fd59b930203885\",\n git_commit_timestamp: \"2024-11-01T16:08:30.000000000Z\",\n git_branch: \"renovate/hyper-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:29.179834683Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1956} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:29.179848284Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1956} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:29.179862606Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1956} {"msg":"Crucible a26cad03-b9d5-487b-bf1d-99e44ba5b2a7 has session id: fa36e0df-8962-4711-99ce-799468ac8e1e","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:29.180314435Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1956,"session_id":"fa36e0df-8962-4711-99ce-799468ac8e1e"} {"msg":"Upstairs opts: Upstairs UUID: a26cad03-b9d5-487b-bf1d-99e44ba5b2a7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:29.18035516Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1956,"session_id":"fa36e0df-8962-4711-99ce-799468ac8e1e"} {"msg":"Crucible stats registered with UUID: a26cad03-b9d5-487b-bf1d-99e44ba5b2a7","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:29.180379543Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1956,"session_id":"fa36e0df-8962-4711-99ce-799468ac8e1e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:29.210915878Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1956,"task":"crutest"} {"msg":"The guest has requested activation with gen:194","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:29.210945581Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1956} {"msg":"a26cad03-b9d5-487b-bf1d-99e44ba5b2a7 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:29.210979716Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1956,"session_id":"fa36e0df-8962-4711-99ce-799468ac8e1e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:29.210996128Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1956,"client":"0","":"downstairs","session_id":"fa36e0df-8962-4711-99ce-799468ac8e1e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:29.211026042Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1956,"client":"0","":"downstairs","session_id":"fa36e0df-8962-4711-99ce-799468ac8e1e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:29.211043714Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1956,"client":"1","":"downstairs","session_id":"fa36e0df-8962-4711-99ce-799468ac8e1e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:29.211064507Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1956,"client":"1","":"downstairs","session_id":"fa36e0df-8962-4711-99ce-799468ac8e1e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:29.211081929Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1956,"client":"2","":"downstairs","session_id":"fa36e0df-8962-4711-99ce-799468ac8e1e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:29.211098681Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1956,"client":"2","":"downstairs","session_id":"fa36e0df-8962-4711-99ce-799468ac8e1e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:29.211115183Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1956,"":"io task","client":"2","":"downstairs","session_id":"fa36e0df-8962-4711-99ce-799468ac8e1e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:29.211143417Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1956,"":"io task","client":"0","":"downstairs","session_id":"fa36e0df-8962-4711-99ce-799468ac8e1e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:29.211174431Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1956,"":"io task","client":"1","":"downstairs","session_id":"fa36e0df-8962-4711-99ce-799468ac8e1e"} {"msg":"ds_connection connected from Ok(127.0.0.1:34417)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:29.21124615Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1956,"":"io task","client":"0","":"downstairs","session_id":"fa36e0df-8962-4711-99ce-799468ac8e1e"} {"msg":"ds_connection connected from Ok(127.0.0.1:40469)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:29.211446556Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1956,"":"io task","client":"1","":"downstairs","session_id":"fa36e0df-8962-4711-99ce-799468ac8e1e"} {"msg":"ds_connection connected from Ok(127.0.0.1:61326)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:29.211488811Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1956,"":"io task","client":"2","":"downstairs","session_id":"fa36e0df-8962-4711-99ce-799468ac8e1e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:29.211951431Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1956,"client":"1","":"downstairs","session_id":"fa36e0df-8962-4711-99ce-799468ac8e1e"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:29.211989446Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1956,"client":"1","":"downstairs","session_id":"fa36e0df-8962-4711-99ce-799468ac8e1e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:29.212017669Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1956,"client":"2","":"downstairs","session_id":"fa36e0df-8962-4711-99ce-799468ac8e1e"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:29.212044373Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1956,"client":"2","":"downstairs","session_id":"fa36e0df-8962-4711-99ce-799468ac8e1e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:29.212233587Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1956,"client":"0","":"downstairs","session_id":"fa36e0df-8962-4711-99ce-799468ac8e1e"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:29.212275563Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1956,"client":"0","":"downstairs","session_id":"fa36e0df-8962-4711-99ce-799468ac8e1e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:29.212625108Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1956,"client":"1","":"downstairs","session_id":"fa36e0df-8962-4711-99ce-799468ac8e1e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:29.212654622Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1956,"client":"2","":"downstairs","session_id":"fa36e0df-8962-4711-99ce-799468ac8e1e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:29.212669824Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1956,"client":"0","":"downstairs","session_id":"fa36e0df-8962-4711-99ce-799468ac8e1e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:29.213180269Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1956,"client":"1","":"downstairs","session_id":"fa36e0df-8962-4711-99ce-799468ac8e1e"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:29.213207633Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1956,"client":"1","":"downstairs","session_id":"fa36e0df-8962-4711-99ce-799468ac8e1e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:29.213222075Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1956,"session_id":"fa36e0df-8962-4711-99ce-799468ac8e1e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:29.213234586Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1956,"client":"2","":"downstairs","session_id":"fa36e0df-8962-4711-99ce-799468ac8e1e"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:29.213247928Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1956,"client":"2","":"downstairs","session_id":"fa36e0df-8962-4711-99ce-799468ac8e1e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:29.21326116Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1956,"session_id":"fa36e0df-8962-4711-99ce-799468ac8e1e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:29.213273581Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1956,"client":"0","":"downstairs","session_id":"fa36e0df-8962-4711-99ce-799468ac8e1e"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:29.213286803Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1956,"client":"0","":"downstairs","session_id":"fa36e0df-8962-4711-99ce-799468ac8e1e"} {"msg":"[0]R flush_numbers[0..12]: [304, 322, 324, 293, 327, 327, 323, 322, 327, 327, 322, 325]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:29.213301045Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1956,"":"downstairs","":"downstairs","session_id":"fa36e0df-8962-4711-99ce-799468ac8e1e"} {"msg":"[0]R generation[0..12]: [180, 190, 192, 173, 193, 193, 191, 190, 193, 193, 190, 192]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:29.213315107Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1956,"":"downstairs","":"downstairs","session_id":"fa36e0df-8962-4711-99ce-799468ac8e1e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:29.213330349Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1956,"":"downstairs","":"downstairs","session_id":"fa36e0df-8962-4711-99ce-799468ac8e1e"} {"msg":"[1]R flush_numbers[0..12]: [304, 322, 324, 293, 327, 327, 323, 322, 327, 327, 322, 325]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:29.21334456Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1956,"":"downstairs","":"downstairs","session_id":"fa36e0df-8962-4711-99ce-799468ac8e1e"} {"msg":"[1]R generation[0..12]: [180, 190, 192, 173, 193, 193, 191, 190, 193, 193, 190, 192]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:29.213361513Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1956,"":"downstairs","":"downstairs","session_id":"fa36e0df-8962-4711-99ce-799468ac8e1e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:29.213375764Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1956,"":"downstairs","":"downstairs","session_id":"fa36e0df-8962-4711-99ce-799468ac8e1e"} {"msg":"[2]R flush_numbers[0..12]: [304, 322, 324, 293, 327, 327, 323, 322, 327, 327, 322, 325]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:29.213389696Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1956,"":"downstairs","":"downstairs","session_id":"fa36e0df-8962-4711-99ce-799468ac8e1e"} {"msg":"[2]R generation[0..12]: [180, 190, 192, 173, 193, 193, 191, 190, 193, 193, 190, 192]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:29.213403338Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1956,"":"downstairs","":"downstairs","session_id":"fa36e0df-8962-4711-99ce-799468ac8e1e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:29.21341733Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1956,"":"downstairs","":"downstairs","session_id":"fa36e0df-8962-4711-99ce-799468ac8e1e"} {"msg":"Max found gen is 194","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:29.213430371Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1956,"":"downstairs","":"downstairs","session_id":"fa36e0df-8962-4711-99ce-799468ac8e1e"} Wait for a query_work_queue command to finish before sending IO {"msg":"Generation requested: 194 >= found:194","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:29.213453814Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1956,"":"downstairs","":"downstairs","session_id":"fa36e0df-8962-4711-99ce-799468ac8e1e"} {"msg":"Next flush: 328","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:29.213466946Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1956,"":"downstairs","":"downstairs","session_id":"fa36e0df-8962-4711-99ce-799468ac8e1e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:29.213480028Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1956,"":"downstairs","":"downstairs","session_id":"fa36e0df-8962-4711-99ce-799468ac8e1e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:29.21349305Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1956,"session_id":"fa36e0df-8962-4711-99ce-799468ac8e1e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:29.213505081Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1956,"client":"0","":"downstairs","session_id":"fa36e0df-8962-4711-99ce-799468ac8e1e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:29.213518783Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1956,"client":"1","":"downstairs","session_id":"fa36e0df-8962-4711-99ce-799468ac8e1e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:29.213531935Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1956,"client":"2","":"downstairs","session_id":"fa36e0df-8962-4711-99ce-799468ac8e1e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:29.213544976Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1956,"session_id":"fa36e0df-8962-4711-99ce-799468ac8e1e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:29.213557318Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1956,"session_id":"fa36e0df-8962-4711-99ce-799468ac8e1e"} {"msg":"a26cad03-b9d5-487b-bf1d-99e44ba5b2a7 is now active with session: fa36e0df-8962-4711-99ce-799468ac8e1e","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:29.21357192Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1956,"session_id":"fa36e0df-8962-4711-99ce-799468ac8e1e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:29.213584371Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1956,"session_id":"fa36e0df-8962-4711-99ce-799468ac8e1e"} {"msg":"The guest has finished waiting for activation with:194","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:29.213596533Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1956} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 112 len 20480 data: 15 15 14 13 13 03/30 Write block 91 len 16384 data: 23 21 20 16 04/30 Write block 63 len 20480 data: 9 11 12 17 18 05/30 Flush 06/30 Write block 40 len 4096 data: 16 07/30 Write block 593 len 28672 data: 13 11 13 13 12 10 6 08/30 Write block 49 len 40960 data: 16 15 12 14 14 12 11 13 13 11 09/30 Read block 112 len 32768 10/30 Read block 228 len 8192 11/30 Write block 210 len 40960 data: 15 16 17 17 17 17 20 20 20 19 12/30 Read block 450 len 8192 13/30 Write block 249 len 28672 data: 23 25 23 21 19 17 14 14/30 Write block 327 len 32768 data: 9 7 9 11 12 11 12 12 15/30 Write block 378 len 8192 data: 15 15 16/30 Write block 347 len 40960 data: 15 13 15 14 16 17 18 14 14 14 17/30 Write block 192 len 4096 data: 19 18/30 Read block 357 len 32768 19/30 Write block 231 len 28672 data: 12 10 9 12 11 13 11 20/30 Write block 587 len 32768 data: 19 15 19 16 15 13 14 12 21/30 Write block 323 len 20480 data: 12 12 9 11 10 22/30 Write block 285 len 28672 data: 10 15 15 15 15 15 17 23/30 Read block 28 len 28672 24/30 Write block 18 len 20480 data: 16 16 16 11 13 25/30 Read block 491 len 12288 26/30 Read block 287 len 12288 27/30 Read block 162 len 16384 28/30 Write block 223 len 20480 data: 19 18 16 14 14 29/30 Write block 37 len 4096 data: 18 30/30 Read block 22 len 20480 ---------------------------------------------------------------- Crucible gen:194 GIO:true work queues: Upstairs:0 downstairs:25 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1005 Acked Write 1 Done Done Done false 1006 Acked Write 7 Done Done Done false 1007 Acked Write 10 Done Done Done false 1008 Acked Read 8 Done Done Done false 1009 Acked Read 2 Done Done Done false 1010 Acked Write 10 Done Done Done false 1011 Acked Read 2 Done Done Done false 1012 Acked Write 7 Done Done Done false 1013 Acked Write 8 Done Done Done false 1014 Acked Write 2 Done Done Done false 1015 Acked Write 10 Done Done Done false 1016 Acked Write 1 Done Done Done false 1017 Acked Read 8 Done Done Done false 1018 Acked Write 7 Done Done Done false 1019 Acked Write 8 Done Done Done false 1020 Acked Write 5 Done Done Done false 1021 Acked Write 7 Done Done Done false 1022 Acked Read 7 Done Done Done false 1023 Acked Write 5 Done Done Done false 1024 Acked Read 3 Done Done Done false 1025 Acked Read 3 Done Done Done false 1026 Acked Read 4 Done Done Done false 1027 Acked Write 5 Sent Done Done false 1028 Acked Write 1 Sent Done Done false 1029 Acked Read 5 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 0 1 4 Done 22 25 24 71 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1004 1004 1004 Downstairs last five completed: 1004 1003 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:57:29.233575167Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1956} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 180 180 180 304 304 304 T T T 1 020-039 190 190 190 322 322 322 T T T 2 040-059 192 192 192 324 324 324 T T T 3 060-079 194 194 194 329 329 329 F F F 4 080-099 194 194 194 329 329 329 F F F 5 100-119 194 194 194 329 329 329 F F F 6 120-139 191 191 191 323 323 323 F F F 7 140-159 190 190 190 322 322 322 F F F 8 160-179 193 193 193 327 327 327 F F F 9 180-199 193 193 193 327 327 327 T T T 10 200-219 190 190 190 322 322 322 T T T 11 220-239 192 192 192 325 325 325 T T T 12 240-259 188 188 188 320 320 320 T T T 13 260-279 192 192 192 325 325 325 F F F 14 280-299 191 191 191 323 323 323 T T T 15 300-319 192 192 192 325 325 325 F F F 16 320-339 187 187 187 319 319 319 T T T 17 340-359 190 190 190 322 322 322 T T T 18 360-379 193 193 193 327 327 327 T T T 19 380-399 193 193 193 327 327 327 F F F 20 400-419 188 188 188 320 320 320 F F F 21 420-439 186 186 186 317 317 317 F F F 22 440-459 186 186 186 317 317 317 F F F 23 460-479 188 188 188 320 320 320 F F F 24 480-499 187 187 187 319 319 319 F F F 25 500-519 192 192 192 326 326 326 F F F 26 520-539 192 192 192 325 325 325 F F F 27 540-559 191 191 191 323 323 323 F F F 28 560-579 189 189 189 321 321 321 F F F 29 580-599 190 190 190 322 322 322 T T T Max gen: 194, Max flush: 329 A Difference in extent metadata is expected here On loop 97 Verifying data now No change after verify Loop: 97 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 195 195 195 330 330 330 F F F 1 020-039 195 195 195 330 330 330 F F F 2 040-059 195 195 195 330 330 330 F F F 3 060-079 194 194 194 329 329 329 F F F 4 080-099 194 194 194 329 329 329 F F F 5 100-119 194 194 194 329 329 329 F F F 6 120-139 191 191 191 323 323 323 F F F 7 140-159 190 190 190 322 322 322 F F F 8 160-179 193 193 193 327 327 327 F F F 9 180-199 195 195 195 330 330 330 F F F 10 200-219 195 195 195 330 330 330 F F F 11 220-239 195 195 195 330 330 330 F F F 12 240-259 195 195 195 330 330 330 F F F 13 260-279 192 192 192 325 325 325 F F F 14 280-299 195 195 195 330 330 330 F F F 15 300-319 192 192 192 325 325 325 F F F 16 320-339 195 195 195 330 330 330 F F F 17 340-359 195 195 195 330 330 330 F F F 18 360-379 195 195 195 330 330 330 F F F 19 380-399 193 193 193 327 327 327 F F F 20 400-419 188 188 188 320 320 320 F F F 21 420-439 186 186 186 317 317 317 F F F 22 440-459 186 186 186 317 317 317 F F F 23 460-479 188 188 188 320 320 320 F F F 24 480-499 187 187 187 319 319 319 F F F 25 500-519 192 192 192 326 326 326 F F F 26 520-539 192 192 192 325 325 325 F F F 27 540-559 191 191 191 323 323 323 F F F 28 560-579 189 189 189 321 321 321 F F F 29 580-599 195 195 195 330 330 330 F F F Max gen: 195, Max flush: 330 Begin loop 98 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:33.66115578Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1964,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:33.70508167Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1964,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:33.70570058Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1964} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"3818f0fc76fe787c6cc412af30fd59b930203885\",\n git_commit_timestamp: \"2024-11-01T16:08:30.000000000Z\",\n git_branch: \"renovate/hyper-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:33.705749496Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1964} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:33.705767468Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1964} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:33.70578501Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1964} {"msg":"Crucible 881e0e3e-7d44-464a-be72-2c13e2781824 has session id: dc16218e-7dc2-4399-b182-9cdb674bb809","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:33.706225767Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1964,"session_id":"dc16218e-7dc2-4399-b182-9cdb674bb809"} {"msg":"Upstairs opts: Upstairs UUID: 881e0e3e-7d44-464a-be72-2c13e2781824, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:33.706266142Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1964,"session_id":"dc16218e-7dc2-4399-b182-9cdb674bb809"} {"msg":"Crucible stats registered with UUID: 881e0e3e-7d44-464a-be72-2c13e2781824","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:33.706290885Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1964,"session_id":"dc16218e-7dc2-4399-b182-9cdb674bb809"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:33.737205599Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1964,"task":"crutest"} {"msg":"The guest has requested activation with gen:196","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:33.737237663Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1964} {"msg":"881e0e3e-7d44-464a-be72-2c13e2781824 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:33.737304822Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1964,"session_id":"dc16218e-7dc2-4399-b182-9cdb674bb809"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:33.737326604Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1964,"client":"0","":"downstairs","session_id":"dc16218e-7dc2-4399-b182-9cdb674bb809"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:33.737355678Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1964,"client":"0","":"downstairs","session_id":"dc16218e-7dc2-4399-b182-9cdb674bb809"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:33.737377551Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1964,"client":"1","":"downstairs","session_id":"dc16218e-7dc2-4399-b182-9cdb674bb809"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:33.737394953Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1964,"client":"1","":"downstairs","session_id":"dc16218e-7dc2-4399-b182-9cdb674bb809"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:33.737411835Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1964,"client":"2","":"downstairs","session_id":"dc16218e-7dc2-4399-b182-9cdb674bb809"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:33.737437919Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1964,"client":"2","":"downstairs","session_id":"dc16218e-7dc2-4399-b182-9cdb674bb809"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:33.737466422Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1964,"":"io task","client":"2","":"downstairs","session_id":"dc16218e-7dc2-4399-b182-9cdb674bb809"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:33.737496956Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1964,"":"io task","client":"0","":"downstairs","session_id":"dc16218e-7dc2-4399-b182-9cdb674bb809"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:33.73752799Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1964,"":"io task","client":"1","":"downstairs","session_id":"dc16218e-7dc2-4399-b182-9cdb674bb809"} {"msg":"ds_connection connected from Ok(127.0.0.1:55613)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:33.737561805Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1964,"":"io task","client":"2","":"downstairs","session_id":"dc16218e-7dc2-4399-b182-9cdb674bb809"} {"msg":"ds_connection connected from Ok(127.0.0.1:61402)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:33.737754339Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1964,"":"io task","client":"1","":"downstairs","session_id":"dc16218e-7dc2-4399-b182-9cdb674bb809"} {"msg":"ds_connection connected from Ok(127.0.0.1:53215)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:33.737801576Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1964,"":"io task","client":"0","":"downstairs","session_id":"dc16218e-7dc2-4399-b182-9cdb674bb809"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:33.738391132Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1964,"client":"2","":"downstairs","session_id":"dc16218e-7dc2-4399-b182-9cdb674bb809"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:33.738420275Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1964,"client":"2","":"downstairs","session_id":"dc16218e-7dc2-4399-b182-9cdb674bb809"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:33.738434577Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1964,"client":"0","":"downstairs","session_id":"dc16218e-7dc2-4399-b182-9cdb674bb809"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:33.738447959Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1964,"client":"0","":"downstairs","session_id":"dc16218e-7dc2-4399-b182-9cdb674bb809"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:33.738461221Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1964,"client":"1","":"downstairs","session_id":"dc16218e-7dc2-4399-b182-9cdb674bb809"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:33.738478573Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1964,"client":"1","":"downstairs","session_id":"dc16218e-7dc2-4399-b182-9cdb674bb809"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:33.739058908Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1964,"client":"2","":"downstairs","session_id":"dc16218e-7dc2-4399-b182-9cdb674bb809"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:33.739138718Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1964,"client":"1","":"downstairs","session_id":"dc16218e-7dc2-4399-b182-9cdb674bb809"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:33.739166801Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1964,"client":"0","":"downstairs","session_id":"dc16218e-7dc2-4399-b182-9cdb674bb809"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:33.739656805Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1964,"client":"2","":"downstairs","session_id":"dc16218e-7dc2-4399-b182-9cdb674bb809"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:33.739684088Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1964,"client":"2","":"downstairs","session_id":"dc16218e-7dc2-4399-b182-9cdb674bb809"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:33.73969872Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1964,"session_id":"dc16218e-7dc2-4399-b182-9cdb674bb809"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:33.739789322Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1964,"client":"1","":"downstairs","session_id":"dc16218e-7dc2-4399-b182-9cdb674bb809"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:33.739808164Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1964,"client":"1","":"downstairs","session_id":"dc16218e-7dc2-4399-b182-9cdb674bb809"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:33.739821526Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1964,"session_id":"dc16218e-7dc2-4399-b182-9cdb674bb809"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:33.739833698Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1964,"client":"0","":"downstairs","session_id":"dc16218e-7dc2-4399-b182-9cdb674bb809"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:33.739846839Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1964,"client":"0","":"downstairs","session_id":"dc16218e-7dc2-4399-b182-9cdb674bb809"} {"msg":"[0]R flush_numbers[0..12]: [330, 330, 330, 329, 329, 329, 323, 322, 327, 330, 330, 330]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:33.739862221Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1964,"":"downstairs","":"downstairs","session_id":"dc16218e-7dc2-4399-b182-9cdb674bb809"} {"msg":"[0]R generation[0..12]: [195, 195, 195, 194, 194, 194, 191, 190, 193, 195, 195, 195]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:33.739876273Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1964,"":"downstairs","":"downstairs","session_id":"dc16218e-7dc2-4399-b182-9cdb674bb809"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:33.739890375Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1964,"":"downstairs","":"downstairs","session_id":"dc16218e-7dc2-4399-b182-9cdb674bb809"} {"msg":"[1]R flush_numbers[0..12]: [330, 330, 330, 329, 329, 329, 323, 322, 327, 330, 330, 330]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:33.739907397Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1964,"":"downstairs","":"downstairs","session_id":"dc16218e-7dc2-4399-b182-9cdb674bb809"} {"msg":"[1]R generation[0..12]: [195, 195, 195, 194, 194, 194, 191, 190, 193, 195, 195, 195]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:33.739921409Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1964,"":"downstairs","":"downstairs","session_id":"dc16218e-7dc2-4399-b182-9cdb674bb809"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:33.739935231Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1964,"":"downstairs","":"downstairs","session_id":"dc16218e-7dc2-4399-b182-9cdb674bb809"} {"msg":"[2]R flush_numbers[0..12]: [330, 330, 330, 329, 329, 329, 323, 322, 327, 330, 330, 330]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:33.739948922Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1964,"":"downstairs","":"downstairs","session_id":"dc16218e-7dc2-4399-b182-9cdb674bb809"} {"msg":"[2]R generation[0..12]: [195, 195, 195, 194, 194, 194, 191, 190, 193, 195, 195, 195]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:33.739962514Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1964,"":"downstairs","":"downstairs","session_id":"dc16218e-7dc2-4399-b182-9cdb674bb809"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:33.739976346Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1964,"":"downstairs","":"downstairs","session_id":"dc16218e-7dc2-4399-b182-9cdb674bb809"} {"msg":"Max found gen is 196","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:33.739989258Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1964,"":"downstairs","":"downstairs","session_id":"dc16218e-7dc2-4399-b182-9cdb674bb809"} {"msg":"Generation requested: 196 >= found:196","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:33.740002569Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1964,"":"downstairs","":"downstairs","session_id":"dc16218e-7dc2-4399-b182-9cdb674bb809"} {"msg":"Next flush: 331","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:33.740015671Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1964,"":"downstairs","":"downstairs","session_id":"dc16218e-7dc2-4399-b182-9cdb674bb809"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:33.740028512Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1964,"":"downstairs","":"downstairs","session_id":"dc16218e-7dc2-4399-b182-9cdb674bb809"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:33.740041954Z","hostname":"Wait for a query_work_queue command to finish before sending IO w-01JBMA2T0XD4W63AXFF321JBBP","pid":1964,"session_id":"dc16218e-7dc2-4399-b182-9cdb674bb809"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:33.740063377Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1964,"client":"0","":"downstairs","session_id":"dc16218e-7dc2-4399-b182-9cdb674bb809"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:33.740076829Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1964,"client":"1","":"downstairs","session_id":"dc16218e-7dc2-4399-b182-9cdb674bb809"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:33.740090111Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1964,"client":"2","":"downstairs","session_id":"dc16218e-7dc2-4399-b182-9cdb674bb809"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:33.740103702Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1964,"session_id":"dc16218e-7dc2-4399-b182-9cdb674bb809"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:33.740117884Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1964,"session_id":"dc16218e-7dc2-4399-b182-9cdb674bb809"} {"msg":"881e0e3e-7d44-464a-be72-2c13e2781824 is now active with session: dc16218e-7dc2-4399-b182-9cdb674bb809","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:33.740130846Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1964,"session_id":"dc16218e-7dc2-4399-b182-9cdb674bb809"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:33.740143607Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1964,"session_id":"dc16218e-7dc2-4399-b182-9cdb674bb809"} {"msg":"The guest has finished waiting for activation with:196","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:33.740155739Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1964} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 88 len 16384 data: 18 21 25 24 02/30 Read block 101 len 36864 03/30 Write block 233 len 16384 data: 10 13 12 14 04/30 Read block 55 len 8192 05/30 Read block 131 len 16384 06/30 Write block 552 len 8192 data: 15 15 07/30 Flush 08/30 Write block 332 len 12288 data: 12 13 13 09/30 Write block 364 len 32768 data: 11 12 10 10 9 11 10 8 10/30 Read block 92 len 4096 11/30 Read block 246 len 36864 12/30 Read block 460 len 12288 13/30 Read block 165 len 32768 14/30 Write block 218 len 8192 data: 21 20 15/30 Write block 7 len 36864 data: 14 16 16 15 16 18 17 18 18 16/30 Read block 111 len 8192 17/30 Read block 469 len 36864 18/30 Read block 4 len 36864 19/30 Read block 344 len 20480 20/30 Read block 401 len 8192 21/30 Read block 565 len 36864 22/30 Read block 33 len 40960 23/30 Write block 462 len 4096 data: 16 24/30 Flush 25/30 Write block 70 len 32768 data: 23 21 21 17 18 17 11 9 26/30 Write block 595 len 12288 data: 14 14 13 27/30 Read block 468 len 8192 28/30 Write block 214 len 8192 data: 18 18 29/30 Read block 170 len 40960 30/30 Write block 393 len 16384 data: 13 13 15 16 ---------------------------------------------------------------- Crucible gen:196 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 8 Done Done Done false 1025 Acked Write 3 Done Done Done false 1026 Acked Read 2 Done Done Done false 1027 Acked Write 2 Done Done Done false 1028 Acked Read 10 Done Done Done false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:57:33.762855194Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1964} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 196 196 196 332 332 332 F F F 1 020-039 195 195 195 330 330 330 F F F 2 040-059 195 195 195 330 330 330 F F F 3 060-079 194 194 194 329 329 329 T T T 4 080-099 196 196 196 331 331 331 F F F 5 100-119 194 194 194 329 329 329 F F F 6 120-139 191 191 191 323 323 323 F F F 7 140-159 190 190 190 322 322 322 F F F 8 160-179 193 193 193 327 327 327 F F F 9 180-199 195 195 195 330 330 330 F F F 10 200-219 196 196 196 332 332 332 T T T 11 220-239 196 196 196 331 331 331 F F F 12 240-259 195 195 195 330 330 330 F F F 13 260-279 192 192 192 325 325 325 F F F 14 280-299 195 195 195 330 330 330 F F F 15 300-319 192 192 192 325 325 325 F F F 16 320-339 196 196 196 332 332 332 F F F 17 340-359 195 195 195 330 330 330 F F F 18 360-379 196 196 196 332 332 332 F F F 19 380-399 193 193 193 327 327 327 T T T 20 400-419 188 188 188 320 320 320 F F F 21 420-439 186 186 186 317 317 317 F F F 22 440-459 186 186 186 317 317 317 F F F 23 460-479 196 196 196 332 332 332 F F F 24 480-499 187 187 187 319 319 319 F F F 25 500-519 192 192 192 326 326 326 F F F 26 520-539 192 192 192 325 325 325 F F F 27 540-559 196 196 196 331 331 331 F F F 28 560-579 189 189 189 321 321 321 F F F 29 580-599 195 195 195 330 330 330 T T T Max gen: 196, Max flush: 332 A Difference in extent metadata is expected here On loop 98 Verifying data now No change after verify Loop: 98 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 196 196 196 332 332 332 F F F 1 020-039 195 195 195 330 330 330 F F F 2 040-059 195 195 195 330 330 330 F F F 3 060-079 197 197 197 333 333 333 F F F 4 080-099 196 196 196 331 331 331 F F F 5 100-119 194 194 194 329 329 329 F F F 6 120-139 191 191 191 323 323 323 F F F 7 140-159 190 190 190 322 322 322 F F F 8 160-179 193 193 193 327 327 327 F F F 9 180-199 195 195 195 330 330 330 F F F 10 200-219 197 197 197 333 333 333 F F F 11 220-239 196 196 196 331 331 331 F F F 12 240-259 195 195 195 330 330 330 F F F 13 260-279 192 192 192 325 325 325 F F F 14 280-299 195 195 195 330 330 330 F F F 15 300-319 192 192 192 325 325 325 F F F 16 320-339 196 196 196 332 332 332 F F F 17 340-359 195 195 195 330 330 330 F F F 18 360-379 196 196 196 332 332 332 F F F 19 380-399 197 197 197 333 333 333 F F F 20 400-419 188 188 188 320 320 320 F F F 21 420-439 186 186 186 317 317 317 F F F 22 440-459 186 186 186 317 317 317 F F F 23 460-479 196 196 196 332 332 332 F F F 24 480-499 187 187 187 319 319 319 F F F 25 500-519 192 192 192 326 326 326 F F F 26 520-539 192 192 192 325 325 325 F F F 27 540-559 196 196 196 331 331 331 F F F 28 560-579 189 189 189 321 321 321 F F F 29 580-599 197 197 197 333 333 333 F F F Max gen: 197, Max flush: 333 Begin loop 99 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:38.058795403Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1972,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:38.102587036Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1972,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:38.103121155Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1972} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"3818f0fc76fe787c6cc412af30fd59b930203885\",\n git_commit_timestamp: \"2024-11-01T16:08:30.000000000Z\",\n git_branch: \"renovate/hyper-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:38.10316118Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1972} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:38.103179952Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1972} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:38.103194824Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1972} {"msg":"Crucible e07bfae7-0f5e-4692-a666-f649f76238e0 has session id: 9c007b1c-fc78-43d3-8563-2df524c3ded8","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:38.103640411Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1972,"session_id":"9c007b1c-fc78-43d3-8563-2df524c3ded8"} {"msg":"Upstairs opts: Upstairs UUID: e07bfae7-0f5e-4692-a666-f649f76238e0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:38.103682457Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1972,"session_id":"9c007b1c-fc78-43d3-8563-2df524c3ded8"} {"msg":"Crucible stats registered with UUID: e07bfae7-0f5e-4692-a666-f649f76238e0","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:38.10370683Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1972,"session_id":"9c007b1c-fc78-43d3-8563-2df524c3ded8"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:38.134599151Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1972,"task":"crutest"} {"msg":"The guest has requested activation with gen:198","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:38.134632925Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1972} {"msg":"e07bfae7-0f5e-4692-a666-f649f76238e0 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:38.134668609Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1972,"session_id":"9c007b1c-fc78-43d3-8563-2df524c3ded8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:38.134686222Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1972,"client":"0","":"downstairs","session_id":"9c007b1c-fc78-43d3-8563-2df524c3ded8"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:38.134714505Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1972,"client":"0","":"downstairs","session_id":"9c007b1c-fc78-43d3-8563-2df524c3ded8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:38.134731788Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1972,"client":"1","":"downstairs","session_id":"9c007b1c-fc78-43d3-8563-2df524c3ded8"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:38.13474869Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1972,"client":"1","":"downstairs","session_id":"9c007b1c-fc78-43d3-8563-2df524c3ded8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:38.134767892Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1972,"client":"2","":"downstairs","session_id":"9c007b1c-fc78-43d3-8563-2df524c3ded8"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:38.134789825Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1972,"client":"2","":"downstairs","session_id":"9c007b1c-fc78-43d3-8563-2df524c3ded8"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:38.13482759Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1972,"":"io task","client":"2","":"downstairs","session_id":"9c007b1c-fc78-43d3-8563-2df524c3ded8"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:38.134862404Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1972,"":"io task","client":"0","":"downstairs","session_id":"9c007b1c-fc78-43d3-8563-2df524c3ded8"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:38.134882797Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1972,"":"io task","client":"1","":"downstairs","session_id":"9c007b1c-fc78-43d3-8563-2df524c3ded8"} {"msg":"ds_connection connected from Ok(127.0.0.1:33625)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:38.134914101Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1972,"":"io task","client":"0","":"downstairs","session_id":"9c007b1c-fc78-43d3-8563-2df524c3ded8"} {"msg":"ds_connection connected from Ok(127.0.0.1:47470)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:38.134950376Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1972,"":"io task","client":"1","":"downstairs","session_id":"9c007b1c-fc78-43d3-8563-2df524c3ded8"} {"msg":"ds_connection connected from Ok(127.0.0.1:42258)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:38.135081182Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1972,"":"io task","client":"2","":"downstairs","session_id":"9c007b1c-fc78-43d3-8563-2df524c3ded8"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:38.135714124Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1972,"client":"0","":"downstairs","session_id":"9c007b1c-fc78-43d3-8563-2df524c3ded8"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:38.135743768Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1972,"client":"0","":"downstairs","session_id":"9c007b1c-fc78-43d3-8563-2df524c3ded8"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:38.13575936Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1972,"client":"1","":"downstairs","session_id":"9c007b1c-fc78-43d3-8563-2df524c3ded8"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:38.135773312Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1972,"client":"1","":"downstairs","session_id":"9c007b1c-fc78-43d3-8563-2df524c3ded8"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:38.135825758Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1972,"client":"2","":"downstairs","session_id":"9c007b1c-fc78-43d3-8563-2df524c3ded8"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:38.135842851Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1972,"client":"2","":"downstairs","session_id":"9c007b1c-fc78-43d3-8563-2df524c3ded8"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:38.136375219Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1972,"client":"1","":"downstairs","session_id":"9c007b1c-fc78-43d3-8563-2df524c3ded8"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:38.136405043Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1972,"client":"0","":"downstairs","session_id":"9c007b1c-fc78-43d3-8563-2df524c3ded8"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:38.136423826Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1972,"client":"2","":"downstairs","session_id":"9c007b1c-fc78-43d3-8563-2df524c3ded8"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:38.136963165Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1972,"client":"1","":"downstairs","session_id":"9c007b1c-fc78-43d3-8563-2df524c3ded8"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:38.13699934Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1972,"client":"1","":"downstairs","session_id":"9c007b1c-fc78-43d3-8563-2df524c3ded8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:38.137026963Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1972,"session_id":"9c007b1c-fc78-43d3-8563-2df524c3ded8"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:38.137050646Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1972,"client":"2","":"downstairs","session_id":"9c007b1c-fc78-43d3-8563-2df524c3ded8"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:38.137066168Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1972,"client":"2","":"downstairs","session_id":"9c007b1c-fc78-43d3-8563-2df524c3ded8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:38.13707972Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1972,"session_id":"9c007b1c-fc78-43d3-8563-2df524c3ded8"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:38.137091972Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1972,"client":"0","":"downstairs","session_id":"9c007b1c-fc78-43d3-8563-2df524c3ded8"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:38.137105243Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1972,"client":"0","":"downstairs","session_id":"9c007b1c-fc78-43d3-8563-2df524c3ded8"} {"msg":"[0]R flush_numbers[0..12]: [332, 330, 330, 333, 331, 329, 323, 322, 327, 330, 333, 331]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:38.137119295Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1972,"":"downstairs","":"downstairs","session_id":"9c007b1c-fc78-43d3-8563-2df524c3ded8"} {"msg":"[0]R generation[0..12]: [196, 195, 195, 197, 196, 194, 191, 190, 193, 195, 197, 196]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:38.137133367Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1972,"":"downstairs","":"downstairs","session_id":"9c007b1c-fc78-43d3-8563-2df524c3ded8"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:38.137147489Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1972,"":"downstairs","":"downstairs","session_id":"9c007b1c-fc78-43d3-8563-2df524c3ded8"} {"msg":"[1]R flush_numbers[0..12]: [332, 330, 330, 333, 331, 329, 323, 322, 327, 330, 333, 331]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:38.137161301Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1972,"":"downstairs","":"downstairs","session_id":"9c007b1c-fc78-43d3-8563-2df524c3ded8"} {"msg":"[1]R generation[0..12]: [196, 195, 195, 197, 196, 194, 191, 190, 193, 195, 197, 196]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:38.137175082Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1972,"":"downstairs","":"downstairs","session_id":"9c007b1c-fc78-43d3-8563-2df524c3ded8"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:38.137196365Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1972,"":"downstairs","":"downstairs","session_id":"9c007b1c-fc78-43d3-8563-2df524c3ded8"} Wait for a query_work_queue command to finish before sending IO {"msg":"[2]R flush_numbers[0..12]: [332, 330, 330, 333, 331, 329, 323, 322, 327, 330, 333, 331]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:38.13723343Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1972,"":"downstairs","":"downstairs","session_id":"9c007b1c-fc78-43d3-8563-2df524c3ded8"} {"msg":"[2]R generation[0..12]: [196, 195, 195, 197, 196, 194, 191, 190, 193, 195, 197, 196]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:38.137256583Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1972,"":"downstairs","":"downstairs","session_id":"9c007b1c-fc78-43d3-8563-2df524c3ded8"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:38.137271365Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1972,"":"downstairs","":"downstairs","session_id":"9c007b1c-fc78-43d3-8563-2df524c3ded8"} {"msg":"Max found gen is 198","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:38.137284566Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1972,"":"downstairs","":"downstairs","session_id":"9c007b1c-fc78-43d3-8563-2df524c3ded8"} {"msg":"Generation requested: 198 >= found:198","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:38.137297828Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1972,"":"downstairs","":"downstairs","session_id":"9c007b1c-fc78-43d3-8563-2df524c3ded8"} {"msg":"Next flush: 334","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:38.13731099Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1972,"":"downstairs","":"downstairs","session_id":"9c007b1c-fc78-43d3-8563-2df524c3ded8"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:38.137324171Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1972,"":"downstairs","":"downstairs","session_id":"9c007b1c-fc78-43d3-8563-2df524c3ded8"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:38.137338243Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1972,"session_id":"9c007b1c-fc78-43d3-8563-2df524c3ded8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:38.137350475Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1972,"client":"0","":"downstairs","session_id":"9c007b1c-fc78-43d3-8563-2df524c3ded8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:38.137363707Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1972,"client":"1","":"downstairs","session_id":"9c007b1c-fc78-43d3-8563-2df524c3ded8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:38.137376818Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1972,"client":"2","":"downstairs","session_id":"9c007b1c-fc78-43d3-8563-2df524c3ded8"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:38.13739087Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1972,"session_id":"9c007b1c-fc78-43d3-8563-2df524c3ded8"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:38.137403282Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1972,"session_id":"9c007b1c-fc78-43d3-8563-2df524c3ded8"} {"msg":"e07bfae7-0f5e-4692-a666-f649f76238e0 is now active with session: 9c007b1c-fc78-43d3-8563-2df524c3ded8","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:38.137416073Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1972,"session_id":"9c007b1c-fc78-43d3-8563-2df524c3ded8"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2024-11-01T16:57:38.137428065Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1972,"session_id":"9c007b1c-fc78-43d3-8563-2df524c3ded8"} {"msg":"The guest has finished waiting for activation with:198","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:57:38.137446217Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1972} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 324 len 4096 data: 13 02/30 Read block 346 len 16384 03/30 Write block 591 len 20480 data: 16 14 15 13 15 04/30 Read block 449 len 36864 05/30 Write block 340 len 40960 data: 12 8 10 12 13 13 14 16 14 16 06/30 Read block 142 len 28672 07/30 Read block 359 len 36864 08/30 Write block 548 len 8192 data: 9 10 09/30 Read block 407 len 20480 10/30 Write block 151 len 28672 data: 18 20 17 19 17 18 17 11/30 Write block 473 len 12288 data: 22 19 18 12/30 Write block 307 len 4096 data: 17 13/30 Write block 194 len 28672 data: 23 22 19 18 16 18 16 14/30 Flush 15/30 Write block 236 len 24576 data: 15 12 10 11 11 13 16/30 Flush 17/30 Write block 551 len 8192 data: 14 16 18/30 Read block 393 len 4096 19/30 Read block 271 len 16384 20/30 Read block 31 len 4096 21/30 Read block 43 len 24576 22/30 Read block 586 len 32768 23/30 Read block 131 len 40960 24/30 Read block 67 len 4096 25/30 Write block 382 len 24576 data: 17 17 16 15 14 12 26/30 Read block 100 len 4096 27/30 Write block 78 len 32768 data: 10 15 15 14 17 15 14 17 28/30 Write block 158 len 16384 data: 16 16 17 15 29/30 Write block 568 len 36864 data: 10 12 9 13 11 10 9 12 10 30/30 Read block 245 len 12288 ---------------------------------------------------------------- Crucible gen:198 GIO:true work queues: Upstairs:0 downstairs:14 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1016 Acked Write 2 Done Done Done false 1017 Acked Read 1 Done Done Done false 1018 Acked Read 4 Done Done Done false 1019 Acked Read 1 Done Done Done false 1020 Acked Read 6 Done Done Done false 1021 Acked Read 8 Done Done Done false 1022 Acked Read 10 Done Done Done false 1023 Acked Read 1 Done Done Done false 1024 Acked Write 6 Done Done Done false 1025 Acked Read 1 Done Done Done false 1026 Acked Write 8 Done Done Done false 1027 Acked Write 4 Done Done Done false 1028 Acked Write 9 Done Done Done false 1029 Acked Read 3 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 14 14 13 41 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1015 1015 1015 Downstairs last five completed: 1015 1014 1013 1012 1011 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:57:38.159563297Z","hostname":"w-01JBMA2T0XD4W63AXFF321JBBP","pid":1972} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 196 196 196 332 332 332 F F F 1 020-039 195 195 195 330 330 330 F F F 2 040-059 195 195 195 330 330 330 F F F 3 060-079 197 197 197 333 333 333 T T T 4 080-099 196 196 196 331 331 331 T T T 5 100-119 194 194 194 329 329 329 F F F 6 120-139 191 191 191 323 323 323 F F F 7 140-159 198 198 198 334 334 334 T T T 8 160-179 193 193 193 327 327 327 T T T 9 180-199 198 198 198 334 334 334 F F F 10 200-219 198 198 198 334 334 334 F F F 11 220-239 198 198 198 335 335 335 F F F 12 240-259 198 198 198 335 335 335 F F F 13 260-279 192 192 192 325 325 325 F F F 14 280-299 195 195 195 330 330 330 F F F 15 300-319 198 198 198 334 334 334 F F F 16 320-339 198 198 198 334 334 334 F F F 17 340-359 198 198 198 334 334 334 F F F 18 360-379 196 196 196 332 332 332 F F F 19 380-399 197 197 197 333 333 333 T T T 20 400-419 188 188 188 320 320 320 F F F 21 420-439 186 186 186 317 317 317 F F F 22 440-459 186 186 186 317 317 317 F F F 23 460-479 198 198 198 334 334 334 F F F 24 480-499 187 187 187 319 319 319 F F F 25 500-519 192 192 192 326 326 326 F F F 26 520-539 192 192 192 325 325 325 F F F 27 540-559 198 198 198 334 334 334 T T T 28 560-579 189 189 189 321 321 321 T T T 29 580-599 198 198 198 334 334 334 F F F Max gen: 198, Max flush: 335 A Difference in extent metadata is expected here On loop 99 Verifying data now No change after verify Loop: 99 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 196 196 196 332 332 332 F F F 1 020-039 195 195 195 330 330 330 F F F 2 040-059 195 195 195 330 330 330 F F F 3 060-079 199 199 199 336 336 336 F F F 4 080-099 199 199 199 336 336 336 F F F 5 100-119 194 194 194 329 329 329 F F F 6 120-139 191 191 191 323 323 323 F F F 7 140-159 199 199 199 336 336 336 F F F 8 160-179 199 199 199 336 336 336 F F F 9 180-199 198 198 198 334 334 334 F F F 10 200-219 198 198 198 334 334 334 F F F 11 220-239 198 198 198 335 335 335 F F F 12 240-259 198 198 198 335 335 335 F F F 13 260-279 192 192 192 325 325 325 F F F 14 280-299 195 195 195 330 330 330 F F F 15 300-319 198 198 198 334 334 334 F F F 16 320-339 198 198 198 334 334 334 F F F 17 340-359 198 198 198 334 334 334 F F F 18 360-379 196 196 196 332 332 332 F F F 19 380-399 199 199 199 336 336 336 F F F 20 400-419 188 188 188 320 320 320 F F F 21 420-439 186 186 186 317 317 317 F F F 22 440-459 186 186 186 317 317 317 F F F 23 460-479 198 198 198 334 334 334 F F F 24 480-499 187 187 187 319 319 319 F F F 25 500-519 192 192 192 326 326 326 F F F 26 520-539 192 192 192 325 325 325 F F F 27 540-559 199 199 199 336 336 336 F F F 28 560-579 199 199 199 336 336 336 F F F 29 580-599 198 198 198 334 334 334 F F F Max gen: 199, Max flush: 336 7:26 Test duration Test completed real 7:25.956544245 user 56.509143069 sys 18.817544634 trap 0.070603926 tflt 0.035421542 dflt 0.107237334 kflt 0.006277903 lock 11:57:08.192885138 slp 36:19.974533646 lat 5.721826117 stop 0.061723327